@indigoai-us/hq-cloud 6.15.75 → 6.15.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +12 -2
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +4 -0
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/bin/sync-runner.test.js +95 -4
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/index.test.d.ts +2 -0
- package/dist/index.test.d.ts.map +1 -0
- package/dist/index.test.js +16 -0
- package/dist/index.test.js.map +1 -0
- package/dist/operation-lock.d.ts +31 -28
- package/dist/operation-lock.d.ts.map +1 -1
- package/dist/operation-lock.js +206 -50
- package/dist/operation-lock.js.map +1 -1
- package/dist/operation-lock.test.js +105 -1
- package/dist/operation-lock.test.js.map +1 -1
- package/dist/sync/area-resolver.d.ts +66 -0
- package/dist/sync/area-resolver.d.ts.map +1 -0
- package/dist/sync/area-resolver.js +306 -0
- package/dist/sync/area-resolver.js.map +1 -0
- package/dist/sync/area-resolver.test.d.ts +2 -0
- package/dist/sync/area-resolver.test.d.ts.map +1 -0
- package/dist/sync/area-resolver.test.js +211 -0
- package/dist/sync/area-resolver.test.js.map +1 -0
- package/dist/sync/event-sync.d.ts +5 -1
- package/dist/sync/event-sync.d.ts.map +1 -1
- package/dist/sync/event-sync.js +3 -0
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/event-sync.test.js +45 -0
- package/dist/sync/event-sync.test.js.map +1 -1
- package/dist/sync/index.d.ts +4 -0
- package/dist/sync/index.d.ts.map +1 -1
- package/dist/sync/index.js +2 -0
- package/dist/sync/index.js.map +1 -1
- package/dist/sync/parallel-sync-coordinator.d.ts +77 -0
- package/dist/sync/parallel-sync-coordinator.d.ts.map +1 -0
- package/dist/sync/parallel-sync-coordinator.js +287 -0
- package/dist/sync/parallel-sync-coordinator.js.map +1 -0
- package/dist/sync/parallel-sync-coordinator.test.d.ts +2 -0
- package/dist/sync/parallel-sync-coordinator.test.d.ts.map +1 -0
- package/dist/sync/parallel-sync-coordinator.test.js +281 -0
- package/dist/sync/parallel-sync-coordinator.test.js.map +1 -0
- package/dist/sync/path-leases.d.ts +31 -0
- package/dist/sync/path-leases.d.ts.map +1 -0
- package/dist/sync/path-leases.js +74 -0
- package/dist/sync/path-leases.js.map +1 -0
- package/dist/sync/path-leases.test.d.ts +2 -0
- package/dist/sync/path-leases.test.d.ts.map +1 -0
- package/dist/sync/path-leases.test.js +51 -0
- package/dist/sync/path-leases.test.js.map +1 -0
- package/dist/sync/state-store.d.ts +7 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +104 -43
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +62 -1
- package/dist/sync/state-store.test.js.map +1 -1
- package/dist/sync/sync-work.d.ts +64 -0
- package/dist/sync/sync-work.d.ts.map +1 -0
- package/dist/sync/sync-work.js +11 -0
- package/dist/sync/sync-work.js.map +1 -0
- package/dist/watcher.d.ts +65 -0
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +280 -24
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +401 -3
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
package/dist/watcher.js
CHANGED
|
@@ -14,7 +14,7 @@ import { createIgnoreFilter } from "./ignore.js";
|
|
|
14
14
|
import { isCoveredByRecursiveRoot, planWatchRoots, } from "./watch-roots.js";
|
|
15
15
|
import { localPathForVaultKey, vaultKeyForLocalPath } from "./local-path-codec.js";
|
|
16
16
|
import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
|
|
17
|
-
import { StateStore } from "./sync/state-store.js";
|
|
17
|
+
import { StateStore, StateStoreLockError } from "./sync/state-store.js";
|
|
18
18
|
import { AGENCY_SYNC_REL, computePersonalCompanySubdirs, CONTINUITY_POINTER_REL, isCurrentContinuityThreadPath, SESSION_LOGS_SYNC_REL, PERSONAL_VAULT_EXCLUDED_TOP_LEVEL, } from "./personal-vault.js";
|
|
19
19
|
import { getPublishFailureOutcome, getPublishSuccessOutcome, serializePublishError, } from "./sync/push-transport.js";
|
|
20
20
|
import { emitCloudTelemetry, } from "./telemetry-events.js";
|
|
@@ -30,8 +30,18 @@ const DEBOUNCE_MS = 2000;
|
|
|
30
30
|
*/
|
|
31
31
|
export const DEFAULT_EVENT_DEBOUNCE_MS = 15_000;
|
|
32
32
|
export const DEFAULT_EVENT_MAX_WAIT_MS = 120_000;
|
|
33
|
+
/**
|
|
34
|
+
* A small, quiet save should not pay the full burst-coalescing window. This
|
|
35
|
+
* is deliberately still long enough to absorb editor atomic-save chatter.
|
|
36
|
+
*/
|
|
37
|
+
export const DEFAULT_EVENT_SMALL_BATCH_DEBOUNCE_MS = 2_500;
|
|
38
|
+
export const DEFAULT_EVENT_SMALL_BATCH_MAX_PATHS = 8;
|
|
39
|
+
export const DEFAULT_EVENT_SMALL_BATCH_MAX_BYTES = 1024 * 1024;
|
|
33
40
|
export const EVENT_DEBOUNCE_MS_ENV = "HQ_SYNC_EVENT_DEBOUNCE_MS";
|
|
34
41
|
export const EVENT_MAX_WAIT_MS_ENV = "HQ_SYNC_EVENT_MAX_WAIT_MS";
|
|
42
|
+
export const EVENT_SMALL_BATCH_DEBOUNCE_MS_ENV = "HQ_SYNC_EVENT_SMALL_BATCH_DEBOUNCE_MS";
|
|
43
|
+
export const EVENT_SMALL_BATCH_MAX_PATHS_ENV = "HQ_SYNC_EVENT_SMALL_BATCH_MAX_PATHS";
|
|
44
|
+
export const EVENT_SMALL_BATCH_MAX_BYTES_ENV = "HQ_SYNC_EVENT_SMALL_BATCH_MAX_BYTES";
|
|
35
45
|
/**
|
|
36
46
|
* Resolve the event-pass debounce configuration from the environment.
|
|
37
47
|
* `HQ_SYNC_EVENT_DEBOUNCE_MS` / `HQ_SYNC_EVENT_MAX_WAIT_MS`; 0, negative,
|
|
@@ -51,7 +61,15 @@ export function resolveEventDebounceConfig(env = process.env) {
|
|
|
51
61
|
};
|
|
52
62
|
const debounceMs = read(EVENT_DEBOUNCE_MS_ENV, DEFAULT_EVENT_DEBOUNCE_MS);
|
|
53
63
|
const maxWaitMs = Math.max(debounceMs, read(EVENT_MAX_WAIT_MS_ENV, DEFAULT_EVENT_MAX_WAIT_MS));
|
|
54
|
-
return {
|
|
64
|
+
return {
|
|
65
|
+
debounceMs,
|
|
66
|
+
maxWaitMs,
|
|
67
|
+
// Never let a tuning error turn the small-batch path into a longer delay
|
|
68
|
+
// than the ordinary debounce window.
|
|
69
|
+
smallBatchDebounceMs: Math.min(debounceMs, read(EVENT_SMALL_BATCH_DEBOUNCE_MS_ENV, DEFAULT_EVENT_SMALL_BATCH_DEBOUNCE_MS)),
|
|
70
|
+
smallBatchMaxPaths: read(EVENT_SMALL_BATCH_MAX_PATHS_ENV, DEFAULT_EVENT_SMALL_BATCH_MAX_PATHS),
|
|
71
|
+
smallBatchMaxBytes: read(EVENT_SMALL_BATCH_MAX_BYTES_ENV, DEFAULT_EVENT_SMALL_BATCH_MAX_BYTES),
|
|
72
|
+
};
|
|
55
73
|
}
|
|
56
74
|
/**
|
|
57
75
|
* A Linux chokidar watcher derives its directory budget from the host's real
|
|
@@ -907,6 +925,9 @@ function estimatePendingEntryBytes(absolutePath, relativePath) {
|
|
|
907
925
|
export class TreeWatcher {
|
|
908
926
|
hqRoot;
|
|
909
927
|
debounceMs;
|
|
928
|
+
smallBatchDebounceMs;
|
|
929
|
+
smallBatchMaxPaths;
|
|
930
|
+
smallBatchMaxBytes;
|
|
910
931
|
maxWaitMs;
|
|
911
932
|
clock;
|
|
912
933
|
shouldEmit;
|
|
@@ -935,6 +956,8 @@ export class TreeWatcher {
|
|
|
935
956
|
/** Paths accumulated for the current (in-flight) debounce window. */
|
|
936
957
|
pending = new Map();
|
|
937
958
|
pendingChanges = new Map();
|
|
959
|
+
/** Latest known payload bytes per path; null makes the whole batch bulk. */
|
|
960
|
+
pendingPayloadSizes = new Map();
|
|
938
961
|
knownKinds = new Map();
|
|
939
962
|
pendingBytes = 0;
|
|
940
963
|
overflowed = false;
|
|
@@ -956,6 +979,12 @@ export class TreeWatcher {
|
|
|
956
979
|
constructor(opts) {
|
|
957
980
|
this.hqRoot = opts.hqRoot;
|
|
958
981
|
this.debounceMs = opts.debounceMs ?? DEBOUNCE_MS;
|
|
982
|
+
this.smallBatchDebounceMs =
|
|
983
|
+
opts.smallBatchDebounceMs !== undefined && opts.smallBatchDebounceMs > 0
|
|
984
|
+
? Math.min(this.debounceMs, Math.floor(opts.smallBatchDebounceMs))
|
|
985
|
+
: null;
|
|
986
|
+
this.smallBatchMaxPaths = Math.max(1, Math.floor(opts.smallBatchMaxPaths ?? 1));
|
|
987
|
+
this.smallBatchMaxBytes = Math.max(0, Math.floor(opts.smallBatchMaxBytes ?? 0));
|
|
959
988
|
this.maxWaitMs =
|
|
960
989
|
opts.maxWaitMs !== undefined && opts.maxWaitMs > 0
|
|
961
990
|
? Math.max(opts.maxWaitMs, this.debounceMs)
|
|
@@ -1343,6 +1372,7 @@ export class TreeWatcher {
|
|
|
1343
1372
|
this.arm();
|
|
1344
1373
|
return;
|
|
1345
1374
|
}
|
|
1375
|
+
this.pendingPayloadSizes.set(abs, this.payloadSizeFor(abs, kind));
|
|
1346
1376
|
const deleteSnapshotEpoch = (this.deleteSnapshotEpochs.get(abs) ?? 0) + 1;
|
|
1347
1377
|
this.deleteSnapshotEpochs.set(abs, deleteSnapshotEpoch);
|
|
1348
1378
|
let deleteSnapshots = [];
|
|
@@ -1371,6 +1401,7 @@ export class TreeWatcher {
|
|
|
1371
1401
|
// A newer live revision supersedes any queued unlink capture for this
|
|
1372
1402
|
// path before that job is allowed to acquire and mutate the journal.
|
|
1373
1403
|
this.deleteSnapshotEpochs.set(abs, (this.deleteSnapshotEpochs.get(abs) ?? 0) + 1);
|
|
1404
|
+
this.pendingPayloadSizes.set(abs, this.payloadSizeFor(abs, kind));
|
|
1374
1405
|
this.pendingChanges.set(abs, { kind });
|
|
1375
1406
|
// Only directories are needed to disambiguate a native recursive
|
|
1376
1407
|
// rename-after-delete. Unknown paths already correctly default to an
|
|
@@ -1495,11 +1526,14 @@ export class TreeWatcher {
|
|
|
1495
1526
|
if (this.windowDeadlineAt === null && Number.isFinite(this.maxWaitMs)) {
|
|
1496
1527
|
this.windowDeadlineAt = now + this.maxWaitMs;
|
|
1497
1528
|
}
|
|
1529
|
+
const quietWindowMs = this.isSmallPendingBatch()
|
|
1530
|
+
? this.smallBatchDebounceMs
|
|
1531
|
+
: this.debounceMs;
|
|
1498
1532
|
// Quiet-period debounce, capped by the window's max-wait deadline so a
|
|
1499
1533
|
// continuous event stream cannot postpone the emit forever.
|
|
1500
1534
|
const delayMs = this.windowDeadlineAt === null
|
|
1501
|
-
?
|
|
1502
|
-
: Math.max(0, Math.min(
|
|
1535
|
+
? quietWindowMs
|
|
1536
|
+
: Math.max(0, Math.min(quietWindowMs, this.windowDeadlineAt - now));
|
|
1503
1537
|
this.timer = this.clock.setTimeout(() => {
|
|
1504
1538
|
this.timer = null;
|
|
1505
1539
|
this.emit();
|
|
@@ -1541,6 +1575,7 @@ export class TreeWatcher {
|
|
|
1541
1575
|
this.pendingStartedAt = null;
|
|
1542
1576
|
this.pending.clear();
|
|
1543
1577
|
this.pendingChanges.clear();
|
|
1578
|
+
this.pendingPayloadSizes.clear();
|
|
1544
1579
|
this.pendingBytes = 0;
|
|
1545
1580
|
this.overflowed = false;
|
|
1546
1581
|
this.overflowLogged = false;
|
|
@@ -1549,6 +1584,39 @@ export class TreeWatcher {
|
|
|
1549
1584
|
this.droppedRouteHints.clear();
|
|
1550
1585
|
this.droppedRoutesUnknown = false;
|
|
1551
1586
|
}
|
|
1587
|
+
payloadSizeFor(absolutePath, kind) {
|
|
1588
|
+
// Exact deletes transfer no content; directory operations can expand to an
|
|
1589
|
+
// arbitrary subtree and therefore never take the latency shortcut.
|
|
1590
|
+
if (kind === "unlink")
|
|
1591
|
+
return 0;
|
|
1592
|
+
if (kind === "unlinkDir" || kind === "addDir")
|
|
1593
|
+
return null;
|
|
1594
|
+
try {
|
|
1595
|
+
const stat = fs.lstatSync(absolutePath);
|
|
1596
|
+
return stat.isFile() ? stat.size : null;
|
|
1597
|
+
}
|
|
1598
|
+
catch {
|
|
1599
|
+
// An upsert whose bytes cannot be read is not proven small.
|
|
1600
|
+
return null;
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
isSmallPendingBatch() {
|
|
1604
|
+
if (this.smallBatchDebounceMs === null ||
|
|
1605
|
+
this.overflowed ||
|
|
1606
|
+
this.pending.size === 0 ||
|
|
1607
|
+
this.pending.size > this.smallBatchMaxPaths) {
|
|
1608
|
+
return false;
|
|
1609
|
+
}
|
|
1610
|
+
let bytes = 0;
|
|
1611
|
+
for (const size of this.pendingPayloadSizes.values()) {
|
|
1612
|
+
if (size === null)
|
|
1613
|
+
return false;
|
|
1614
|
+
bytes += size;
|
|
1615
|
+
if (bytes > this.smallBatchMaxBytes)
|
|
1616
|
+
return false;
|
|
1617
|
+
}
|
|
1618
|
+
return true;
|
|
1619
|
+
}
|
|
1552
1620
|
/**
|
|
1553
1621
|
* Recover mutations that happened while chokidar was building its initial
|
|
1554
1622
|
* watch set. `ignoreInitial` intentionally suppresses those initial events,
|
|
@@ -1709,8 +1777,23 @@ async function computeContentHash(absolutePath) {
|
|
|
1709
1777
|
}
|
|
1710
1778
|
/** Maximum paths remembered by one emitter to suppress unchanged announcements. */
|
|
1711
1779
|
export const DEFAULT_PUBLISHED_CONTENT_HASH_MAX_PATHS = 50_000;
|
|
1780
|
+
/** Minimum delay between successful upsert announcements for one path. */
|
|
1781
|
+
export const DEFAULT_PUBLISH_MIN_INTERVAL_MS = 60_000;
|
|
1782
|
+
export const PUBLISH_MIN_INTERVAL_MS_ENV = "HQ_SYNC_PUBLISH_MIN_INTERVAL_MS";
|
|
1783
|
+
/** Resolve the optional per-path announcement interval without accepting invalid values. */
|
|
1784
|
+
export function resolvePublishMinIntervalMs(env = process.env) {
|
|
1785
|
+
const raw = env[PUBLISH_MIN_INTERVAL_MS_ENV];
|
|
1786
|
+
if (raw === undefined || raw.trim() === "")
|
|
1787
|
+
return DEFAULT_PUBLISH_MIN_INTERVAL_MS;
|
|
1788
|
+
const value = Number(raw);
|
|
1789
|
+
return Number.isFinite(value) && value >= 0
|
|
1790
|
+
? Math.floor(value)
|
|
1791
|
+
: DEFAULT_PUBLISH_MIN_INTERVAL_MS;
|
|
1792
|
+
}
|
|
1712
1793
|
const PUBLISHED_CONTENT_HASH_RECORD_SET = 0x5031;
|
|
1713
1794
|
const PUBLISHED_CONTENT_HASH_RECORD_DELETE = 0x5032;
|
|
1795
|
+
const DURABLE_PUBLISHED_HASH_RETRY_INITIAL_MS = 250;
|
|
1796
|
+
const DURABLE_PUBLISHED_HASH_RETRY_MAX_MS = 10_000;
|
|
1714
1797
|
/**
|
|
1715
1798
|
* Bounded, crash-safe state of hashes whose push event transport acknowledged
|
|
1716
1799
|
* publication. This is intentionally separate from the sync journal: a
|
|
@@ -1738,9 +1821,19 @@ class DurablePublishedContentHashes {
|
|
|
1738
1821
|
maxPaths,
|
|
1739
1822
|
});
|
|
1740
1823
|
}
|
|
1824
|
+
async setIfCurrent(relativePath, contentHash, maxPaths, shouldPersist) {
|
|
1825
|
+
return (await this.store.appendIfAsync(PUBLISHED_CONTENT_HASH_RECORD_SET, {
|
|
1826
|
+
relativePath,
|
|
1827
|
+
contentHash,
|
|
1828
|
+
maxPaths,
|
|
1829
|
+
}, shouldPersist)) !== undefined;
|
|
1830
|
+
}
|
|
1741
1831
|
async delete(relativePath) {
|
|
1742
1832
|
await this.store.appendAsync(PUBLISHED_CONTENT_HASH_RECORD_DELETE, { relativePath });
|
|
1743
1833
|
}
|
|
1834
|
+
async deleteIfCurrent(relativePath, shouldPersist) {
|
|
1835
|
+
return (await this.store.appendIfAsync(PUBLISHED_CONTENT_HASH_RECORD_DELETE, { relativePath }, shouldPersist)) !== undefined;
|
|
1836
|
+
}
|
|
1744
1837
|
}
|
|
1745
1838
|
function boundedDurablePublishedContentHashState(hashes, maxPaths) {
|
|
1746
1839
|
const order = Object.keys(hashes).slice(-maxPaths);
|
|
@@ -1797,6 +1890,7 @@ function reduceDurablePublishedContentHashState(state, record) {
|
|
|
1797
1890
|
class BoundedPublishedContentHashes {
|
|
1798
1891
|
maxPaths;
|
|
1799
1892
|
hashes = new Map();
|
|
1893
|
+
publishedAt = new Map();
|
|
1800
1894
|
constructor(maxPaths) {
|
|
1801
1895
|
this.maxPaths = Math.max(1, Math.floor(maxPaths ?? DEFAULT_PUBLISHED_CONTENT_HASH_MAX_PATHS));
|
|
1802
1896
|
}
|
|
@@ -1808,18 +1902,27 @@ class BoundedPublishedContentHashes {
|
|
|
1808
1902
|
this.hashes.set(relativePath, hash);
|
|
1809
1903
|
return hash;
|
|
1810
1904
|
}
|
|
1811
|
-
set(relativePath, contentHash) {
|
|
1905
|
+
set(relativePath, contentHash, publishedAt) {
|
|
1812
1906
|
this.hashes.delete(relativePath);
|
|
1813
1907
|
this.hashes.set(relativePath, contentHash);
|
|
1908
|
+
if (publishedAt !== undefined)
|
|
1909
|
+
this.publishedAt.set(relativePath, publishedAt);
|
|
1910
|
+
else
|
|
1911
|
+
this.publishedAt.delete(relativePath);
|
|
1814
1912
|
while (this.hashes.size > this.maxPaths) {
|
|
1815
1913
|
const oldest = this.hashes.keys().next().value;
|
|
1816
1914
|
if (oldest === undefined)
|
|
1817
1915
|
return;
|
|
1818
1916
|
this.hashes.delete(oldest);
|
|
1917
|
+
this.publishedAt.delete(oldest);
|
|
1819
1918
|
}
|
|
1820
1919
|
}
|
|
1920
|
+
getPublishedAt(relativePath) {
|
|
1921
|
+
return this.publishedAt.get(relativePath);
|
|
1922
|
+
}
|
|
1821
1923
|
delete(relativePath) {
|
|
1822
1924
|
this.hashes.delete(relativePath);
|
|
1925
|
+
this.publishedAt.delete(relativePath);
|
|
1823
1926
|
}
|
|
1824
1927
|
}
|
|
1825
1928
|
/** Successful attempts retained in the batch outcome record for context. */
|
|
@@ -1879,6 +1982,7 @@ export class PushEventEmitter {
|
|
|
1879
1982
|
transport;
|
|
1880
1983
|
flagProvider;
|
|
1881
1984
|
now;
|
|
1985
|
+
clock;
|
|
1882
1986
|
onError;
|
|
1883
1987
|
onPublishOutcome;
|
|
1884
1988
|
logger;
|
|
@@ -1887,6 +1991,16 @@ export class PushEventEmitter {
|
|
|
1887
1991
|
lastPublishedContentHashes;
|
|
1888
1992
|
durablePublishedContentHashes;
|
|
1889
1993
|
lastPublishedContentHashMaxPaths;
|
|
1994
|
+
publishMinIntervalMs;
|
|
1995
|
+
/**
|
|
1996
|
+
* A captured event whose bytes were already uploaded by the scoped push that
|
|
1997
|
+
* called emitForBatch. Never retain only a path here: a later local rewrite
|
|
1998
|
+
* can otherwise make the trailing announcement describe bytes absent from S3.
|
|
1999
|
+
*/
|
|
2000
|
+
deferredPublishes = new Map();
|
|
2001
|
+
deferredPublishTimers = new Map();
|
|
2002
|
+
durablePublishedHashRetries = new Map();
|
|
2003
|
+
disposed = false;
|
|
1890
2004
|
internalSeq = 0;
|
|
1891
2005
|
nextSeq;
|
|
1892
2006
|
publishTail = Promise.resolve();
|
|
@@ -1896,6 +2010,7 @@ export class PushEventEmitter {
|
|
|
1896
2010
|
this.transport = opts.transport;
|
|
1897
2011
|
this.flagProvider = opts.flagProvider;
|
|
1898
2012
|
this.now = opts.now ?? (() => new Date());
|
|
2013
|
+
this.clock = opts.clock ?? systemClock;
|
|
1899
2014
|
this.logger = opts.logger;
|
|
1900
2015
|
this.telemetryClient = opts.telemetryClient;
|
|
1901
2016
|
this.telemetryClaims = opts.telemetryClaims;
|
|
@@ -1905,6 +2020,11 @@ export class PushEventEmitter {
|
|
|
1905
2020
|
this.lastPublishedContentHashMaxPaths = Math.max(1, Math.floor(opts.lastPublishedContentHashMaxPaths ??
|
|
1906
2021
|
DEFAULT_PUBLISHED_CONTENT_HASH_MAX_PATHS));
|
|
1907
2022
|
this.lastPublishedContentHashes = new BoundedPublishedContentHashes(this.lastPublishedContentHashMaxPaths);
|
|
2023
|
+
const configuredPublishMinIntervalMs = opts.publishMinIntervalMs ?? resolvePublishMinIntervalMs();
|
|
2024
|
+
this.publishMinIntervalMs =
|
|
2025
|
+
Number.isFinite(configuredPublishMinIntervalMs) && configuredPublishMinIntervalMs >= 0
|
|
2026
|
+
? Math.floor(configuredPublishMinIntervalMs)
|
|
2027
|
+
: DEFAULT_PUBLISH_MIN_INTERVAL_MS;
|
|
1908
2028
|
this.durablePublishedContentHashes = opts.lastPublishedContentHashStateDir
|
|
1909
2029
|
? this.openDurablePublishedContentHashes(opts.lastPublishedContentHashStateDir, opts.initialPublishedContentHashes ?? {})
|
|
1910
2030
|
: null;
|
|
@@ -1942,7 +2062,7 @@ export class PushEventEmitter {
|
|
|
1942
2062
|
* poll covers any miss).
|
|
1943
2063
|
*/
|
|
1944
2064
|
async emitForBatch(batch) {
|
|
1945
|
-
if (!this.enabled)
|
|
2065
|
+
if (this.disposed || !this.enabled)
|
|
1946
2066
|
return;
|
|
1947
2067
|
const entriesByRel = new Map();
|
|
1948
2068
|
for (const [absolutePath, relativePath] of batch.paths.entries()) {
|
|
@@ -1960,6 +2080,9 @@ export class PushEventEmitter {
|
|
|
1960
2080
|
const settled = await Promise.all(chunk.map(([absolutePath, relativePath]) => this.emitOne(absolutePath, relativePath)));
|
|
1961
2081
|
attempts.push(...settled.filter((attempt) => attempt !== undefined));
|
|
1962
2082
|
}
|
|
2083
|
+
this.reportPublishOutcome(attempts);
|
|
2084
|
+
}
|
|
2085
|
+
reportPublishOutcome(attempts) {
|
|
1963
2086
|
try {
|
|
1964
2087
|
this.onPublishOutcome(summarizePublishBatchOutcome(attempts));
|
|
1965
2088
|
}
|
|
@@ -1977,13 +2100,12 @@ export class PushEventEmitter {
|
|
|
1977
2100
|
return undefined;
|
|
1978
2101
|
const contentHash = await computeContentHash(absolutePath);
|
|
1979
2102
|
if (this.wasPublished(relativePath, contentHash)) {
|
|
2103
|
+
// This later, successfully-pushed revision restores the current
|
|
2104
|
+
// publication. Its stale deferred predecessor must not announce after
|
|
2105
|
+
// the scoped push has already replaced its S3 bytes.
|
|
2106
|
+
this.clearDeferredPublish(relativePath);
|
|
1980
2107
|
return undefined;
|
|
1981
2108
|
}
|
|
1982
|
-
// A different observed revision breaks the old publication guarantee even
|
|
1983
|
-
// if this transport attempt fails. Clearing first makes a later revert
|
|
1984
|
-
// re-announce instead of silently treating it as the old content.
|
|
1985
|
-
this.lastPublishedContentHashes.delete(relativePath);
|
|
1986
|
-
await this.deleteDurablePublishedContentHash(relativePath);
|
|
1987
2109
|
event = {
|
|
1988
2110
|
kind: "upsert",
|
|
1989
2111
|
relativePath,
|
|
@@ -1994,6 +2116,17 @@ export class PushEventEmitter {
|
|
|
1994
2116
|
sequenceNumber: this.nextSeq(),
|
|
1995
2117
|
eventTimestamp: this.now().toISOString(),
|
|
1996
2118
|
};
|
|
2119
|
+
const lastPublishedAt = this.lastPublishedContentHashes.getPublishedAt(relativePath);
|
|
2120
|
+
if (lastPublishedAt !== undefined &&
|
|
2121
|
+
this.clock.now() - lastPublishedAt < this.publishMinIntervalMs) {
|
|
2122
|
+
this.deferUpsert(event, lastPublishedAt);
|
|
2123
|
+
return undefined;
|
|
2124
|
+
}
|
|
2125
|
+
// A different observed revision breaks the old publication guarantee even
|
|
2126
|
+
// if this transport attempt fails. Clearing first makes a later revert
|
|
2127
|
+
// re-announce instead of silently treating it as the old content.
|
|
2128
|
+
this.lastPublishedContentHashes.delete(relativePath);
|
|
2129
|
+
await this.deleteDurablePublishedContentHash(relativePath);
|
|
1997
2130
|
}
|
|
1998
2131
|
catch (err) {
|
|
1999
2132
|
const code = err && typeof err === "object" && "code" in err
|
|
@@ -2011,6 +2144,7 @@ export class PushEventEmitter {
|
|
|
2011
2144
|
// targeted pull and let the vault-confirmed tombstone path remove it.
|
|
2012
2145
|
// Invalidate before the attempt for the same failure/recreation posture
|
|
2013
2146
|
// as a changed upsert above.
|
|
2147
|
+
this.clearDeferredPublish(relativePath);
|
|
2014
2148
|
this.lastPublishedContentHashes.delete(relativePath);
|
|
2015
2149
|
await this.deleteDurablePublishedContentHash(relativePath);
|
|
2016
2150
|
event = {
|
|
@@ -2022,6 +2156,9 @@ export class PushEventEmitter {
|
|
|
2022
2156
|
eventTimestamp: this.now().toISOString(),
|
|
2023
2157
|
};
|
|
2024
2158
|
}
|
|
2159
|
+
return this.publishEvent(event);
|
|
2160
|
+
}
|
|
2161
|
+
async publishEvent(event) {
|
|
2025
2162
|
// US-011: 1st link of the 3-log diagnostic chain. Stamps the same
|
|
2026
2163
|
// `sequenceNumber` the server `push.receive` log and the client
|
|
2027
2164
|
// `fanout.receive` log carry, so an operator can walk one event
|
|
@@ -2040,14 +2177,14 @@ export class PushEventEmitter {
|
|
|
2040
2177
|
try {
|
|
2041
2178
|
await this.transport.publish(event);
|
|
2042
2179
|
if (event.kind === "upsert" && event.contentHash !== undefined) {
|
|
2043
|
-
this.lastPublishedContentHashes.set(relativePath, event.contentHash);
|
|
2044
|
-
await this.setDurablePublishedContentHash(relativePath, event.contentHash);
|
|
2180
|
+
this.lastPublishedContentHashes.set(event.relativePath, event.contentHash, this.clock.now());
|
|
2181
|
+
await this.setDurablePublishedContentHash(event.relativePath, event.contentHash);
|
|
2045
2182
|
}
|
|
2046
2183
|
else {
|
|
2047
2184
|
// A later recreation with the same bytes is a real state transition,
|
|
2048
2185
|
// so a successfully published tombstone must never suppress it.
|
|
2049
|
-
this.lastPublishedContentHashes.delete(relativePath);
|
|
2050
|
-
await this.deleteDurablePublishedContentHash(relativePath);
|
|
2186
|
+
this.lastPublishedContentHashes.delete(event.relativePath);
|
|
2187
|
+
await this.deleteDurablePublishedContentHash(event.relativePath);
|
|
2051
2188
|
}
|
|
2052
2189
|
const outcome = getPublishSuccessOutcome(event);
|
|
2053
2190
|
return {
|
|
@@ -2059,7 +2196,7 @@ export class PushEventEmitter {
|
|
|
2059
2196
|
// Push failure (network / non-2xx / timeout). The cadence poll is the
|
|
2060
2197
|
// safety net — log + continue, never throw.
|
|
2061
2198
|
this.onError(err instanceof Error ? err : new Error(String(err)), {
|
|
2062
|
-
relativePath,
|
|
2199
|
+
relativePath: event.relativePath,
|
|
2063
2200
|
});
|
|
2064
2201
|
this.emitPublishFailureTelemetry(event.kind, "publish");
|
|
2065
2202
|
return {
|
|
@@ -2099,23 +2236,142 @@ export class PushEventEmitter {
|
|
|
2099
2236
|
}
|
|
2100
2237
|
async setDurablePublishedContentHash(relativePath, contentHash) {
|
|
2101
2238
|
try {
|
|
2102
|
-
await this.
|
|
2239
|
+
await this.persistDurablePublishedContentHash(relativePath, { kind: "set", contentHash });
|
|
2240
|
+
this.clearDurablePublishedContentHashRetry(relativePath);
|
|
2103
2241
|
}
|
|
2104
2242
|
catch (err) {
|
|
2105
|
-
this.
|
|
2106
|
-
relativePath,
|
|
2107
|
-
});
|
|
2243
|
+
this.handleDurablePublishedContentHashFailure(relativePath, { kind: "set", contentHash }, err);
|
|
2108
2244
|
}
|
|
2109
2245
|
}
|
|
2110
2246
|
async deleteDurablePublishedContentHash(relativePath) {
|
|
2111
2247
|
try {
|
|
2112
|
-
await this.
|
|
2248
|
+
await this.persistDurablePublishedContentHash(relativePath, { kind: "delete" });
|
|
2249
|
+
this.clearDurablePublishedContentHashRetry(relativePath);
|
|
2113
2250
|
}
|
|
2114
2251
|
catch (err) {
|
|
2115
|
-
this.
|
|
2116
|
-
|
|
2117
|
-
|
|
2252
|
+
this.handleDurablePublishedContentHashFailure(relativePath, { kind: "delete" }, err);
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
async persistDurablePublishedContentHash(relativePath, operation, shouldPersist) {
|
|
2256
|
+
if (!this.durablePublishedContentHashes)
|
|
2257
|
+
return true;
|
|
2258
|
+
if (operation.kind === "set") {
|
|
2259
|
+
if (shouldPersist) {
|
|
2260
|
+
return this.durablePublishedContentHashes.setIfCurrent(relativePath, operation.contentHash, this.lastPublishedContentHashMaxPaths, shouldPersist);
|
|
2261
|
+
}
|
|
2262
|
+
await this.durablePublishedContentHashes.set(relativePath, operation.contentHash, this.lastPublishedContentHashMaxPaths);
|
|
2263
|
+
}
|
|
2264
|
+
else {
|
|
2265
|
+
if (shouldPersist) {
|
|
2266
|
+
return this.durablePublishedContentHashes.deleteIfCurrent(relativePath, shouldPersist);
|
|
2267
|
+
}
|
|
2268
|
+
await this.durablePublishedContentHashes.delete(relativePath);
|
|
2269
|
+
}
|
|
2270
|
+
return true;
|
|
2271
|
+
}
|
|
2272
|
+
handleDurablePublishedContentHashFailure(relativePath, operation, err) {
|
|
2273
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
2274
|
+
this.onError(error, { relativePath });
|
|
2275
|
+
if (error instanceof StateStoreLockError) {
|
|
2276
|
+
this.scheduleDurablePublishedContentHashRetry(relativePath, operation);
|
|
2277
|
+
}
|
|
2278
|
+
}
|
|
2279
|
+
clearDurablePublishedContentHashRetry(relativePath) {
|
|
2280
|
+
const retry = this.durablePublishedHashRetries.get(relativePath);
|
|
2281
|
+
if (!retry)
|
|
2282
|
+
return;
|
|
2283
|
+
if (retry.timer !== undefined)
|
|
2284
|
+
this.clock.clearTimeout(retry.timer);
|
|
2285
|
+
this.durablePublishedHashRetries.delete(relativePath);
|
|
2286
|
+
}
|
|
2287
|
+
scheduleDurablePublishedContentHashRetry(relativePath, operation) {
|
|
2288
|
+
if (this.disposed || !this.durablePublishedContentHashes)
|
|
2289
|
+
return;
|
|
2290
|
+
const existing = this.durablePublishedHashRetries.get(relativePath);
|
|
2291
|
+
if (existing?.timer !== undefined) {
|
|
2292
|
+
existing.operation = operation;
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
const retry = {
|
|
2296
|
+
operation,
|
|
2297
|
+
attempts: (existing?.attempts ?? 0) + 1,
|
|
2298
|
+
};
|
|
2299
|
+
const delay = Math.min(DURABLE_PUBLISHED_HASH_RETRY_INITIAL_MS * 2 ** Math.min(retry.attempts - 1, 6), DURABLE_PUBLISHED_HASH_RETRY_MAX_MS);
|
|
2300
|
+
retry.timer = this.clock.setTimeout(() => {
|
|
2301
|
+
retry.timer = undefined;
|
|
2302
|
+
void this.retryDurablePublishedContentHash(relativePath);
|
|
2303
|
+
}, delay);
|
|
2304
|
+
this.durablePublishedHashRetries.set(relativePath, retry);
|
|
2305
|
+
}
|
|
2306
|
+
async retryDurablePublishedContentHash(relativePath) {
|
|
2307
|
+
if (this.disposed)
|
|
2308
|
+
return;
|
|
2309
|
+
const retry = this.durablePublishedHashRetries.get(relativePath);
|
|
2310
|
+
if (!retry)
|
|
2311
|
+
return;
|
|
2312
|
+
try {
|
|
2313
|
+
const persisted = await this.persistDurablePublishedContentHash(relativePath, retry.operation, () => this.durablePublishedHashRetries.get(relativePath) === retry);
|
|
2314
|
+
if (persisted && this.durablePublishedHashRetries.get(relativePath) === retry) {
|
|
2315
|
+
this.durablePublishedHashRetries.delete(relativePath);
|
|
2316
|
+
}
|
|
2317
|
+
}
|
|
2318
|
+
catch (err) {
|
|
2319
|
+
if (this.durablePublishedHashRetries.get(relativePath) === retry) {
|
|
2320
|
+
this.handleDurablePublishedContentHashFailure(relativePath, retry.operation, err);
|
|
2321
|
+
}
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
/** Remember one latest-wins uploaded revision until its publish interval ends. */
|
|
2325
|
+
deferUpsert(event, lastPublishedAt) {
|
|
2326
|
+
const relativePath = event.relativePath;
|
|
2327
|
+
this.deferredPublishes.delete(relativePath);
|
|
2328
|
+
this.deferredPublishes.set(relativePath, event);
|
|
2329
|
+
while (this.deferredPublishes.size > this.lastPublishedContentHashMaxPaths) {
|
|
2330
|
+
const oldest = this.deferredPublishes.keys().next().value;
|
|
2331
|
+
if (oldest === undefined)
|
|
2332
|
+
break;
|
|
2333
|
+
this.clearDeferredPublish(oldest);
|
|
2334
|
+
}
|
|
2335
|
+
if (this.deferredPublishTimers.has(relativePath))
|
|
2336
|
+
return;
|
|
2337
|
+
const delay = Math.max(0, lastPublishedAt + this.publishMinIntervalMs - this.clock.now());
|
|
2338
|
+
const timer = this.clock.setTimeout(() => this.flushDeferredPublish(relativePath), delay);
|
|
2339
|
+
this.deferredPublishTimers.set(relativePath, timer);
|
|
2340
|
+
}
|
|
2341
|
+
flushDeferredPublish(relativePath) {
|
|
2342
|
+
const deferred = this.deferredPublishes.get(relativePath);
|
|
2343
|
+
this.clearDeferredPublish(relativePath);
|
|
2344
|
+
if (this.disposed || !deferred)
|
|
2345
|
+
return;
|
|
2346
|
+
const run = this.publishTail.then(async () => {
|
|
2347
|
+
this.reportPublishOutcome([await this.publishEvent(deferred)]);
|
|
2348
|
+
}, async () => {
|
|
2349
|
+
this.reportPublishOutcome([await this.publishEvent(deferred)]);
|
|
2350
|
+
});
|
|
2351
|
+
this.publishTail = run.catch(() => undefined);
|
|
2352
|
+
}
|
|
2353
|
+
clearDeferredPublish(relativePath) {
|
|
2354
|
+
this.deferredPublishes.delete(relativePath);
|
|
2355
|
+
const timer = this.deferredPublishTimers.get(relativePath);
|
|
2356
|
+
if (timer !== undefined)
|
|
2357
|
+
this.clock.clearTimeout(timer);
|
|
2358
|
+
this.deferredPublishTimers.delete(relativePath);
|
|
2359
|
+
}
|
|
2360
|
+
/** Cancel every trailing publish and make future batch delivery inert. */
|
|
2361
|
+
dispose() {
|
|
2362
|
+
if (this.disposed)
|
|
2363
|
+
return;
|
|
2364
|
+
this.disposed = true;
|
|
2365
|
+
for (const timer of this.deferredPublishTimers.values()) {
|
|
2366
|
+
this.clock.clearTimeout(timer);
|
|
2367
|
+
}
|
|
2368
|
+
this.deferredPublishTimers.clear();
|
|
2369
|
+
this.deferredPublishes.clear();
|
|
2370
|
+
for (const retry of this.durablePublishedHashRetries.values()) {
|
|
2371
|
+
if (retry.timer !== undefined)
|
|
2372
|
+
this.clock.clearTimeout(retry.timer);
|
|
2118
2373
|
}
|
|
2374
|
+
this.durablePublishedHashRetries.clear();
|
|
2119
2375
|
}
|
|
2120
2376
|
emitPublishFailureTelemetry(kind, stage) {
|
|
2121
2377
|
void emitCloudTelemetry(this.telemetryClient, {
|