@indigoai-us/hq-cloud 6.16.4 → 6.16.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-events.d.ts +10 -0
- package/dist/bin/sync-runner-events.d.ts.map +1 -1
- package/dist/bin/sync-runner-events.js +34 -0
- package/dist/bin/sync-runner-events.js.map +1 -1
- package/dist/bin/sync-runner-events.test.js +65 -1
- package/dist/bin/sync-runner-events.test.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.d.ts +313 -45
- package/dist/bin/sync-runner-heap-telemetry.d.ts.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.js +503 -114
- package/dist/bin/sync-runner-heap-telemetry.js.map +1 -1
- package/dist/bin/sync-runner-heap-telemetry.test.js +436 -11
- package/dist/bin/sync-runner-heap-telemetry.test.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +69 -9
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +13 -1
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.d.ts +27 -1
- package/dist/bin/sync-runner.d.ts.map +1 -1
- package/dist/bin/sync-runner.js +10 -1
- package/dist/bin/sync-runner.js.map +1 -1
- package/dist/sync/event-sync.js +2 -2
- package/dist/sync/event-sync.js.map +1 -1
- package/dist/sync/state-store.d.ts +8 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +37 -0
- package/dist/sync/state-store.js.map +1 -1
- package/dist/watcher.d.ts +49 -9
- package/dist/watcher.d.ts.map +1 -1
- package/dist/watcher.js +425 -158
- package/dist/watcher.js.map +1 -1
- package/dist/watcher.test.js +349 -23
- package/dist/watcher.test.js.map +1 -1
- package/package.json +1 -1
package/dist/watcher.js
CHANGED
|
@@ -16,6 +16,7 @@ import { localPathForVaultKey, vaultKeyForLocalPath } from "./local-path-codec.j
|
|
|
16
16
|
import { isPersonalVaultExcluded } from "./personal-vault-exclusions.js";
|
|
17
17
|
import { StateStore, StateStoreLockError } from "./sync/state-store.js";
|
|
18
18
|
import { AGENCY_SYNC_REL, computePersonalCompanySubdirs, CONTINUITY_POINTER_REL, isCurrentContinuityThreadPath, SESSION_LOGS_SYNC_REL, PERSONAL_VAULT_EXCLUDED_TOP_LEVEL, } from "./personal-vault.js";
|
|
19
|
+
import { toPosixKey } from "./s3.js";
|
|
19
20
|
import { getPublishFailureOutcome, getPublishSuccessOutcome, isRealtimeUnavailableError, serializePublishError, } from "./sync/push-transport.js";
|
|
20
21
|
import { emitCloudTelemetry, } from "./telemetry-events.js";
|
|
21
22
|
const DEBOUNCE_MS = 2000;
|
|
@@ -1683,7 +1684,8 @@ export class TreeWatcher {
|
|
|
1683
1684
|
}
|
|
1684
1685
|
};
|
|
1685
1686
|
visit(this.hqRoot);
|
|
1686
|
-
for (const
|
|
1687
|
+
for (const journalRelativePath of journalRelativePaths) {
|
|
1688
|
+
const relative = toPosixKey(journalRelativePath);
|
|
1687
1689
|
if (observed.has(relative))
|
|
1688
1690
|
continue;
|
|
1689
1691
|
const absolute = localPathForVaultKey(this.hqRoot, relative);
|
|
@@ -1810,8 +1812,16 @@ export function resolvePublishMinIntervalMs(env = process.env) {
|
|
|
1810
1812
|
}
|
|
1811
1813
|
const PUBLISHED_CONTENT_HASH_RECORD_SET = 0x5031;
|
|
1812
1814
|
const PUBLISHED_CONTENT_HASH_RECORD_DELETE = 0x5032;
|
|
1815
|
+
const PUBLISHED_CONTENT_HASH_RECORD_BATCH = 0x5033;
|
|
1816
|
+
/** Limit one WAL frame to a modest size while collapsing a reconcile pass. */
|
|
1817
|
+
export const DURABLE_PUBLISHED_HASH_BATCH_MAX_ENTRIES = 4_096;
|
|
1818
|
+
/** A crash can lose at most this small durable-memo window under light load. */
|
|
1819
|
+
export const DURABLE_PUBLISHED_HASH_BATCH_FLUSH_MS = 25;
|
|
1813
1820
|
const DURABLE_PUBLISHED_HASH_RETRY_INITIAL_MS = 250;
|
|
1814
1821
|
const DURABLE_PUBLISHED_HASH_RETRY_MAX_MS = 10_000;
|
|
1822
|
+
// Match HttpPushTransport's existing per-request abort deadline. If a custom
|
|
1823
|
+
// transport ignores it, disposal returns without closing the durable memo.
|
|
1824
|
+
const PUSH_EVENT_EMITTER_DISPOSE_DRAIN_MS = 60_000;
|
|
1815
1825
|
/**
|
|
1816
1826
|
* JSON snapshots deliberately retain the long-lived `{ hashes, order }` wire
|
|
1817
1827
|
* shape. At runtime, keep the order as a Map so replay and live writes can
|
|
@@ -1826,9 +1836,21 @@ const durablePublishedContentHashOrders = new WeakMap();
|
|
|
1826
1836
|
* journal row proves the object upload completed, while the event publish that
|
|
1827
1837
|
* follows can still fail or time out.
|
|
1828
1838
|
*/
|
|
1829
|
-
class DurablePublishedContentHashes {
|
|
1839
|
+
export class DurablePublishedContentHashes {
|
|
1830
1840
|
store;
|
|
1831
|
-
|
|
1841
|
+
clock;
|
|
1842
|
+
onBatchLockFailure;
|
|
1843
|
+
pending = [];
|
|
1844
|
+
batchRetries = new Set();
|
|
1845
|
+
flushTimer;
|
|
1846
|
+
flushPromise;
|
|
1847
|
+
flushRequested = false;
|
|
1848
|
+
microtaskFlushScheduled = false;
|
|
1849
|
+
closed = false;
|
|
1850
|
+
nextSequence = 0;
|
|
1851
|
+
constructor(stateDir, scopeId, maxPaths, initialHashes, clock, onBatchLockFailure = () => undefined) {
|
|
1852
|
+
this.clock = clock;
|
|
1853
|
+
this.onBatchLockFailure = onBatchLockFailure;
|
|
1832
1854
|
this.store = StateStore.open({
|
|
1833
1855
|
rootDir: stateDir,
|
|
1834
1856
|
scopeId: `event-push-published-content:${scopeId}`,
|
|
@@ -1841,25 +1863,258 @@ class DurablePublishedContentHashes {
|
|
|
1841
1863
|
get(relativePath) {
|
|
1842
1864
|
return durablePublishedContentHashValue(this.store.getState().hashes, relativePath);
|
|
1843
1865
|
}
|
|
1844
|
-
|
|
1845
|
-
|
|
1866
|
+
get pendingRetryCount() {
|
|
1867
|
+
return this.batchRetries.size;
|
|
1868
|
+
}
|
|
1869
|
+
async set(relativePath, contentHash, maxPaths, flushImmediately) {
|
|
1870
|
+
await this.enqueue({
|
|
1871
|
+
kind: "set",
|
|
1846
1872
|
relativePath,
|
|
1847
1873
|
contentHash,
|
|
1848
1874
|
maxPaths,
|
|
1849
|
-
});
|
|
1875
|
+
}, undefined, flushImmediately);
|
|
1850
1876
|
}
|
|
1851
1877
|
async setIfCurrent(relativePath, contentHash, maxPaths, shouldPersist) {
|
|
1852
|
-
return
|
|
1878
|
+
return this.enqueue({
|
|
1879
|
+
kind: "set",
|
|
1853
1880
|
relativePath,
|
|
1854
1881
|
contentHash,
|
|
1855
1882
|
maxPaths,
|
|
1856
|
-
}, shouldPersist)
|
|
1883
|
+
}, shouldPersist, true);
|
|
1857
1884
|
}
|
|
1858
|
-
async delete(relativePath) {
|
|
1859
|
-
await this.
|
|
1885
|
+
async delete(relativePath, flushImmediately) {
|
|
1886
|
+
await this.enqueue({ kind: "delete", relativePath }, undefined, flushImmediately);
|
|
1860
1887
|
}
|
|
1861
1888
|
async deleteIfCurrent(relativePath, shouldPersist) {
|
|
1862
|
-
return
|
|
1889
|
+
return this.enqueue({ kind: "delete", relativePath }, shouldPersist, true);
|
|
1890
|
+
}
|
|
1891
|
+
async dispose() {
|
|
1892
|
+
this.closed = true;
|
|
1893
|
+
this.clearFlushTimer();
|
|
1894
|
+
this.clearBatchRetries();
|
|
1895
|
+
let failure;
|
|
1896
|
+
while (this.flushPromise || this.pending.length > 0) {
|
|
1897
|
+
try {
|
|
1898
|
+
if (this.flushPromise) {
|
|
1899
|
+
await this.flushPromise;
|
|
1900
|
+
}
|
|
1901
|
+
else {
|
|
1902
|
+
await this.requestFlush();
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
catch (error) {
|
|
1906
|
+
failure ??= error;
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
if (failure !== undefined)
|
|
1910
|
+
throw failure;
|
|
1911
|
+
}
|
|
1912
|
+
enqueue(mutation, shouldPersist, flushImmediately) {
|
|
1913
|
+
if (this.closed)
|
|
1914
|
+
return Promise.resolve(false);
|
|
1915
|
+
this.supersedeBatchRetries(mutation.relativePath);
|
|
1916
|
+
let resolve;
|
|
1917
|
+
let reject;
|
|
1918
|
+
const result = new Promise((resolveResult, rejectResult) => {
|
|
1919
|
+
resolve = resolveResult;
|
|
1920
|
+
reject = rejectResult;
|
|
1921
|
+
});
|
|
1922
|
+
this.pending.push({
|
|
1923
|
+
mutation,
|
|
1924
|
+
sequence: ++this.nextSequence,
|
|
1925
|
+
shouldPersist,
|
|
1926
|
+
resolve,
|
|
1927
|
+
reject,
|
|
1928
|
+
settled: false,
|
|
1929
|
+
});
|
|
1930
|
+
if (flushImmediately) {
|
|
1931
|
+
this.requestMicrotaskFlush();
|
|
1932
|
+
}
|
|
1933
|
+
else if (this.pending.length >= DURABLE_PUBLISHED_HASH_BATCH_MAX_ENTRIES) {
|
|
1934
|
+
void this.requestFlush().catch(() => undefined);
|
|
1935
|
+
}
|
|
1936
|
+
else {
|
|
1937
|
+
this.armFlushTimer();
|
|
1938
|
+
}
|
|
1939
|
+
return result;
|
|
1940
|
+
}
|
|
1941
|
+
requestMicrotaskFlush() {
|
|
1942
|
+
if (this.microtaskFlushScheduled)
|
|
1943
|
+
return;
|
|
1944
|
+
this.microtaskFlushScheduled = true;
|
|
1945
|
+
queueMicrotask(() => {
|
|
1946
|
+
this.microtaskFlushScheduled = false;
|
|
1947
|
+
void this.requestFlush().catch(() => undefined);
|
|
1948
|
+
});
|
|
1949
|
+
}
|
|
1950
|
+
armFlushTimer() {
|
|
1951
|
+
if (this.flushTimer !== undefined || this.closed)
|
|
1952
|
+
return;
|
|
1953
|
+
this.flushTimer = this.clock.setTimeout(() => {
|
|
1954
|
+
this.flushTimer = undefined;
|
|
1955
|
+
void this.requestFlush().catch(() => undefined);
|
|
1956
|
+
}, DURABLE_PUBLISHED_HASH_BATCH_FLUSH_MS);
|
|
1957
|
+
}
|
|
1958
|
+
clearFlushTimer() {
|
|
1959
|
+
if (this.flushTimer === undefined)
|
|
1960
|
+
return;
|
|
1961
|
+
this.clock.clearTimeout(this.flushTimer);
|
|
1962
|
+
this.flushTimer = undefined;
|
|
1963
|
+
}
|
|
1964
|
+
requestFlush() {
|
|
1965
|
+
this.flushRequested = true;
|
|
1966
|
+
if (this.flushPromise)
|
|
1967
|
+
return this.flushPromise;
|
|
1968
|
+
if (this.pending.length === 0)
|
|
1969
|
+
return Promise.resolve();
|
|
1970
|
+
this.clearFlushTimer();
|
|
1971
|
+
this.flushRequested = false;
|
|
1972
|
+
const batch = this.pending.splice(0, DURABLE_PUBLISHED_HASH_BATCH_MAX_ENTRIES);
|
|
1973
|
+
return this.startFlush(batch);
|
|
1974
|
+
}
|
|
1975
|
+
startFlush(batch, retry) {
|
|
1976
|
+
const flush = this.flushBatch(batch, retry);
|
|
1977
|
+
const tracked = flush.finally(() => {
|
|
1978
|
+
if (this.flushPromise !== tracked)
|
|
1979
|
+
return;
|
|
1980
|
+
this.flushPromise = undefined;
|
|
1981
|
+
if (this.pending.length === 0)
|
|
1982
|
+
return;
|
|
1983
|
+
if (this.closed || this.flushRequested || this.pending.length >= DURABLE_PUBLISHED_HASH_BATCH_MAX_ENTRIES) {
|
|
1984
|
+
void this.requestFlush().catch(() => undefined);
|
|
1985
|
+
}
|
|
1986
|
+
else {
|
|
1987
|
+
this.armFlushTimer();
|
|
1988
|
+
}
|
|
1989
|
+
});
|
|
1990
|
+
this.flushPromise = tracked;
|
|
1991
|
+
return tracked;
|
|
1992
|
+
}
|
|
1993
|
+
async flushBatch(batch, retry) {
|
|
1994
|
+
const included = [];
|
|
1995
|
+
let prepared = false;
|
|
1996
|
+
try {
|
|
1997
|
+
const appended = await this.store.appendGeneratedIfAsync(PUBLISHED_CONTENT_HASH_RECORD_BATCH, () => {
|
|
1998
|
+
prepared = true;
|
|
1999
|
+
const lastIncludedByPath = new Map();
|
|
2000
|
+
for (const entry of batch) {
|
|
2001
|
+
// This runs under append.lock, immediately before the mutation is
|
|
2002
|
+
// selected for the durable batch. Retain older candidates until
|
|
2003
|
+
// now: a stale later guard must not erase an earlier valid mark.
|
|
2004
|
+
if (entry.shouldPersist && !entry.shouldPersist()) {
|
|
2005
|
+
this.settle(entry, false);
|
|
2006
|
+
continue;
|
|
2007
|
+
}
|
|
2008
|
+
lastIncludedByPath.set(entry.mutation.relativePath, entry);
|
|
2009
|
+
}
|
|
2010
|
+
included.push(...lastIncludedByPath.values());
|
|
2011
|
+
included.sort((left, right) => left.sequence - right.sequence);
|
|
2012
|
+
for (const entry of batch) {
|
|
2013
|
+
if (!entry.settled && !included.includes(entry))
|
|
2014
|
+
this.settle(entry, false);
|
|
2015
|
+
}
|
|
2016
|
+
if (included.length === 0)
|
|
2017
|
+
return undefined;
|
|
2018
|
+
return {
|
|
2019
|
+
entries: included.map((entry) => entry.mutation),
|
|
2020
|
+
};
|
|
2021
|
+
});
|
|
2022
|
+
if (appended === undefined) {
|
|
2023
|
+
if (retry)
|
|
2024
|
+
this.batchRetries.delete(retry);
|
|
2025
|
+
return;
|
|
2026
|
+
}
|
|
2027
|
+
for (const entry of included)
|
|
2028
|
+
this.settle(entry, true);
|
|
2029
|
+
if (retry)
|
|
2030
|
+
this.batchRetries.delete(retry);
|
|
2031
|
+
}
|
|
2032
|
+
catch (error) {
|
|
2033
|
+
if (error instanceof StateStoreLockError) {
|
|
2034
|
+
const affected = prepared
|
|
2035
|
+
? included
|
|
2036
|
+
: retry
|
|
2037
|
+
? batch
|
|
2038
|
+
: batch.filter((entry) => !entry.settled);
|
|
2039
|
+
this.handleBatchLockFailure(error, affected, retry);
|
|
2040
|
+
return;
|
|
2041
|
+
}
|
|
2042
|
+
for (const entry of batch) {
|
|
2043
|
+
if (!entry.settled) {
|
|
2044
|
+
entry.settled = true;
|
|
2045
|
+
entry.reject(error);
|
|
2046
|
+
}
|
|
2047
|
+
}
|
|
2048
|
+
throw error;
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
settle(entry, persisted) {
|
|
2052
|
+
if (entry.settled)
|
|
2053
|
+
return;
|
|
2054
|
+
entry.settled = true;
|
|
2055
|
+
entry.resolve(persisted);
|
|
2056
|
+
}
|
|
2057
|
+
handleBatchLockFailure(error, entries, existingRetry) {
|
|
2058
|
+
if (entries.length === 0) {
|
|
2059
|
+
if (existingRetry)
|
|
2060
|
+
this.batchRetries.delete(existingRetry);
|
|
2061
|
+
return;
|
|
2062
|
+
}
|
|
2063
|
+
for (const entry of entries)
|
|
2064
|
+
this.settle(entry, false);
|
|
2065
|
+
this.onBatchLockFailure(error, entries.length, entries.length === 1 ? entries[0]?.mutation.relativePath : undefined);
|
|
2066
|
+
if (this.closed) {
|
|
2067
|
+
if (existingRetry)
|
|
2068
|
+
this.batchRetries.delete(existingRetry);
|
|
2069
|
+
return;
|
|
2070
|
+
}
|
|
2071
|
+
const retry = existingRetry ?? { entries, attempts: 0 };
|
|
2072
|
+
if (existingRetry) {
|
|
2073
|
+
existingRetry.entries.splice(0, existingRetry.entries.length, ...entries);
|
|
2074
|
+
}
|
|
2075
|
+
else {
|
|
2076
|
+
this.batchRetries.add(retry);
|
|
2077
|
+
}
|
|
2078
|
+
retry.attempts += 1;
|
|
2079
|
+
const delay = Math.min(DURABLE_PUBLISHED_HASH_RETRY_INITIAL_MS * 2 ** Math.min(retry.attempts - 1, 6), DURABLE_PUBLISHED_HASH_RETRY_MAX_MS);
|
|
2080
|
+
retry.timer = this.clock.setTimeout(() => {
|
|
2081
|
+
retry.timer = undefined;
|
|
2082
|
+
void this.retryBatch(retry).catch(() => undefined);
|
|
2083
|
+
}, delay);
|
|
2084
|
+
}
|
|
2085
|
+
async retryBatch(retry) {
|
|
2086
|
+
if (this.closed || !this.batchRetries.has(retry))
|
|
2087
|
+
return;
|
|
2088
|
+
if (this.flushPromise)
|
|
2089
|
+
await this.flushPromise;
|
|
2090
|
+
if (this.closed || !this.batchRetries.has(retry))
|
|
2091
|
+
return;
|
|
2092
|
+
if (retry.entries.length === 0) {
|
|
2093
|
+
this.batchRetries.delete(retry);
|
|
2094
|
+
return;
|
|
2095
|
+
}
|
|
2096
|
+
await this.startFlush(retry.entries, retry);
|
|
2097
|
+
}
|
|
2098
|
+
supersedeBatchRetries(relativePath) {
|
|
2099
|
+
for (const retry of this.batchRetries) {
|
|
2100
|
+
for (let index = retry.entries.length - 1; index >= 0; index -= 1) {
|
|
2101
|
+
if (retry.entries[index]?.mutation.relativePath === relativePath) {
|
|
2102
|
+
retry.entries.splice(index, 1);
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
if (retry.entries.length > 0)
|
|
2106
|
+
continue;
|
|
2107
|
+
if (retry.timer !== undefined)
|
|
2108
|
+
this.clock.clearTimeout(retry.timer);
|
|
2109
|
+
this.batchRetries.delete(retry);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
clearBatchRetries() {
|
|
2113
|
+
for (const retry of this.batchRetries) {
|
|
2114
|
+
if (retry.timer !== undefined)
|
|
2115
|
+
this.clock.clearTimeout(retry.timer);
|
|
2116
|
+
}
|
|
2117
|
+
this.batchRetries.clear();
|
|
1863
2118
|
}
|
|
1864
2119
|
}
|
|
1865
2120
|
function boundedDurablePublishedContentHashState(hashes, maxPaths) {
|
|
@@ -1871,58 +2126,31 @@ function boundedDurablePublishedContentHashState(hashes, maxPaths) {
|
|
|
1871
2126
|
}
|
|
1872
2127
|
function reduceDurablePublishedContentHashState(state, record) {
|
|
1873
2128
|
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_SET) {
|
|
1874
|
-
|
|
1875
|
-
const maxPaths = payload.maxPaths;
|
|
1876
|
-
const hashes = { ...state.hashes, [payload.relativePath]: payload.contentHash };
|
|
1877
|
-
const order = state.order.filter((path) => path !== payload.relativePath);
|
|
1878
|
-
order.push(payload.relativePath);
|
|
1879
|
-
while (order.length > maxPaths) {
|
|
1880
|
-
const oldest = order.shift();
|
|
1881
|
-
if (oldest !== undefined)
|
|
1882
|
-
delete hashes[oldest];
|
|
1883
|
-
}
|
|
1884
|
-
return { hashes, order };
|
|
2129
|
+
return reduceDurablePublishedContentHashMutation(state, { kind: "set", ...durablePublishedContentHashSetRecord(record) });
|
|
1885
2130
|
}
|
|
1886
2131
|
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_DELETE) {
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
};
|
|
2132
|
+
return reduceDurablePublishedContentHashMutation(state, { kind: "delete", ...durablePublishedContentHashDeleteRecord(record) });
|
|
2133
|
+
}
|
|
2134
|
+
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_BATCH) {
|
|
2135
|
+
let next = state;
|
|
2136
|
+
for (const mutation of durablePublishedContentHashBatchRecord(record).entries) {
|
|
2137
|
+
next = reduceDurablePublishedContentHashMutation(next, mutation);
|
|
2138
|
+
}
|
|
2139
|
+
return next;
|
|
1896
2140
|
}
|
|
1897
2141
|
throw new Error("published-content-hashes: unknown record type");
|
|
1898
2142
|
}
|
|
1899
2143
|
function reduceDurablePublishedContentHashStateInPlace(state, record) {
|
|
1900
2144
|
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_SET) {
|
|
1901
|
-
|
|
1902
|
-
const order = durablePublishedContentHashOrder(state);
|
|
1903
|
-
order.delete(payload.relativePath);
|
|
1904
|
-
order.set(payload.relativePath, payload.contentHash);
|
|
1905
|
-
Object.defineProperty(state.hashes, payload.relativePath, {
|
|
1906
|
-
configurable: true,
|
|
1907
|
-
enumerable: true,
|
|
1908
|
-
value: payload.contentHash,
|
|
1909
|
-
writable: true,
|
|
1910
|
-
});
|
|
1911
|
-
while (order.size > payload.maxPaths) {
|
|
1912
|
-
const oldest = order.keys().next().value;
|
|
1913
|
-
if (oldest === undefined)
|
|
1914
|
-
break;
|
|
1915
|
-
order.delete(oldest);
|
|
1916
|
-
delete state.hashes[oldest];
|
|
1917
|
-
}
|
|
1918
|
-
return state;
|
|
2145
|
+
return reduceDurablePublishedContentHashMutationInPlace(state, { kind: "set", ...durablePublishedContentHashSetRecord(record) });
|
|
1919
2146
|
}
|
|
1920
2147
|
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_DELETE) {
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
2148
|
+
return reduceDurablePublishedContentHashMutationInPlace(state, { kind: "delete", ...durablePublishedContentHashDeleteRecord(record) });
|
|
2149
|
+
}
|
|
2150
|
+
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_BATCH) {
|
|
2151
|
+
for (const mutation of durablePublishedContentHashBatchRecord(record).entries) {
|
|
2152
|
+
reduceDurablePublishedContentHashMutationInPlace(state, mutation);
|
|
2153
|
+
}
|
|
1926
2154
|
return state;
|
|
1927
2155
|
}
|
|
1928
2156
|
throw new Error("published-content-hashes: unknown record type");
|
|
@@ -1936,8 +2164,59 @@ function validateDurablePublishedContentHashRecord(record) {
|
|
|
1936
2164
|
durablePublishedContentHashDeleteRecord(record);
|
|
1937
2165
|
return;
|
|
1938
2166
|
}
|
|
2167
|
+
if (record.type === PUBLISHED_CONTENT_HASH_RECORD_BATCH) {
|
|
2168
|
+
durablePublishedContentHashBatchRecord(record);
|
|
2169
|
+
return;
|
|
2170
|
+
}
|
|
1939
2171
|
throw new Error("published-content-hashes: unknown record type");
|
|
1940
2172
|
}
|
|
2173
|
+
function reduceDurablePublishedContentHashMutation(state, mutation) {
|
|
2174
|
+
if (mutation.kind === "delete") {
|
|
2175
|
+
if (!(mutation.relativePath in state.hashes))
|
|
2176
|
+
return state;
|
|
2177
|
+
const hashes = { ...state.hashes };
|
|
2178
|
+
delete hashes[mutation.relativePath];
|
|
2179
|
+
return {
|
|
2180
|
+
hashes,
|
|
2181
|
+
order: state.order.filter((path) => path !== mutation.relativePath),
|
|
2182
|
+
};
|
|
2183
|
+
}
|
|
2184
|
+
const hashes = { ...state.hashes, [mutation.relativePath]: mutation.contentHash };
|
|
2185
|
+
const order = state.order.filter((path) => path !== mutation.relativePath);
|
|
2186
|
+
order.push(mutation.relativePath);
|
|
2187
|
+
while (order.length > mutation.maxPaths) {
|
|
2188
|
+
const oldest = order.shift();
|
|
2189
|
+
if (oldest !== undefined)
|
|
2190
|
+
delete hashes[oldest];
|
|
2191
|
+
}
|
|
2192
|
+
return { hashes, order };
|
|
2193
|
+
}
|
|
2194
|
+
function reduceDurablePublishedContentHashMutationInPlace(state, mutation) {
|
|
2195
|
+
if (mutation.kind === "delete") {
|
|
2196
|
+
if (!Object.hasOwn(state.hashes, mutation.relativePath))
|
|
2197
|
+
return state;
|
|
2198
|
+
durablePublishedContentHashOrder(state).delete(mutation.relativePath);
|
|
2199
|
+
delete state.hashes[mutation.relativePath];
|
|
2200
|
+
return state;
|
|
2201
|
+
}
|
|
2202
|
+
const order = durablePublishedContentHashOrder(state);
|
|
2203
|
+
order.delete(mutation.relativePath);
|
|
2204
|
+
order.set(mutation.relativePath, mutation.contentHash);
|
|
2205
|
+
Object.defineProperty(state.hashes, mutation.relativePath, {
|
|
2206
|
+
configurable: true,
|
|
2207
|
+
enumerable: true,
|
|
2208
|
+
value: mutation.contentHash,
|
|
2209
|
+
writable: true,
|
|
2210
|
+
});
|
|
2211
|
+
while (order.size > mutation.maxPaths) {
|
|
2212
|
+
const oldest = order.keys().next().value;
|
|
2213
|
+
if (oldest === undefined)
|
|
2214
|
+
break;
|
|
2215
|
+
order.delete(oldest);
|
|
2216
|
+
delete state.hashes[oldest];
|
|
2217
|
+
}
|
|
2218
|
+
return state;
|
|
2219
|
+
}
|
|
1941
2220
|
function durablePublishedContentHashSetRecord(record) {
|
|
1942
2221
|
const payload = record.payload;
|
|
1943
2222
|
if (typeof payload.relativePath !== "string" ||
|
|
@@ -1958,6 +2237,33 @@ function durablePublishedContentHashDeleteRecord(record) {
|
|
|
1958
2237
|
}
|
|
1959
2238
|
return { relativePath: payload.relativePath };
|
|
1960
2239
|
}
|
|
2240
|
+
function durablePublishedContentHashBatchRecord(record) {
|
|
2241
|
+
const payload = record.payload;
|
|
2242
|
+
if (!Array.isArray(payload.entries) || payload.entries.length === 0) {
|
|
2243
|
+
throw new Error("published-content-hashes: invalid batch record");
|
|
2244
|
+
}
|
|
2245
|
+
return {
|
|
2246
|
+
entries: payload.entries.map((entry) => {
|
|
2247
|
+
if (typeof entry !== "object" || entry === null) {
|
|
2248
|
+
throw new Error("published-content-hashes: invalid batch record");
|
|
2249
|
+
}
|
|
2250
|
+
const mutation = entry;
|
|
2251
|
+
if (mutation.kind === "set") {
|
|
2252
|
+
return {
|
|
2253
|
+
kind: "set",
|
|
2254
|
+
...durablePublishedContentHashSetRecord({ ...record, payload: entry }),
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2257
|
+
if (mutation.kind === "delete") {
|
|
2258
|
+
return {
|
|
2259
|
+
kind: "delete",
|
|
2260
|
+
...durablePublishedContentHashDeleteRecord({ ...record, payload: entry }),
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
throw new Error("published-content-hashes: invalid batch record");
|
|
2264
|
+
}),
|
|
2265
|
+
};
|
|
2266
|
+
}
|
|
1961
2267
|
function durablePublishedContentHashOrder(state) {
|
|
1962
2268
|
const existing = durablePublishedContentHashOrders.get(state);
|
|
1963
2269
|
if (existing)
|
|
@@ -2180,8 +2486,8 @@ export class PushEventEmitter {
|
|
|
2180
2486
|
*/
|
|
2181
2487
|
deferredPublishes = new Map();
|
|
2182
2488
|
deferredPublishTimers = new Map();
|
|
2183
|
-
durablePublishedHashRetries = new Map();
|
|
2184
2489
|
disposed = false;
|
|
2490
|
+
disposePromise = null;
|
|
2185
2491
|
_realtimeUnavailable = false;
|
|
2186
2492
|
internalSeq = 0;
|
|
2187
2493
|
nextSeq;
|
|
@@ -2281,7 +2587,7 @@ export class PushEventEmitter {
|
|
|
2281
2587
|
// A pulled revision was never locally announced, so it must not begin the
|
|
2282
2588
|
// per-path announcement quiet period for a subsequent local edit.
|
|
2283
2589
|
this.lastPublishedContentHashes.set(relativePath, contentHash);
|
|
2284
|
-
void this.setDurablePublishedContentHash(relativePath, contentHash);
|
|
2590
|
+
void this.setDurablePublishedContentHash(relativePath, contentHash, undefined, false);
|
|
2285
2591
|
}
|
|
2286
2592
|
async emitEntries(entries) {
|
|
2287
2593
|
const attempts = [];
|
|
@@ -2305,6 +2611,10 @@ export class PushEventEmitter {
|
|
|
2305
2611
|
}
|
|
2306
2612
|
}
|
|
2307
2613
|
async emitOne(absolutePath, relativePath) {
|
|
2614
|
+
// TreeChangeBatch has multiple producers. Treat its relative path as
|
|
2615
|
+
// untrusted until this wire boundary so no producer can publish a
|
|
2616
|
+
// platform-separated key to the push protocol.
|
|
2617
|
+
relativePath = toPosixKey(relativePath);
|
|
2308
2618
|
let event;
|
|
2309
2619
|
try {
|
|
2310
2620
|
const st = await stat(absolutePath);
|
|
@@ -2446,7 +2756,10 @@ export class PushEventEmitter {
|
|
|
2446
2756
|
}
|
|
2447
2757
|
openDurablePublishedContentHashes(stateDir, initialHashes) {
|
|
2448
2758
|
try {
|
|
2449
|
-
return new DurablePublishedContentHashes(stateDir, `${this.originTenantId}:${this.originDeviceId}`, this.lastPublishedContentHashMaxPaths, initialHashes)
|
|
2759
|
+
return new DurablePublishedContentHashes(stateDir, `${this.originTenantId}:${this.originDeviceId}`, this.lastPublishedContentHashMaxPaths, initialHashes, this.clock, (error, mutationCount, relativePath) => {
|
|
2760
|
+
error.message += `; durable published-content-hash batch affected ${mutationCount} mutation${mutationCount === 1 ? "" : "s"}`;
|
|
2761
|
+
this.onError(error, relativePath === undefined ? {} : { relativePath });
|
|
2762
|
+
});
|
|
2450
2763
|
}
|
|
2451
2764
|
catch (err) {
|
|
2452
2765
|
this.onError(err instanceof Error ? err : new Error(String(err)), {});
|
|
@@ -2477,30 +2790,32 @@ export class PushEventEmitter {
|
|
|
2477
2790
|
return false;
|
|
2478
2791
|
}
|
|
2479
2792
|
}
|
|
2480
|
-
async setDurablePublishedContentHash(relativePath, contentHash, generation) {
|
|
2481
|
-
const operation = { kind: "set", contentHash, generation };
|
|
2793
|
+
async setDurablePublishedContentHash(relativePath, contentHash, generation, flushImmediately = true) {
|
|
2482
2794
|
try {
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2795
|
+
if (!this.durablePublishedContentHashes)
|
|
2796
|
+
return;
|
|
2797
|
+
if (generation === undefined) {
|
|
2798
|
+
await this.durablePublishedContentHashes.set(relativePath, contentHash, this.lastPublishedContentHashMaxPaths, flushImmediately);
|
|
2799
|
+
return;
|
|
2800
|
+
}
|
|
2801
|
+
await this.durablePublishedContentHashes.setIfCurrent(relativePath, contentHash, this.lastPublishedContentHashMaxPaths, () => this.isPublishedContentHashGenerationCurrent(relativePath, generation));
|
|
2488
2802
|
}
|
|
2489
2803
|
catch (err) {
|
|
2490
|
-
this.
|
|
2804
|
+
this.onError(err instanceof Error ? err : new Error(String(err)), { relativePath });
|
|
2491
2805
|
}
|
|
2492
2806
|
}
|
|
2493
|
-
async deleteDurablePublishedContentHash(relativePath, generation) {
|
|
2494
|
-
const operation = { kind: "delete", generation };
|
|
2807
|
+
async deleteDurablePublishedContentHash(relativePath, generation, flushImmediately = true) {
|
|
2495
2808
|
try {
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
|
|
2500
|
-
|
|
2809
|
+
if (!this.durablePublishedContentHashes)
|
|
2810
|
+
return;
|
|
2811
|
+
if (generation === undefined) {
|
|
2812
|
+
await this.durablePublishedContentHashes.delete(relativePath, flushImmediately);
|
|
2813
|
+
return;
|
|
2814
|
+
}
|
|
2815
|
+
await this.durablePublishedContentHashes.deleteIfCurrent(relativePath, () => this.isPublishedContentHashGenerationCurrent(relativePath, generation));
|
|
2501
2816
|
}
|
|
2502
2817
|
catch (err) {
|
|
2503
|
-
this.
|
|
2818
|
+
this.onError(err instanceof Error ? err : new Error(String(err)), { relativePath });
|
|
2504
2819
|
}
|
|
2505
2820
|
}
|
|
2506
2821
|
/**
|
|
@@ -2524,79 +2839,6 @@ export class PushEventEmitter {
|
|
|
2524
2839
|
return true;
|
|
2525
2840
|
return this.publishedContentHashUnchangedSince(relativePath, generationCounterAtStart);
|
|
2526
2841
|
}
|
|
2527
|
-
async persistDurablePublishedContentHash(relativePath, operation, shouldPersist) {
|
|
2528
|
-
if (!this.durablePublishedContentHashes)
|
|
2529
|
-
return true;
|
|
2530
|
-
if (operation.kind === "set") {
|
|
2531
|
-
if (shouldPersist) {
|
|
2532
|
-
return this.durablePublishedContentHashes.setIfCurrent(relativePath, operation.contentHash, this.lastPublishedContentHashMaxPaths, shouldPersist);
|
|
2533
|
-
}
|
|
2534
|
-
await this.durablePublishedContentHashes.set(relativePath, operation.contentHash, this.lastPublishedContentHashMaxPaths);
|
|
2535
|
-
}
|
|
2536
|
-
else {
|
|
2537
|
-
if (shouldPersist) {
|
|
2538
|
-
return this.durablePublishedContentHashes.deleteIfCurrent(relativePath, shouldPersist);
|
|
2539
|
-
}
|
|
2540
|
-
await this.durablePublishedContentHashes.delete(relativePath);
|
|
2541
|
-
}
|
|
2542
|
-
return true;
|
|
2543
|
-
}
|
|
2544
|
-
handleDurablePublishedContentHashFailure(relativePath, operation, err) {
|
|
2545
|
-
const error = err instanceof Error ? err : new Error(String(err));
|
|
2546
|
-
this.onError(error, { relativePath });
|
|
2547
|
-
if (error instanceof StateStoreLockError) {
|
|
2548
|
-
this.scheduleDurablePublishedContentHashRetry(relativePath, operation);
|
|
2549
|
-
}
|
|
2550
|
-
}
|
|
2551
|
-
clearDurablePublishedContentHashRetry(relativePath) {
|
|
2552
|
-
const retry = this.durablePublishedHashRetries.get(relativePath);
|
|
2553
|
-
if (!retry)
|
|
2554
|
-
return;
|
|
2555
|
-
if (retry.timer !== undefined)
|
|
2556
|
-
this.clock.clearTimeout(retry.timer);
|
|
2557
|
-
this.durablePublishedHashRetries.delete(relativePath);
|
|
2558
|
-
}
|
|
2559
|
-
scheduleDurablePublishedContentHashRetry(relativePath, operation) {
|
|
2560
|
-
if (this.disposed || !this.durablePublishedContentHashes)
|
|
2561
|
-
return;
|
|
2562
|
-
const existing = this.durablePublishedHashRetries.get(relativePath);
|
|
2563
|
-
if (existing?.timer !== undefined) {
|
|
2564
|
-
existing.operation = operation;
|
|
2565
|
-
return;
|
|
2566
|
-
}
|
|
2567
|
-
const retry = {
|
|
2568
|
-
operation,
|
|
2569
|
-
attempts: (existing?.attempts ?? 0) + 1,
|
|
2570
|
-
};
|
|
2571
|
-
const delay = Math.min(DURABLE_PUBLISHED_HASH_RETRY_INITIAL_MS * 2 ** Math.min(retry.attempts - 1, 6), DURABLE_PUBLISHED_HASH_RETRY_MAX_MS);
|
|
2572
|
-
retry.timer = this.clock.setTimeout(() => {
|
|
2573
|
-
retry.timer = undefined;
|
|
2574
|
-
void this.retryDurablePublishedContentHash(relativePath);
|
|
2575
|
-
}, delay);
|
|
2576
|
-
this.durablePublishedHashRetries.set(relativePath, retry);
|
|
2577
|
-
}
|
|
2578
|
-
async retryDurablePublishedContentHash(relativePath) {
|
|
2579
|
-
if (this.disposed)
|
|
2580
|
-
return;
|
|
2581
|
-
const retry = this.durablePublishedHashRetries.get(relativePath);
|
|
2582
|
-
if (!retry)
|
|
2583
|
-
return;
|
|
2584
|
-
try {
|
|
2585
|
-
const persisted = await this.persistDurablePublishedContentHash(relativePath, retry.operation, () => this.durablePublishedHashRetries.get(relativePath) === retry &&
|
|
2586
|
-
this.isPublishedContentHashGenerationCurrent(relativePath, retry.operation.generation));
|
|
2587
|
-
if (this.durablePublishedHashRetries.get(relativePath) === retry && !persisted) {
|
|
2588
|
-
this.durablePublishedHashRetries.delete(relativePath);
|
|
2589
|
-
}
|
|
2590
|
-
else if (persisted && this.durablePublishedHashRetries.get(relativePath) === retry) {
|
|
2591
|
-
this.durablePublishedHashRetries.delete(relativePath);
|
|
2592
|
-
}
|
|
2593
|
-
}
|
|
2594
|
-
catch (err) {
|
|
2595
|
-
if (this.durablePublishedHashRetries.get(relativePath) === retry) {
|
|
2596
|
-
this.handleDurablePublishedContentHashFailure(relativePath, retry.operation, err);
|
|
2597
|
-
}
|
|
2598
|
-
}
|
|
2599
|
-
}
|
|
2600
2842
|
/** Remember one latest-wins uploaded revision until its publish interval ends. */
|
|
2601
2843
|
deferUpsert(event, lastPublishedAt) {
|
|
2602
2844
|
const relativePath = event.relativePath;
|
|
@@ -2637,21 +2879,46 @@ export class PushEventEmitter {
|
|
|
2637
2879
|
this.clock.clearTimeout(timer);
|
|
2638
2880
|
this.deferredPublishTimers.delete(relativePath);
|
|
2639
2881
|
}
|
|
2640
|
-
/**
|
|
2882
|
+
/** Flush durable publication state, cancel trailing publishes, and become inert. */
|
|
2641
2883
|
dispose() {
|
|
2642
|
-
if (this.
|
|
2643
|
-
return;
|
|
2884
|
+
if (this.disposePromise !== null)
|
|
2885
|
+
return this.disposePromise;
|
|
2644
2886
|
this.disposed = true;
|
|
2645
|
-
|
|
2646
|
-
this.
|
|
2887
|
+
this.disposePromise = (async () => {
|
|
2888
|
+
for (const timer of this.deferredPublishTimers.values()) {
|
|
2889
|
+
this.clock.clearTimeout(timer);
|
|
2890
|
+
}
|
|
2891
|
+
this.deferredPublishTimers.clear();
|
|
2892
|
+
this.deferredPublishes.clear();
|
|
2893
|
+
if (await this.waitForPublishDrain()) {
|
|
2894
|
+
await this.disposeDurablePublishedContentHashes();
|
|
2895
|
+
return;
|
|
2896
|
+
}
|
|
2897
|
+
this.onError(new Error(`PushEventEmitter: timed out draining active publish after ${PUSH_EVENT_EMITTER_DISPOSE_DRAIN_MS}ms; durable memo remains open until it settles`), {});
|
|
2898
|
+
void this.publishTail.then(() => this.disposeDurablePublishedContentHashes());
|
|
2899
|
+
})();
|
|
2900
|
+
return this.disposePromise;
|
|
2901
|
+
}
|
|
2902
|
+
async waitForPublishDrain() {
|
|
2903
|
+
let timeout;
|
|
2904
|
+
const deadline = new Promise((resolve) => {
|
|
2905
|
+
timeout = this.clock.setTimeout(() => resolve(false), PUSH_EVENT_EMITTER_DISPOSE_DRAIN_MS);
|
|
2906
|
+
});
|
|
2907
|
+
const drained = await Promise.race([
|
|
2908
|
+
this.publishTail.then(() => true, () => true),
|
|
2909
|
+
deadline,
|
|
2910
|
+
]);
|
|
2911
|
+
if (timeout !== undefined)
|
|
2912
|
+
this.clock.clearTimeout(timeout);
|
|
2913
|
+
return drained;
|
|
2914
|
+
}
|
|
2915
|
+
async disposeDurablePublishedContentHashes() {
|
|
2916
|
+
try {
|
|
2917
|
+
await this.durablePublishedContentHashes?.dispose();
|
|
2647
2918
|
}
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
for (const retry of this.durablePublishedHashRetries.values()) {
|
|
2651
|
-
if (retry.timer !== undefined)
|
|
2652
|
-
this.clock.clearTimeout(retry.timer);
|
|
2919
|
+
catch (err) {
|
|
2920
|
+
this.onError(err instanceof Error ? err : new Error(String(err)), {});
|
|
2653
2921
|
}
|
|
2654
|
-
this.durablePublishedHashRetries.clear();
|
|
2655
2922
|
// Release the per-path maps so a disposed emitter retains nothing.
|
|
2656
2923
|
this.publishedContentHashGenerations.clear();
|
|
2657
2924
|
}
|
|
@@ -2664,7 +2931,7 @@ export class PushEventEmitter {
|
|
|
2664
2931
|
lastPublishedContentHashes: this.lastPublishedContentHashes.size,
|
|
2665
2932
|
publishedContentHashGenerations: this.publishedContentHashGenerations.size,
|
|
2666
2933
|
deferredPublishes: this.deferredPublishes.size,
|
|
2667
|
-
durablePublishedHashRetries: this.
|
|
2934
|
+
durablePublishedHashRetries: this.durablePublishedContentHashes?.pendingRetryCount ?? 0,
|
|
2668
2935
|
// Runner callers no longer pass the session-wide object. Keep a fixed
|
|
2669
2936
|
// zero-valued token so heap telemetry explicitly records its absence.
|
|
2670
2937
|
initialPublishedContentHashes: 0,
|