@indigoai-us/hq-cloud 6.16.8 → 6.16.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/sync-runner-watch-loop.d.ts +21 -7
- package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
- package/dist/bin/sync-runner-watch-loop.js +79 -21
- package/dist/bin/sync-runner-watch-loop.js.map +1 -1
- package/dist/bin/sync-runner-watch-loop.test.js +40 -1
- package/dist/bin/sync-runner-watch-loop.test.js.map +1 -1
- package/dist/bin/sync-runner.test.js +147 -38
- package/dist/bin/sync-runner.test.js.map +1 -1
- package/dist/sync/state-store.d.ts +7 -0
- package/dist/sync/state-store.d.ts.map +1 -1
- package/dist/sync/state-store.js +113 -17
- package/dist/sync/state-store.js.map +1 -1
- package/dist/sync/state-store.test.js +79 -1
- package/dist/sync/state-store.test.js.map +1 -1
- package/package.json +1 -1
|
@@ -30,6 +30,12 @@ export declare function setStateStoreScopeLockWriteHookForTest(hook: (() => void
|
|
|
30
30
|
export declare function setStateStoreAsyncLockTimeoutForTest(timeoutMs: number | undefined): void;
|
|
31
31
|
/** Test-only seam for exercising synchronous lock contention without a five-second wait. */
|
|
32
32
|
export declare function setStateStoreSyncLockTimeoutForTest(timeoutMs: number | undefined): void;
|
|
33
|
+
/** Test-only clock for contended fold retries. Production uses `Date.now()`. */
|
|
34
|
+
export declare function setStateStoreNowMsForTest(nowMs: (() => number) | undefined): void;
|
|
35
|
+
/** Test-only sleep for contended fold backoff. Production uses `Atomics.wait`. */
|
|
36
|
+
export declare function setStateStoreSleepForTest(sleepMs: ((ms: number) => void) | undefined): void;
|
|
37
|
+
/** Test-only RNG for fold-retry jitter. Production uses `Math.random()`. */
|
|
38
|
+
export declare function setStateStoreRandomForTest(random: (() => number) | undefined): void;
|
|
33
39
|
export interface StateStoreRecord<T = unknown> {
|
|
34
40
|
sequence: number;
|
|
35
41
|
type: number;
|
|
@@ -256,6 +262,7 @@ export declare class StateStore<T> {
|
|
|
256
262
|
* still validates the source generation after the off-lock snapshot stage.
|
|
257
263
|
*/
|
|
258
264
|
replaceStateWith(transform: (state: T) => T): void;
|
|
265
|
+
private generationPublishRetryBudgetMs;
|
|
259
266
|
private replaceStateAttempt;
|
|
260
267
|
/** Create G+1 without ever merging two generations during recovery. */
|
|
261
268
|
compact(): void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../../src/sync/state-store.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../../src/sync/state-store.ts"],"names":[],"mappings":"AAkFA,wBAAgB,2CAA2C,IAAI,IAAI,CAElE;AAED,wBAAgB,yCAAyC,IAAI,MAAM,CAElE;AAED,wBAAgB,kDAAkD,IAAI,IAAI,CAEzE;AAED,wBAAgB,gDAAgD,IAAI,MAAM,CAEzE;AAED,wBAAgB,yCAAyC,IAAI,IAAI,CAEhE;AAED,wBAAgB,uCAAuC,IAAI,MAAM,CAEhE;AAED,6EAA6E;AAC7E,wBAAgB,qCAAqC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE1F;AAED,kFAAkF;AAClF,wBAAgB,oCAAoC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEzF;AAED,2EAA2E;AAC3E,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEpF;AAED,oFAAoF;AACpF,wBAAgB,4CAA4C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjG;AAED,0EAA0E;AAC1E,wBAAgB,mCAAmC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAExF;AAED,2DAA2D;AAC3D,wBAAgB,mCAAmC,CACjD,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,MAAM,CAAC,GAAG,SAAS,GAC3G,IAAI,CAEN;AAED,4DAA4D;AAC5D,wBAAgB,0CAA0C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE/F;AAED,+EAA+E;AAC/E,wBAAgB,4CAA4C,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjG;AAED,sFAAsF;AACtF,wBAAgB,yCAAyC,CACvD,IAAI,EAAE,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAC7C,IAAI,CAEN;AAED,+EAA+E;AAC/E,wBAAgB,+CAA+C,CAC7D,IAAI,EAAE,CAAC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,SAAS,GAC7C,IAAI,CAEN;AAED,6EAA6E;AAC7E,wBAAgB,sCAAsC,CAAC,IAAI,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE3F;AAED,qFAAqF;AACrF,wBAAgB,oCAAoC,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAExF;AAED,4FAA4F;AAC5F,wBAAgB,mCAAmC,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,CAEvF;AAED,gFAAgF;AAChF,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAEjF;AAED,kFAAkF;AAClF,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC,GAAG,SAAS,GAAG,IAAI,CAE3F;AAED,4EAA4E;AAC5E,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAEnF;AA+ED,MAAM,WAAW,gBAAgB,CAAC,CAAC,GAAG,OAAO;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,CAAC,CAAC;CACZ;AAED,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,CAAC,CAAC;IAChB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,KAAK,CAAC,CAAC;IAClD;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACpD,4DAA4D;IAC5D,0BAA0B,CAAC,EAAE,MAAM,CAAC;IACpC,gFAAgF;IAChF,2BAA2B,CAAC,EAAE,MAAM,CAAC;CACtC;AAED,MAAM,WAAW,uBAAuB;IACtC,iEAAiE;IACjE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IAC5B,4EAA4E;IAC5E,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,sBAAuB,SAAQ,gBAAgB;IAC9D,OAAO,EAAE,uBAAuB,CAAC;CAClC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,qBAAqB,CAAC,CAAC;IACtC,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,uBAAuB;IACtC,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,kEAAkE;AAClE,MAAM,WAAW,4BAA4B;IAC3C,UAAU,EAAE,MAAM,CAAC;IACnB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,QAAQ,EAAE,4BAA4B,CAAC;IAChD,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,OAAO,IAAI,IAAI,CAAC;CACjB;AAED,+DAA+D;AAC/D,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA0B;IACzC,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,2BAA2B,GACnC;IAAE,MAAM,EAAE,SAAS,GAAG,YAAY,CAAA;CAAE,GACpC,CAAC;IAAE,MAAM,EAAE,UAAU,CAAA;CAAE,GAAG,sBAAsB,CAAC,CAAC;AAEtD,MAAM,WAAW,4BAA4B;IAC3C,yEAAyE;IACzE,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,uEAAuE;IACvE,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;CAC1D;AAiCD,qBAAa,yBAA0B,SAAQ,KAAK;gBACtC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAIpD;AAED,MAAM,MAAM,yBAAyB,GAAG,yBAAyB,CAAC;AAElE,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,QAAQ,CAAC,MAAM,CAAC,EAAE,yBAAyB,CAAC;gBAEhC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,MAAM,CAAC,EAAE,yBAAyB,CAAA;KAAE;CAK7F;AAYD,+DAA+D;AAC/D,qBAAa,UAAU,CAAC,CAAC;IAarB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAZ1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,KAAK,CAAI;IACjB,OAAO,CAAC,kBAAkB,CAAS;IACnC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,iCAAiC,CAAS;IAElD,OAAO;IAiBP,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO;IAUxD,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;IAuC5D,uEAAuE;WAC1D,SAAS,CAAC,CAAC,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAsChF;;;;;;;;;OASG;IACH,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,qBAAqB,CAAC,CAAC,CAAC,EAAE,GAAG,sBAAsB,EAAE;IA2D7F;;;;;OAKG;IACH,MAAM,CAAC,cAAc,CAAC,CAAC,EACrB,OAAO,EAAE,iBAAiB,CAAC,CAAC,CAAC,EAC7B,WAAW,GAAE,4BAAiC,GAC7C,2BAA2B;IAqG9B,QAAQ,IAAI,CAAC;IAIb,aAAa,IAAI,MAAM;IAIvB,qBAAqB,IAAI,MAAM;IAI/B,gBAAgB,IAAI,uBAAuB;IAI3C;;;;;;OAMG;IACH,cAAc,CACZ,WAAW,GAAE,4BAAiC,GAC7C,2BAA2B;IAI9B,8EAA8E;IAC9E,eAAe,IAAI,OAAO;IAO1B,oEAAoE;YACtD,oBAAoB;IAiFlC,qBAAqB,IAAI,4BAA4B;IAQrD;;;;;;;OAOG;IACH,gBAAgB,CAAC,QAAQ,GAAE,4BAA2D,GAAG,OAAO;IAUhG;;;;OAIG;IACH,oBAAoB,IAAI,uBAAuB;IA4B/C;;;;OAIG;IACF,mBAAmB,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,gBAAgB,CAAC,oBAAoB,CAAC;IAgBrF,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,sBAAsB;IA4BzE;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,SAAI,GAAG,OAAO,CAAC,sBAAsB,CAAC;IAgC7F,iEAAiE;IAC3D,aAAa,CACjB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,OAAO,EAChB,YAAY,EAAE,MAAM,OAAO,EAC3B,KAAK,SAAI,GACR,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAiC9C;;;;;;OAMG;IACG,sBAAsB,CAC1B,IAAI,EAAE,MAAM,EACZ,cAAc,EAAE,MAAM,OAAO,GAAG,SAAS,EACzC,KAAK,SAAI,GACR,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAkC9C,sEAAsE;IACtE,OAAO,IAAI,uBAAuB;IAUlC,6DAA6D;IACvD,YAAY,IAAI,OAAO,CAAC,uBAAuB,CAAC;IAUtD,OAAO,CAAC,cAAc;IAiCtB;;;;;;;;OAQG;IACH,YAAY,CAAC,SAAS,EAAE,CAAC,GAAG,IAAI;IAIhC;;;;;;;OAOG;IACH,gBAAgB,CAAC,SAAS,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI;IAWlD,OAAO,CAAC,8BAA8B;IAWtC,OAAO,CAAC,mBAAmB;IAgE3B,uEAAuE;IACvE,OAAO,IAAI,IAAI;IAuBf,OAAO,CAAC,eAAe;IA+BvB,OAAO,CAAC,kCAAkC;YAU5B,uCAAuC;IAUrD,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,cAAc;IAqBtB,gFAAgF;IAChF,OAAO,CAAC,qBAAqB;IAsB7B,OAAO,CAAC,iBAAiB;CAS1B"}
|
package/dist/sync/state-store.js
CHANGED
|
@@ -45,8 +45,17 @@ const MALFORMED_LOCK_GRACE_MS = 250;
|
|
|
45
45
|
// exceptional event-loop pause bounded; async callers wait without parking.
|
|
46
46
|
const SYNC_LOCK_TIMEOUT_MS = 5_000;
|
|
47
47
|
const ASYNC_LOCK_TIMEOUT_MS = 60_000;
|
|
48
|
-
const REPLACE_STATE_MAX_ATTEMPTS = 3;
|
|
49
48
|
const OPTIMISTIC_RECOVERY_MAX_RESAMPLES = 3;
|
|
49
|
+
/** Base wall-clock budget for one contended fold. Scaled by WAL/record pressure. */
|
|
50
|
+
const COMPACTION_RETRY_BASE_BUDGET_MS = 50;
|
|
51
|
+
/** Append must not block indefinitely waiting to fold. */
|
|
52
|
+
const COMPACTION_RETRY_MAX_BUDGET_MS = 2_000;
|
|
53
|
+
const COMPACTION_RETRY_BACKOFF_BASE_MS = 5;
|
|
54
|
+
const COMPACTION_RETRY_BACKOFF_MAX_MS = 250;
|
|
55
|
+
/** Preserve the historical three-shot floor so one slow stage cannot exhaust the budget. */
|
|
56
|
+
const COMPACTION_RETRY_MIN_ATTEMPTS = 3;
|
|
57
|
+
/** Safety cap so a frozen test clock cannot spin forever. */
|
|
58
|
+
const COMPACTION_RETRY_MAX_ATTEMPTS = 10_000;
|
|
50
59
|
// Test instrumentation for materialisation regressions. Kept at the storage
|
|
51
60
|
// seam so a fixture counts actual snapshot JSON decodes, not API invocations.
|
|
52
61
|
let snapshotRecoveryCount = 0;
|
|
@@ -65,6 +74,9 @@ let scopeLockWriteHookForTest;
|
|
|
65
74
|
let afterOptimisticRecoveryHookForTest;
|
|
66
75
|
let asyncLockTimeoutMsForTest;
|
|
67
76
|
let syncLockTimeoutMsForTest;
|
|
77
|
+
let nowMsForTest;
|
|
78
|
+
let sleepMsForTest;
|
|
79
|
+
let randomForTest;
|
|
68
80
|
export function resetStateStoreSnapshotRecoveryCountForTest() {
|
|
69
81
|
snapshotRecoveryCount = 0;
|
|
70
82
|
}
|
|
@@ -135,6 +147,71 @@ export function setStateStoreAsyncLockTimeoutForTest(timeoutMs) {
|
|
|
135
147
|
export function setStateStoreSyncLockTimeoutForTest(timeoutMs) {
|
|
136
148
|
syncLockTimeoutMsForTest = timeoutMs;
|
|
137
149
|
}
|
|
150
|
+
/** Test-only clock for contended fold retries. Production uses `Date.now()`. */
|
|
151
|
+
export function setStateStoreNowMsForTest(nowMs) {
|
|
152
|
+
nowMsForTest = nowMs;
|
|
153
|
+
}
|
|
154
|
+
/** Test-only sleep for contended fold backoff. Production uses `Atomics.wait`. */
|
|
155
|
+
export function setStateStoreSleepForTest(sleepMs) {
|
|
156
|
+
sleepMsForTest = sleepMs;
|
|
157
|
+
}
|
|
158
|
+
/** Test-only RNG for fold-retry jitter. Production uses `Math.random()`. */
|
|
159
|
+
export function setStateStoreRandomForTest(random) {
|
|
160
|
+
randomForTest = random;
|
|
161
|
+
}
|
|
162
|
+
function currentTimeMs() {
|
|
163
|
+
return nowMsForTest?.() ?? Date.now();
|
|
164
|
+
}
|
|
165
|
+
function sleepSyncMs(ms) {
|
|
166
|
+
if (sleepMsForTest) {
|
|
167
|
+
sleepMsForTest(ms);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
if (ms <= 0)
|
|
171
|
+
return;
|
|
172
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
173
|
+
}
|
|
174
|
+
async function sleepAsyncMs(ms) {
|
|
175
|
+
if (sleepMsForTest) {
|
|
176
|
+
sleepMsForTest(ms);
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
if (ms <= 0)
|
|
180
|
+
return;
|
|
181
|
+
await new Promise((resolve) => {
|
|
182
|
+
setTimeout(resolve, ms);
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
function compactionRetryPressure(walBytes, maxWalBytes, recordCount, maxRecords) {
|
|
186
|
+
const walRatio = walBytes / Math.max(1, maxWalBytes);
|
|
187
|
+
const recordRatio = recordCount / Math.max(1, maxRecords);
|
|
188
|
+
return Math.max(1, walRatio, recordRatio);
|
|
189
|
+
}
|
|
190
|
+
function compactionRetryBudgetMs(pressure) {
|
|
191
|
+
return Math.min(COMPACTION_RETRY_MAX_BUDGET_MS, Math.max(COMPACTION_RETRY_BASE_BUDGET_MS, Math.ceil(COMPACTION_RETRY_BASE_BUDGET_MS * pressure)));
|
|
192
|
+
}
|
|
193
|
+
function compactionRetryBackoffMs(failedAttempts) {
|
|
194
|
+
const shift = Math.min(16, Math.max(0, failedAttempts - 1));
|
|
195
|
+
const exponential = Math.min(COMPACTION_RETRY_BACKOFF_MAX_MS, COMPACTION_RETRY_BACKOFF_BASE_MS * (2 ** shift));
|
|
196
|
+
const unit = Math.min(1, Math.max(0, randomForTest?.() ?? Math.random()));
|
|
197
|
+
return Math.max(1, Math.floor(exponential * (0.5 + 0.5 * unit)));
|
|
198
|
+
}
|
|
199
|
+
function retryGenerationPublishSync(tryOnce, budgetMs) {
|
|
200
|
+
const startedAt = currentTimeMs();
|
|
201
|
+
let attempts = 0;
|
|
202
|
+
while (attempts < COMPACTION_RETRY_MAX_ATTEMPTS) {
|
|
203
|
+
attempts += 1;
|
|
204
|
+
if (tryOnce())
|
|
205
|
+
return { ok: true, attempts, elapsedMs: currentTimeMs() - startedAt };
|
|
206
|
+
const elapsedMs = currentTimeMs() - startedAt;
|
|
207
|
+
if (attempts >= COMPACTION_RETRY_MIN_ATTEMPTS && elapsedMs >= budgetMs) {
|
|
208
|
+
return { ok: false, attempts, elapsedMs };
|
|
209
|
+
}
|
|
210
|
+
const delay = compactionRetryBackoffMs(attempts);
|
|
211
|
+
sleepSyncMs(attempts < COMPACTION_RETRY_MIN_ATTEMPTS ? delay : Math.min(delay, Math.max(1, budgetMs - elapsedMs)));
|
|
212
|
+
}
|
|
213
|
+
return { ok: false, attempts, elapsedMs: currentTimeMs() - startedAt };
|
|
214
|
+
}
|
|
138
215
|
export class StateStoreCorruptionError extends Error {
|
|
139
216
|
constructor(message, options) {
|
|
140
217
|
super(`state-store corruption: ${message}`, options);
|
|
@@ -478,7 +555,10 @@ export class StateStore {
|
|
|
478
555
|
/** Async counterpart used by callback-originated recovery paths. */
|
|
479
556
|
async compactIfNeededAsync() {
|
|
480
557
|
const compacted = await withRotationClaimAsync(this.stateDir, async () => {
|
|
481
|
-
|
|
558
|
+
const startedAt = currentTimeMs();
|
|
559
|
+
const budget = this.generationPublishRetryBudgetMs();
|
|
560
|
+
let attempts = 0;
|
|
561
|
+
while (attempts < COMPACTION_RETRY_MAX_ATTEMPTS) {
|
|
482
562
|
let resolveStaged;
|
|
483
563
|
let rejectStaged;
|
|
484
564
|
const stagedReady = new Promise((resolve, reject) => {
|
|
@@ -521,10 +601,18 @@ export class StateStore {
|
|
|
521
601
|
});
|
|
522
602
|
if (plan === undefined) {
|
|
523
603
|
await this.recoverNewestGenerationOutsideLockAsync();
|
|
604
|
+
if (currentTimeMs() - startedAt >= budget)
|
|
605
|
+
return false;
|
|
524
606
|
continue;
|
|
525
607
|
}
|
|
526
608
|
if (plan === null)
|
|
527
609
|
return false;
|
|
610
|
+
const reservedPublish = publish;
|
|
611
|
+
const resolve = resolveStaged;
|
|
612
|
+
const reject = rejectStaged;
|
|
613
|
+
if (reservedPublish === undefined || resolve === undefined || reject === undefined)
|
|
614
|
+
return false;
|
|
615
|
+
attempts += 1;
|
|
528
616
|
try {
|
|
529
617
|
const staged = stageGeneration(this.stateDir, this.scopeDigest, plan.next);
|
|
530
618
|
try {
|
|
@@ -534,16 +622,22 @@ export class StateStore {
|
|
|
534
622
|
discardStagedGeneration(staged);
|
|
535
623
|
throw error;
|
|
536
624
|
}
|
|
537
|
-
|
|
625
|
+
resolve(staged);
|
|
538
626
|
}
|
|
539
627
|
catch (error) {
|
|
540
|
-
|
|
541
|
-
await
|
|
628
|
+
reject(error);
|
|
629
|
+
await reservedPublish.catch(() => undefined);
|
|
542
630
|
throw error;
|
|
543
631
|
}
|
|
544
|
-
if (await
|
|
632
|
+
if (await reservedPublish)
|
|
545
633
|
return true;
|
|
634
|
+
const elapsedMs = currentTimeMs() - startedAt;
|
|
635
|
+
if (attempts >= COMPACTION_RETRY_MIN_ATTEMPTS && elapsedMs >= budget)
|
|
636
|
+
return false;
|
|
637
|
+
const delay = compactionRetryBackoffMs(attempts);
|
|
638
|
+
await sleepAsyncMs(attempts < COMPACTION_RETRY_MIN_ATTEMPTS ? delay : Math.min(delay, Math.max(1, budget - elapsedMs)));
|
|
546
639
|
}
|
|
640
|
+
return false;
|
|
547
641
|
});
|
|
548
642
|
return compacted ?? false;
|
|
549
643
|
}
|
|
@@ -824,11 +918,13 @@ export class StateStore {
|
|
|
824
918
|
* still validates the source generation after the off-lock snapshot stage.
|
|
825
919
|
*/
|
|
826
920
|
replaceStateWith(transform) {
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
}
|
|
831
|
-
|
|
921
|
+
const result = retryGenerationPublishSync(() => this.replaceStateAttempt(transform), this.generationPublishRetryBudgetMs());
|
|
922
|
+
if (result.ok)
|
|
923
|
+
return;
|
|
924
|
+
throw new StateStoreLockError(`cannot replace state after ${result.attempts} concurrent writes over ${result.elapsedMs}ms (wal ${this.walBytes} bytes)`);
|
|
925
|
+
}
|
|
926
|
+
generationPublishRetryBudgetMs() {
|
|
927
|
+
return compactionRetryBudgetMs(compactionRetryPressure(this.walBytes, this.options.maxWalBytesBeforeCompaction, this.recordCount, this.options.maxRecordsBeforeCompaction));
|
|
832
928
|
}
|
|
833
929
|
replaceStateAttempt(transform) {
|
|
834
930
|
const plan = withScopeLock(this.stateDir, () => {
|
|
@@ -903,8 +999,9 @@ export class StateStore {
|
|
|
903
999
|
withRotationClaim(this.stateDir, () => {
|
|
904
1000
|
// Appenders do not wait for rotation.lock. If one advances the source
|
|
905
1001
|
// while this claim owns off-lock staging, it will skip its own compact;
|
|
906
|
-
// keep this claim and retry from the refreshed source instead.
|
|
907
|
-
|
|
1002
|
+
// keep this claim and retry from the refreshed source instead. The retry
|
|
1003
|
+
// is time-bounded so a contended append cannot hang indefinitely.
|
|
1004
|
+
retryGenerationPublishSync(() => {
|
|
908
1005
|
const plan = withScopeLock(this.stateDir, () => {
|
|
909
1006
|
if (!this.refreshFromDisk())
|
|
910
1007
|
return undefined;
|
|
@@ -912,11 +1009,10 @@ export class StateStore {
|
|
|
912
1009
|
});
|
|
913
1010
|
if (!plan) {
|
|
914
1011
|
this.recoverNewestGenerationOutsideLock();
|
|
915
|
-
|
|
1012
|
+
return false;
|
|
916
1013
|
}
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
}
|
|
1014
|
+
return this.publishCompactionPlan(plan);
|
|
1015
|
+
}, this.generationPublishRetryBudgetMs());
|
|
920
1016
|
});
|
|
921
1017
|
}
|
|
922
1018
|
refreshFromDisk() {
|