actual-mcp-server 0.16.7 → 0.16.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/lib/ActualConnectionPool.js +97 -20
- package/dist/src/lib/actual-adapter.js +200 -9
- package/dist/src/lib/apiLock.js +198 -36
- package/dist/src/lib/apiState.js +14 -0
- package/dist/src/lib/budgetLoader.js +51 -3
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -775,7 +775,7 @@ The software is provided **as-is**, without warranty of any kind. The author acc
|
|
|
775
775
|
|
|
776
776
|
---
|
|
777
777
|
|
|
778
|
-
**Version:** 0.16.
|
|
778
|
+
**Version:** 0.16.9 | **Tool Count:** 74 (verified LibreChat-compatible)
|
|
779
779
|
|
|
780
780
|
## Sponsor
|
|
781
781
|
|
|
@@ -17,7 +17,7 @@ import path from 'path';
|
|
|
17
17
|
import os from 'os';
|
|
18
18
|
import fs from 'fs';
|
|
19
19
|
import { isApiInitialized, setApiInitialized } from './apiState.js';
|
|
20
|
-
import { withApiLock } from './apiLock.js';
|
|
20
|
+
import { withApiLock, isApiLockHeld } from './apiLock.js';
|
|
21
21
|
import { loadBudgetTracked } from './budgetLoader.js';
|
|
22
22
|
import { withOpTimeout } from './opTimeout.js';
|
|
23
23
|
const DEFAULT_DATA_DIR = path.resolve(os.homedir() || '.', '.actual');
|
|
@@ -417,7 +417,7 @@ class ActualConnectionPool {
|
|
|
417
417
|
await withApiLock(() => this.shutdownConnectionLocked(sessionId, opts)).catch(async (err) => {
|
|
418
418
|
logger.error(`[ConnectionPool] Could not acquire the api lock to shut down ${sessionId}; ` +
|
|
419
419
|
`tearing the entry down without it: ${err instanceof Error ? err.message : String(err)}`);
|
|
420
|
-
await this.shutdownConnectionLocked(sessionId, opts).catch(() => { });
|
|
420
|
+
await this.shutdownConnectionLocked(sessionId, { ...opts, expectUnlocked: true }).catch(() => { });
|
|
421
421
|
});
|
|
422
422
|
}
|
|
423
423
|
/**
|
|
@@ -434,9 +434,24 @@ class ActualConnectionPool {
|
|
|
434
434
|
* - actual-adapter switchBudget slow path (its own withApiLock block has closed by then)
|
|
435
435
|
* - actualConnection.shutdownActualForSession (session close)
|
|
436
436
|
* - this.cleanupIdleConnections (the idle sweep)
|
|
437
|
-
*
|
|
437
|
+
*
|
|
438
|
+
* Since #412, `shutdownAll` HOLDS the lock (one acquisition for the whole teardown instead of
|
|
439
|
+
* one per session) and therefore calls this variant directly. Its fallback path runs unlocked
|
|
440
|
+
* on purpose and passes `expectUnlocked` to say so.
|
|
438
441
|
*/
|
|
439
442
|
async shutdownConnectionLocked(sessionId, opts = {}) {
|
|
443
|
+
// #411: assert the precondition rather than documenting it. The audit above says which callers
|
|
444
|
+
// hold the lock; this is what keeps that audit true as call sites are added.
|
|
445
|
+
//
|
|
446
|
+
// A WARNING, not a throw, and deliberately so: this is reached from cleanup paths, and a false
|
|
447
|
+
// positive that threw would turn a tidy-up into an outage, which is a worse trade than the
|
|
448
|
+
// hazard it guards. The nearest legitimate exception is the fallback in shutdownConnection
|
|
449
|
+
// itself, which runs unlocked ON PURPOSE when the acquisition failed, so it passes
|
|
450
|
+
// `expectUnlocked` to say so.
|
|
451
|
+
if (!opts.expectUnlocked && !isApiLockHeld()) {
|
|
452
|
+
logger.warn(`[ConnectionPool] shutdownConnectionLocked called for ${sessionId} without the api mutex. ` +
|
|
453
|
+
'Callers that do not hold it must use shutdownConnection, which acquires it.');
|
|
454
|
+
}
|
|
440
455
|
const conn = this.connections.get(sessionId);
|
|
441
456
|
// #392 review: the idle sweep decides who is expired BEFORE waiting for the mutex, and that
|
|
442
457
|
// wait is no longer negligible. Re-checking here means a session that was touched while we
|
|
@@ -479,11 +494,16 @@ class ActualConnectionPool {
|
|
|
479
494
|
}
|
|
480
495
|
}
|
|
481
496
|
/**
|
|
482
|
-
*
|
|
497
|
+
* Acquire the mutex and tear down the shared connection.
|
|
498
|
+
*
|
|
499
|
+
* #412 left this with NO caller in `src/`: `shutdownAll` now holds the lock itself and calls the
|
|
500
|
+
* Locked variant. It is kept, rather than deleted, because `USE_CONNECTION_POOL=false` uses the
|
|
501
|
+
* shared connection and a future teardown path for it must not hand-roll the acquisition. That
|
|
502
|
+
* is a deliberate decision rather than an oversight: knip does not analyse class members, so
|
|
503
|
+
* nothing else would have flagged it, and CLAUDE.md's own lesson from `adapter.deleteRule` is
|
|
504
|
+
* that a callerless wrapper left reachable is how the next caller finds the unguarded path.
|
|
483
505
|
*/
|
|
484
506
|
async shutdownSharedConnection() {
|
|
485
|
-
// #392: same reasoning as shutdownConnection. Its only caller is shutdownAll, which does not
|
|
486
|
-
// hold the mutex.
|
|
487
507
|
await withApiLock(() => this.shutdownSharedConnectionLocked()).catch(async (err) => {
|
|
488
508
|
logger.error(`[ConnectionPool] Could not acquire the api lock to shut down the shared connection; ` +
|
|
489
509
|
`tearing it down without it: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -491,7 +511,14 @@ class ActualConnectionPool {
|
|
|
491
511
|
});
|
|
492
512
|
}
|
|
493
513
|
/** The body of shutdownSharedConnection, for callers that already hold the api mutex. */
|
|
494
|
-
async shutdownSharedConnectionLocked() {
|
|
514
|
+
async shutdownSharedConnectionLocked(opts = {}) {
|
|
515
|
+
// #411: same precondition check as shutdownConnectionLocked. Round 2 pointed out that keeping
|
|
516
|
+
// the callerless wrapper (M3) is only defensible if the guard that would catch a future
|
|
517
|
+
// hand-rolled caller exists on BOTH variants.
|
|
518
|
+
if (!opts.expectUnlocked && !isApiLockHeld()) {
|
|
519
|
+
logger.warn('[ConnectionPool] shutdownSharedConnectionLocked called without the api mutex. ' +
|
|
520
|
+
'Callers that do not hold it must use shutdownSharedConnection, which acquires it.');
|
|
521
|
+
}
|
|
495
522
|
if (!this.sharedConnection?.initialized) {
|
|
496
523
|
return;
|
|
497
524
|
}
|
|
@@ -519,12 +546,38 @@ class ActualConnectionPool {
|
|
|
519
546
|
async forceCloseStaleConnections() {
|
|
520
547
|
try {
|
|
521
548
|
logger.info('[ConnectionPool] Force closing any stale connections from previous instance');
|
|
522
|
-
// Try to shutdown the API if it was left initialized
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
549
|
+
// Try to shutdown the API if it was left initialized.
|
|
550
|
+
//
|
|
551
|
+
// #411 put this under the api mutex: it was the last api.shutdown() running without it.
|
|
552
|
+
// #416 then separated the two failures it was conflating. The body is a raw api.shutdown(),
|
|
553
|
+
// which can reject on its own, and reporting that as "could not take the api lock" sent the
|
|
554
|
+
// reader to the wrong subsystem while the retry repeated the call that had just failed.
|
|
555
|
+
//
|
|
556
|
+
// So: the guarded shutdown happens INSIDE the lock, matching every other shutdown path
|
|
557
|
+
// (which check isApiInitialized first, per #164, or a second call surfaces as
|
|
558
|
+
// "not initialized"). The catch below therefore genuinely means the ACQUISITION failed, and
|
|
559
|
+
// its fallback runs the same guarded shutdown once, unlocked and best effort.
|
|
560
|
+
const guardedShutdown = async () => {
|
|
561
|
+
if (!isApiInitialized())
|
|
562
|
+
return;
|
|
563
|
+
const maybeApi = api;
|
|
564
|
+
if (typeof maybeApi.shutdown !== 'function')
|
|
565
|
+
return;
|
|
566
|
+
try {
|
|
567
|
+
await maybeApi.shutdown();
|
|
568
|
+
logger.info('[ConnectionPool] Successfully closed stale API connection');
|
|
569
|
+
}
|
|
570
|
+
catch (shutdownErr) {
|
|
571
|
+
logger.debug(`[ConnectionPool] Stale API connection would not close (ignoring): ${shutdownErr}`);
|
|
572
|
+
}
|
|
573
|
+
finally {
|
|
574
|
+
setApiInitialized(false);
|
|
575
|
+
}
|
|
576
|
+
};
|
|
577
|
+
await withApiLock(guardedShutdown).catch(async (lockErr) => {
|
|
578
|
+
logger.debug(`[ConnectionPool] Could not take the api lock to close a stale connection: ${lockErr}`);
|
|
579
|
+
await guardedShutdown();
|
|
580
|
+
});
|
|
528
581
|
}
|
|
529
582
|
catch (err) {
|
|
530
583
|
// Ignore errors - connection may not have been initialized
|
|
@@ -596,13 +649,37 @@ class ActualConnectionPool {
|
|
|
596
649
|
// singleton ("not initialized" during graceful shutdown). Sessions are few
|
|
597
650
|
// (15 max) and shutdown is rare, so sequential is fine. The first call does
|
|
598
651
|
// the real shutdown; the isApiInitialized() guard makes the rest no-ops.
|
|
599
|
-
//
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
652
|
+
// #412: ONE acquisition for the whole teardown, not one per session.
|
|
653
|
+
//
|
|
654
|
+
// Since #392 each shutdownConnection takes the api mutex, and since #393 acquiring it settles
|
|
655
|
+
// any abandoned budget load, bounded, and throws on timeout while KEEPING the registration. So
|
|
656
|
+
// with a stuck load, N sessions cost N times ACTUAL_OP_TIMEOUT_MS: 15 sessions at the 30s
|
|
657
|
+
// default is 450 seconds, and Docker sends SIGKILL after a 10 second grace period. The
|
|
658
|
+
// container died with a page of "could not acquire" lines and no clean api.shutdown().
|
|
659
|
+
//
|
|
660
|
+
// Acquiring once also makes the teardown ONE critical section rather than 16, which is more
|
|
661
|
+
// correct: nothing can interleave between the sessions being torn down.
|
|
662
|
+
//
|
|
663
|
+
// Swallowed for the same reason the per-session wrapper swallows: this is a cleanup path, and
|
|
664
|
+
// a stuck load must not stop the entries being removed. The fallback repeats the loop with the
|
|
665
|
+
// Locked variants, which is safe because nothing can be holding the lock while a registration
|
|
666
|
+
// is stuck (every acquisition throws in that state).
|
|
667
|
+
const teardown = async (expectUnlocked = false) => {
|
|
668
|
+
// Snapshot the keys: shutdownConnectionLocked deletes from `connections`.
|
|
669
|
+
for (const sessionId of [...this.connections.keys()]) {
|
|
670
|
+
await this.shutdownConnectionLocked(sessionId, { expectUnlocked });
|
|
671
|
+
}
|
|
672
|
+
if (this.sharedConnection?.initialized) {
|
|
673
|
+
await this.shutdownSharedConnectionLocked({ expectUnlocked });
|
|
674
|
+
}
|
|
675
|
+
};
|
|
676
|
+
await withApiLock(() => teardown()).catch(async (err) => {
|
|
677
|
+
logger.error(`[ConnectionPool] Could not acquire the api lock for shutdownAll; tearing down without it: ` +
|
|
678
|
+
`${err instanceof Error ? err.message : String(err)}`);
|
|
679
|
+
// `expectUnlocked`: this path runs without the mutex ON PURPOSE, so #411's warning would
|
|
680
|
+
// otherwise fire once per session at the exact moment the code chose not to take it.
|
|
681
|
+
await teardown(true).catch(() => { });
|
|
682
|
+
});
|
|
606
683
|
logger.info('[ConnectionPool] All connections shut down');
|
|
607
684
|
}
|
|
608
685
|
/**
|
|
@@ -21,7 +21,7 @@ import { parseBudgetRegistry } from './budget-registry.js';
|
|
|
21
21
|
import { getPreferredBudgetSyncId, setPreferredBudgetSyncId, pickAllowedPreferredBudget } from './budget-preference-store.js';
|
|
22
22
|
import { requestContext } from './requestContext.js';
|
|
23
23
|
import { connectionPool } from './ActualConnectionPool.js';
|
|
24
|
-
import { isApiInitialized, setApiInitialized, getLoadedBudgetSyncId, awaitAbandonedBudgetLoad } from './apiState.js';
|
|
24
|
+
import { isApiInitialized, setApiInitialized, getLoadedBudgetSyncId, awaitAbandonedBudgetLoad, hasPendingBudgetLoad } from './apiState.js';
|
|
25
25
|
import { withApiLock } from './apiLock.js';
|
|
26
26
|
import { loadBudgetTracked, importBudgetTracked } from './budgetLoader.js';
|
|
27
27
|
/**
|
|
@@ -200,6 +200,44 @@ let connectionReuseCount = 0;
|
|
|
200
200
|
// update it without a circular import. The pool's hasConnection() returns
|
|
201
201
|
// true based on its own per-session record; this flag is the second guard
|
|
202
202
|
// — the singleton's actual state. Both must agree before reuse is safe.
|
|
203
|
+
/**
|
|
204
|
+
* #391: the budget this acquisition will operate on, as an AFFINITY HINT for the api lock.
|
|
205
|
+
*
|
|
206
|
+
* Never an assertion, and never used for a correctness decision: the loaded-budget PRECONDITION
|
|
207
|
+
* (`ensureLoadedBudgetMatchesSession`) is what makes an operation safe, and it runs inside the
|
|
208
|
+
* lock regardless of how the lock chose to order us. This only lets the lock group a run of
|
|
209
|
+
* same-budget work so the process pays one re-selection instead of one per call.
|
|
210
|
+
*
|
|
211
|
+
* Resolved from the same source the precondition uses, so a hint can never name a budget the
|
|
212
|
+
* operation would not have selected anyway.
|
|
213
|
+
*/
|
|
214
|
+
function _affinityBudget() {
|
|
215
|
+
try {
|
|
216
|
+
return getActiveBudgetConfig().syncId;
|
|
217
|
+
}
|
|
218
|
+
catch {
|
|
219
|
+
return undefined; // no registry, no hint: FIFO is always correct
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* #417: the budget every operation in a batch resolves to, or undefined when they differ.
|
|
224
|
+
*
|
|
225
|
+
* Exported so the unanimity rule can be tested without driving the lock. NOTE the side effect,
|
|
226
|
+
* which is bounded and idempotent but worth knowing: `getActiveBudgetConfig()` memoises a restored
|
|
227
|
+
* preference into `sessionBudgetState` and may read the preference file once per un-memoised
|
|
228
|
+
* session. The operation would do exactly that anyway a moment later, so this only moves it.
|
|
229
|
+
*/
|
|
230
|
+
export function resolveBatchBudget(entries) {
|
|
231
|
+
const budgets = new Set(entries.map((e) => {
|
|
232
|
+
try {
|
|
233
|
+
return requestContext.run(e.requestStore ?? {}, () => getActiveBudgetConfig().syncId);
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
return undefined;
|
|
237
|
+
}
|
|
238
|
+
}));
|
|
239
|
+
return budgets.size === 1 ? [...budgets][0] : undefined;
|
|
240
|
+
}
|
|
203
241
|
function _resolveSessionId() {
|
|
204
242
|
return requestContext.getStore()?.sessionId;
|
|
205
243
|
}
|
|
@@ -331,6 +369,8 @@ export async function withActualApi(rawOperation) {
|
|
|
331
369
|
const sessionId = _resolveSessionId();
|
|
332
370
|
if (_hasPooledConnection(sessionId)) {
|
|
333
371
|
// Pooled mode: skip init+shutdown.
|
|
372
|
+
// #391: name the budget this operation will run against, so the lock can group a run of
|
|
373
|
+
// same-budget work and pay ONE re-selection instead of one per call.
|
|
334
374
|
return withApiLock(async () => {
|
|
335
375
|
try {
|
|
336
376
|
connectionReuseCount++;
|
|
@@ -355,7 +395,7 @@ export async function withActualApi(rawOperation) {
|
|
|
355
395
|
}
|
|
356
396
|
throw err;
|
|
357
397
|
}
|
|
358
|
-
});
|
|
398
|
+
}, { budget: _affinityBudget() });
|
|
359
399
|
}
|
|
360
400
|
if (sessionId) {
|
|
361
401
|
logger.warn(`[ADAPTER] Pool miss for session ${sessionId}; falling back to per-op init`);
|
|
@@ -369,7 +409,7 @@ export async function withActualApi(rawOperation) {
|
|
|
369
409
|
finally {
|
|
370
410
|
await shutdownActualApi();
|
|
371
411
|
}
|
|
372
|
-
});
|
|
412
|
+
}, { budget: _affinityBudget() });
|
|
373
413
|
}
|
|
374
414
|
/**
|
|
375
415
|
* Variant of `withActualApi` for write operations. Identical to `withActualApi`
|
|
@@ -431,7 +471,7 @@ export async function withActualApiWrite(operation) {
|
|
|
431
471
|
}
|
|
432
472
|
throw err;
|
|
433
473
|
}
|
|
434
|
-
});
|
|
474
|
+
}, { budget: _affinityBudget() });
|
|
435
475
|
}
|
|
436
476
|
if (sessionId) {
|
|
437
477
|
logger.warn(`[ADAPTER] Pool miss for session ${sessionId}; falling back to per-op init (write)`);
|
|
@@ -444,7 +484,7 @@ export async function withActualApiWrite(operation) {
|
|
|
444
484
|
finally {
|
|
445
485
|
await shutdownActualApi();
|
|
446
486
|
}
|
|
447
|
-
});
|
|
487
|
+
}, { budget: _affinityBudget() });
|
|
448
488
|
}
|
|
449
489
|
/**
|
|
450
490
|
* Test-only: reset the connection-reuse counter. NOT exported via the package
|
|
@@ -623,6 +663,26 @@ import { BANK_SYNC_SETTLE_MS, WRITE_SESSION_DELAY_MS } from './constants.js';
|
|
|
623
663
|
// and setMaxConcurrency is re-exported.
|
|
624
664
|
import { withConcurrency, setMaxConcurrency, getConcurrencySnapshot } from './actual-adapter/concurrency.js';
|
|
625
665
|
export { setMaxConcurrency };
|
|
666
|
+
/**
|
|
667
|
+
* #389: how long an entry may sit DISPATCHED BUT NOT STARTED before it is rejected.
|
|
668
|
+
*
|
|
669
|
+
* Armed only when ACTUAL_OP_TIMEOUT_MS is 0, which is the one configuration where a stalled
|
|
670
|
+
* operation strands the rest of its batch in silence. Deliberately not derived from the op
|
|
671
|
+
* timeout: that is the knob the operator turned off. Fifteen minutes clears the longest
|
|
672
|
+
* legitimate single operation (a budget import at ACTUAL_IMPORT_TIMEOUT_MS, default ten minutes)
|
|
673
|
+
* with room to spare, so a healthy slow batch is never falsely rejected, while an operation that
|
|
674
|
+
* genuinely never runs gets an error instead of silence.
|
|
675
|
+
*/
|
|
676
|
+
let STRANDED_BATCH_LIMIT_MS = 900000;
|
|
677
|
+
/**
|
|
678
|
+
* Test-only: shrink the stranded bound so the arming behaviour is testable in reasonable time.
|
|
679
|
+
*
|
|
680
|
+
* Without a seam the riskiest timer in this file could only be exercised by waiting fifteen
|
|
681
|
+
* minutes, which means in practice it would never be exercised at all.
|
|
682
|
+
*/
|
|
683
|
+
export function _setStrandedBatchLimitForTests(ms) {
|
|
684
|
+
STRANDED_BATCH_LIMIT_MS = ms;
|
|
685
|
+
}
|
|
626
686
|
let writeQueue = [];
|
|
627
687
|
let isProcessingWrites = false;
|
|
628
688
|
let writeSessionTimeout = null;
|
|
@@ -767,6 +827,68 @@ async function processWriteQueue() {
|
|
|
767
827
|
entry.residencyTimer = undefined;
|
|
768
828
|
}
|
|
769
829
|
}
|
|
830
|
+
// #389: "dispatched" stopped meaning "started" when #378 made the batch SEQUENTIAL.
|
|
831
|
+
//
|
|
832
|
+
// Before that, a hanging operation left its siblings running. Now operations k+1..N never
|
|
833
|
+
// start, and because their residency timers were cleared just above (correctly, for the
|
|
834
|
+
// concurrent model) nothing rejects them: every one of those callers waits forever and NO
|
|
835
|
+
// ERROR IS EMITTED ANYWHERE. That is #278's signature exactly, and #278 took a "flaky" E2E
|
|
836
|
+
// test to surface because the only tell was the absence of an error.
|
|
837
|
+
//
|
|
838
|
+
// It needs ACTUAL_OP_TIMEOUT_MS=0 to be reachable, which is a documented escape hatch for
|
|
839
|
+
// debugging against a slow upstream: the one situation where a stall is most likely and an
|
|
840
|
+
// operator is least likely to suspect the write queue.
|
|
841
|
+
//
|
|
842
|
+
// So each entry keeps a bound on the part that is still a QUEUE WAIT. It is cleared the
|
|
843
|
+
// moment the operation actually starts, so it can never fire on an in-flight op, and it does
|
|
844
|
+
// NOT bound the operation itself, which is what the escape hatch exists to disable.
|
|
845
|
+
//
|
|
846
|
+
// The bound cannot be ACTUAL_OP_TIMEOUT_MS, because that is exactly what the operator disabled
|
|
847
|
+
// and what makes this reachable. It is a separate, generous constant armed ONLY in that
|
|
848
|
+
// configuration: setting it to 0 asks for unbounded OPERATIONS, not for unbounded QUEUEING, and
|
|
849
|
+
// those are different promises. It must also exceed the longest LEGITIMATE batch, or it would
|
|
850
|
+
// reject work that was about to run, which the ticket names as the way to get this wrong. The
|
|
851
|
+
// longest legitimate single operation is a budget import at ACTUAL_IMPORT_TIMEOUT_MS (default
|
|
852
|
+
// 600000), so 15 minutes clears it with room and is still finite.
|
|
853
|
+
//
|
|
854
|
+
// RE-ARMED on each start, not pre-armed once (review round 1). Arming all N timers at dispatch
|
|
855
|
+
// with the same delay makes entry k's allowance "900s minus everything before it", so a batch of
|
|
856
|
+
// healthy but slow operations rejects its own tail while nothing has stalled. Four budget
|
|
857
|
+
// re-selections at four minutes each on the slow link that made the operator disable the timeout
|
|
858
|
+
// in the first place is enough. Re-arming makes the bound mean what it should: no SINGLE
|
|
859
|
+
// predecessor has been running for 15 minutes.
|
|
860
|
+
const started = new Set();
|
|
861
|
+
const strandedLimitMs = config.ACTUAL_OP_TIMEOUT_MS === 0 ? STRANDED_BATCH_LIMIT_MS : 0;
|
|
862
|
+
const armStrandedBound = (entries) => {
|
|
863
|
+
if (strandedLimitMs <= 0)
|
|
864
|
+
return;
|
|
865
|
+
for (const entry of entries) {
|
|
866
|
+
if (started.has(entry) || entry.settledEarly)
|
|
867
|
+
continue;
|
|
868
|
+
if (entry.residencyTimer)
|
|
869
|
+
clearTimeout(entry.residencyTimer);
|
|
870
|
+
entry.residencyTimer = setTimeout(() => {
|
|
871
|
+
if (started.has(entry))
|
|
872
|
+
return;
|
|
873
|
+
logger.error(`[WRITE QUEUE] Operation never started: the operation ahead of it in this batch has not ` +
|
|
874
|
+
`returned within ${strandedLimitMs}ms. Rejecting rather than stranding the caller (#389).`);
|
|
875
|
+
entry.reject(new Error('This write never started: an earlier operation in the same batch stalled. ' +
|
|
876
|
+
'The batch runs sequentially (#378), so it was queued behind that operation. Retry it.'));
|
|
877
|
+
entry.settledEarly = true;
|
|
878
|
+
}, strandedLimitMs);
|
|
879
|
+
if (typeof entry.residencyTimer.unref === 'function')
|
|
880
|
+
entry.residencyTimer.unref();
|
|
881
|
+
}
|
|
882
|
+
};
|
|
883
|
+
const clearStrandedBounds = () => {
|
|
884
|
+
for (const entry of batch) {
|
|
885
|
+
if (entry.residencyTimer) {
|
|
886
|
+
clearTimeout(entry.residencyTimer);
|
|
887
|
+
entry.residencyTimer = undefined;
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
};
|
|
891
|
+
armStrandedBound(batch);
|
|
770
892
|
writeQueueBatchCount++;
|
|
771
893
|
// Pool-cooperation decision (#158): use the first queued op's captured
|
|
772
894
|
// sessionId as the batch's sessionId. In practice all ops batched together
|
|
@@ -786,6 +908,18 @@ async function processWriteQueue() {
|
|
|
786
908
|
// assignment to clear and therefore no path, fatal or otherwise, that can leak it: when
|
|
787
909
|
// the callback returns the store is gone. This replaced a module-level variable plus a
|
|
788
910
|
// `finally` that nulled it, which worked but rested on that finally being reached.
|
|
911
|
+
// #417: hint the batch's budget, but ONLY when every operation in it resolves to the same one.
|
|
912
|
+
//
|
|
913
|
+
// #391 made an unhinted waiter a BARRIER affinity may not cross, which is what stops a drain
|
|
914
|
+
// being overtaken by reads queued after it. The side effect is that an unhinted drain also
|
|
915
|
+
// blocks affinity for everything behind it: measured at 5 re-selections for an alternating load
|
|
916
|
+
// with a drain in every fourth slot, against 2 with no drains.
|
|
917
|
+
//
|
|
918
|
+
// Deliberately NOT the `batch[0]` heuristic, which CLAUDE.md already names as a hazard for the
|
|
919
|
+
// connection choice: one entry's budget says nothing about the rest, and a wrong hint would
|
|
920
|
+
// reorder a mixed batch. Unanimity or nothing. The hint is only ever an ordering preference;
|
|
921
|
+
// `ensureLoadedBudgetMatchesSession` inside the lock still decides what is safe.
|
|
922
|
+
const batchBudget = resolveBatchBudget(batch);
|
|
789
923
|
await drainListingStore.run(new Map(), async () => {
|
|
790
924
|
await withApiLock(async () => {
|
|
791
925
|
try {
|
|
@@ -830,7 +964,43 @@ async function processWriteQueue() {
|
|
|
830
964
|
//
|
|
831
965
|
// Each op keeps its own withOpTimeout (#270) and still settles individually, so one
|
|
832
966
|
// stalled or rejected operation cannot hold the api mutex or fail its siblings.
|
|
833
|
-
|
|
967
|
+
let stuckLoadError = null;
|
|
968
|
+
// Counted so the event is logged ONCE with a total. Without this the fail-fast path sits
|
|
969
|
+
// outside the per-operation try, so it never reaches the '[WRITE QUEUE] Operation failed'
|
|
970
|
+
// line and N operations fail with no log entry anywhere. Given this whole ticket family
|
|
971
|
+
// exists because absence-of-error is the failure signature, that would be the wrong silence
|
|
972
|
+
// to introduce while removing another.
|
|
973
|
+
let failedFast = 0;
|
|
974
|
+
for (const entry of batch) {
|
|
975
|
+
const { operation, resolve, reject, preservesListings, requestStore, timeoutMs } = entry;
|
|
976
|
+
// #389: this operation is STARTING, so its queue-wait bound no longer applies. Cleared
|
|
977
|
+
// before any await, so the timer can never fire on an in-flight op.
|
|
978
|
+
started.add(entry);
|
|
979
|
+
if (entry.residencyTimer) {
|
|
980
|
+
clearTimeout(entry.residencyTimer);
|
|
981
|
+
entry.residencyTimer = undefined;
|
|
982
|
+
}
|
|
983
|
+
// Restart the bound for everything still waiting, so it measures THIS operation's
|
|
984
|
+
// runtime rather than the batch's cumulative one.
|
|
985
|
+
armStrandedBound(batch);
|
|
986
|
+
if (entry.settledEarly)
|
|
987
|
+
continue; // its residency bound already rejected it
|
|
988
|
+
// #414: fail FAST while a budget load is genuinely stuck, rather than paying the
|
|
989
|
+
// abandoned-load bound once per remaining operation. A stuck load makes every one of
|
|
990
|
+
// them fail identically, so N operations would hold the process-global mutex for N
|
|
991
|
+
// times ACTUAL_OP_TIMEOUT_MS. The condition is RE-CHECKED rather than latched, because
|
|
992
|
+
// a load can land just after a bound expires and the whole point of keeping the
|
|
993
|
+
// registration is that a late landing is still waited for.
|
|
994
|
+
if (stuckLoadError !== null && hasPendingBudgetLoad()) {
|
|
995
|
+
// A DISTINCT error, not the one the earlier operation's wait produced (review round 1).
|
|
996
|
+
// Handing this operation that error claims it waited and timed out, which it never did,
|
|
997
|
+
// and its message matches TRANSIENT_ERROR_PATTERNS while describing something that is
|
|
998
|
+
// not this operation's failure. `cause` keeps the real origin attached.
|
|
999
|
+
failedFast++;
|
|
1000
|
+
reject(new Error('This write was queued behind an operation whose budget load is still outstanding, ' +
|
|
1001
|
+
'so it was failed immediately rather than waiting for the same bound. Retry it.', { cause: stuckLoadError }));
|
|
1002
|
+
continue;
|
|
1003
|
+
}
|
|
834
1004
|
try {
|
|
835
1005
|
// #390 round 2: run each op in the context it was ENQUEUED in.
|
|
836
1006
|
//
|
|
@@ -866,7 +1036,15 @@ async function processWriteQueue() {
|
|
|
866
1036
|
// A rejection here is caught by this loop's own per-op catch, so a stuck load fails
|
|
867
1037
|
// ONE operation rather than the whole batch, which is the behaviour #393 round 4
|
|
868
1038
|
// established for the drain.
|
|
869
|
-
|
|
1039
|
+
try {
|
|
1040
|
+
await awaitAbandonedBudgetLoad(withOpTimeout);
|
|
1041
|
+
}
|
|
1042
|
+
catch (waitErr) {
|
|
1043
|
+
// #414: remember it, so the rest of the batch fails fast instead of each paying
|
|
1044
|
+
// the same bound. Still rethrown, so THIS operation fails exactly as before.
|
|
1045
|
+
stuckLoadError = waitErr instanceof Error ? waitErr : new Error(String(waitErr));
|
|
1046
|
+
throw waitErr;
|
|
1047
|
+
}
|
|
870
1048
|
await ensureLoadedBudgetMatchesSession();
|
|
871
1049
|
// #407: an operation may carry its OWN bound. Only a budget import does today, and
|
|
872
1050
|
// it needs one: the import is legitimately long, and bounding it at the general
|
|
@@ -931,6 +1109,10 @@ async function processWriteQueue() {
|
|
|
931
1109
|
connectionPool.touch(sid);
|
|
932
1110
|
}
|
|
933
1111
|
}
|
|
1112
|
+
if (failedFast > 0) {
|
|
1113
|
+
logger.error(`[WRITE QUEUE] ${failedFast} operation(s) failed fast because a budget load is still ` +
|
|
1114
|
+
`outstanding. They were never attempted; the stalled load is the cause (#414).`);
|
|
1115
|
+
}
|
|
934
1116
|
// Explicitly sync changes to server before shutdown (legacy) or just
|
|
935
1117
|
// before returning (pool). Persistence guarantee in both branches.
|
|
936
1118
|
logger.debug(`[WRITE QUEUE] Syncing ${batch.length} operations to server`);
|
|
@@ -994,10 +1176,19 @@ async function processWriteQueue() {
|
|
|
994
1176
|
await shutdownActualApi();
|
|
995
1177
|
}
|
|
996
1178
|
}
|
|
997
|
-
});
|
|
1179
|
+
}, { budget: batchBudget });
|
|
998
1180
|
});
|
|
1181
|
+
clearStrandedBounds();
|
|
999
1182
|
}
|
|
1000
1183
|
catch (lockError) {
|
|
1184
|
+
// #389 review (I2): clear the stranded bounds here too. When the ACQUISITION rejects (which it
|
|
1185
|
+
// can, since #393, on an abandoned-load timeout) the loop is never entered, so without this all
|
|
1186
|
+
// N timers stay armed and each one logs "the operation ahead of it in this batch has not
|
|
1187
|
+
// returned" fifteen minutes later, for a batch that no longer exists and whose failure was a
|
|
1188
|
+
// lock acquisition rather than a stalled sibling. The rejects are harmless no-ops on settled
|
|
1189
|
+
// promises; the false signal is the problem, and it is the same wrong-subsystem misdirection
|
|
1190
|
+
// #416 exists to remove.
|
|
1191
|
+
clearStrandedBounds();
|
|
1001
1192
|
// #393 review: THE LOCK ITSELF CAN NOW REJECT, before `fn()` ever runs.
|
|
1002
1193
|
//
|
|
1003
1194
|
// The batch-rejection handler above lives INSIDE the lock callback, which was sound while
|
|
@@ -1398,7 +1589,7 @@ export async function setBudgetAmount(month, categoryId, amount) {
|
|
|
1398
1589
|
export async function transferBudgetAmount(month, fromCategoryId, toCategoryId, amount) {
|
|
1399
1590
|
observability.incrementToolCall('actual.budgets.transfer').catch(() => { });
|
|
1400
1591
|
return queueWriteOperation(async () => {
|
|
1401
|
-
// Inside processWriteQueue we already hold
|
|
1592
|
+
// Inside processWriteQueue we already hold the api mutex in apiLock.ts and the api is
|
|
1402
1593
|
// initialised. Call raw functions only: adapter wrappers would re-enter
|
|
1403
1594
|
// queueWriteOperation / withActualApi and defeat the single-cycle goal.
|
|
1404
1595
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
package/dist/src/lib/apiLock.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { awaitAbandonedBudgetLoad } from './apiState.js';
|
|
1
|
+
import { awaitAbandonedBudgetLoad, getLoadedBudgetSyncId } from './apiState.js';
|
|
2
2
|
import { withOpTimeout } from './opTimeout.js';
|
|
3
3
|
import { createModuleLogger } from './loggerFactory.js';
|
|
4
4
|
const logger = createModuleLogger('API LOCK');
|
|
@@ -17,44 +17,206 @@ const logger = createModuleLogger('API LOCK');
|
|
|
17
17
|
* this first: reads via `withActualApi`, writes via `processWriteQueue`, and since #390 the
|
|
18
18
|
* pool's session-open init and budget download.
|
|
19
19
|
*
|
|
20
|
-
* IT IS NOT REENTRANT, and that is load bearing.
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
20
|
+
* IT IS NOT REENTRANT, and that is load bearing. An inner acquisition queues behind the outer
|
|
21
|
+
* call's own release, which cannot happen until the outer callback resolves. The observable
|
|
22
|
+
* symptom is a stall ended by #270's operation timeout, NOT a hang, and it should be read as a
|
|
23
|
+
* probable nesting bug rather than a slow upstream. Before adding an acquisition, check that no
|
|
24
|
+
* caller above you already holds it.
|
|
25
25
|
*/
|
|
26
|
-
let _apiSessionLock = Promise.resolve();
|
|
27
26
|
/**
|
|
28
|
-
* #
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
27
|
+
* #391: BUDGET AFFINITY, replacing strict FIFO.
|
|
28
|
+
*
|
|
29
|
+
* The problem this solves, measured rather than asserted: with two sessions on different
|
|
30
|
+
* budgets, 20 alternating tool calls produced 19 full budget downloads, 19 syncs of the outgoing
|
|
31
|
+
* budget and 19 post-condition probes, all serialised through this mutex. That is #390's
|
|
32
|
+
* isolation working correctly and costing a re-selection per operation. On a large budget the
|
|
33
|
+
* download can exceed ACTUAL_OP_TIMEOUT_MS, at which point neither user completes anything while
|
|
34
|
+
* the other is active, and any two authenticated accounts on different budgets can hold the
|
|
35
|
+
* process there deliberately.
|
|
36
|
+
*
|
|
37
|
+
* The fix is to serialise by BUDGET rather than by operation: when the lock is released, prefer a
|
|
38
|
+
* waiter whose budget is already loaded. A run of same-budget operations then pays ONE
|
|
39
|
+
* re-selection instead of one each, so the cost tracks ALTERNATIONS rather than call count.
|
|
40
|
+
*
|
|
41
|
+
* THE MUTEX SHAPE CHANGED to make this possible. It was a chained-promise FIFO, which has no
|
|
42
|
+
* waiter list to reorder. It is now an explicit queue with a held flag. Three properties are
|
|
43
|
+
* load bearing, and each has a mutation-proven test in tests/unit/api_lock_affinity.test.js:
|
|
44
|
+
*
|
|
45
|
+
* 1. BOUNDED STARVATION. Preferring the loaded budget indefinitely starves the other one, which
|
|
46
|
+
* is a failure mode strict FIFO did not have. At most MAX_AFFINITY_SKIPS consecutive grants
|
|
47
|
+
* may skip the head of the queue; the next grant then goes to the head regardless of budget.
|
|
48
|
+
* 2. FIFO WITHIN A BUDGET. Among waiters on the same budget the oldest is always chosen.
|
|
49
|
+
* Several correctness arguments elsewhere rest on same-session ordering.
|
|
50
|
+
* 3. THE ABANDONED-LOAD WAIT STILL HAPPENS on every acquisition (#393), and per operation inside
|
|
51
|
+
* a drain (#406). Reordering acquisitions must not let one skip it.
|
|
52
|
+
*/
|
|
53
|
+
/**
|
|
54
|
+
* How many consecutive grants may skip the head of the queue.
|
|
55
|
+
*
|
|
56
|
+
* The trade is thrash against latency, and neither end is free. Zero is strict FIFO, which is
|
|
57
|
+
* what #391 exists to stop. Unbounded starves the other budget entirely.
|
|
58
|
+
*
|
|
59
|
+
* Eight, because a switch costs three upstream operations (sync the outgoing budget, download the
|
|
60
|
+
* incoming one, probe it), so amortising one switch over eight operations puts the per-operation
|
|
61
|
+
* cost well below the un-batched case, while the waiter at the head of the queue can be delayed by
|
|
62
|
+
* at most eight operations rather than indefinitely. It is deliberately a small number: the goal
|
|
63
|
+
* is to remove the per-CALL switch, not to let one tenant monopolise the process.
|
|
64
|
+
*
|
|
65
|
+
* The bound is in OPERATIONS, not wall clock, and the difference matters in the scenario this
|
|
66
|
+
* ticket cites. Eight operations that each run to ACTUAL_OP_TIMEOUT_MS is 240 seconds at the
|
|
67
|
+
* default, which is past most clients' own request timeouts. That is the accepted trade: bounding
|
|
68
|
+
* by time instead would need the lock to know how long an operation has left, which it cannot.
|
|
45
69
|
*/
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
70
|
+
const MAX_AFFINITY_SKIPS = 8;
|
|
71
|
+
let held = false;
|
|
72
|
+
const waiters = [];
|
|
73
|
+
let consecutiveSkips = 0;
|
|
74
|
+
/**
|
|
75
|
+
* The release path's diagnostic, behind a seam.
|
|
76
|
+
*
|
|
77
|
+
* Not indirection for its own sake: this exists so a test can install a THROWING hook and prove
|
|
78
|
+
* the lock is still released. Round 2 caught the previous test asserting source-text ordering
|
|
79
|
+
* instead, which stayed green when the exact regression was reintroduced a few lines away.
|
|
80
|
+
*/
|
|
81
|
+
let onSkipGrant = (info) => {
|
|
82
|
+
logger.debug('granting out of order to keep the loaded budget', info);
|
|
83
|
+
};
|
|
84
|
+
export function _setApiLockLogHookForTests(hook) {
|
|
85
|
+
onSkipGrant = hook ?? ((info) => {
|
|
86
|
+
logger.debug('granting out of order to keep the loaded budget', info);
|
|
51
87
|
});
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
88
|
+
}
|
|
89
|
+
/** Test hook: the queue is module state, so a suite must be able to observe and reset it. */
|
|
90
|
+
export function _getApiLockStateForTests() {
|
|
91
|
+
return { held, waiting: waiters.length, consecutiveSkips };
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* #411: is the api mutex currently held?
|
|
95
|
+
*
|
|
96
|
+
* Exists so the `...Locked` variants can ASSERT their precondition instead of documenting it. The
|
|
97
|
+
* audit of which callers hold the lock lives in a doc comment today, and this repo has paid four
|
|
98
|
+
* times (#371, #376, #390, #393) for the difference between a convention and a check.
|
|
99
|
+
*
|
|
100
|
+
* Deliberately coarse: it reports that SOMEONE holds the lock, not that YOU do, because a
|
|
101
|
+
* chained-promise mutex has no owner identity. That is still enough to catch the mistake that
|
|
102
|
+
* matters, which is calling a Locked variant from a path that holds nothing at all.
|
|
103
|
+
*/
|
|
104
|
+
export function isApiLockHeld() {
|
|
105
|
+
return held;
|
|
106
|
+
}
|
|
107
|
+
export function _resetApiLockForTests() {
|
|
108
|
+
// REJECT queued waiters rather than granting or dropping them.
|
|
109
|
+
//
|
|
110
|
+
// Dropping leaves promises that never settle (the hazard `_clearPendingBudgetLoadsForTests`
|
|
111
|
+
// documents in apiState.ts). Granting is worse and was the first attempt: a granted waiter RUNS
|
|
112
|
+
// ITS BODY and then calls grantNext(), releasing a lock it was never exclusively handed. Probed
|
|
113
|
+
// at four concurrent holders. Rejecting settles the promise without running anything and without
|
|
114
|
+
// a spurious release, which is what "reset" should mean.
|
|
115
|
+
const pending = waiters.splice(0, waiters.length);
|
|
116
|
+
held = false;
|
|
117
|
+
consecutiveSkips = 0;
|
|
118
|
+
for (const w of pending)
|
|
119
|
+
w.abort(new Error('api lock reset by a test hook'));
|
|
120
|
+
}
|
|
121
|
+
function acquire(budget) {
|
|
122
|
+
if (!held) {
|
|
123
|
+
held = true;
|
|
124
|
+
return Promise.resolve();
|
|
125
|
+
}
|
|
126
|
+
return new Promise((grant, abort) => {
|
|
127
|
+
waiters.push({ budget, grant, abort });
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Hand the lock to exactly one waiter, or release it if none are queued.
|
|
132
|
+
*
|
|
133
|
+
* Ownership TRANSFERS: `held` stays true when a waiter is granted, so no third party can slip in
|
|
134
|
+
* between the release and the grant. That is what makes this a mutex rather than a suggestion.
|
|
135
|
+
*/
|
|
136
|
+
function grantNext() {
|
|
137
|
+
if (waiters.length === 0) {
|
|
138
|
+
held = false;
|
|
139
|
+
consecutiveSkips = 0;
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
let index = 0; // strict FIFO by default, which is also the anti-starvation fallback
|
|
143
|
+
if (consecutiveSkips < MAX_AFFINITY_SKIPS) {
|
|
144
|
+
const loaded = getLoadedBudgetSyncId();
|
|
145
|
+
if (loaded !== null) {
|
|
146
|
+
// The OLDEST waiter on the loaded budget, so ordering within a budget is preserved.
|
|
147
|
+
const match = waiters.findIndex((w) => w.budget === loaded);
|
|
148
|
+
// An UNHINTED waiter is a BARRIER that affinity may not cross.
|
|
149
|
+
//
|
|
150
|
+
// Review caught this, and it was the worst thing in the change. An unhinted waiter can never
|
|
151
|
+
// equal `loaded`, so without this it was freely skipped, and the three unhinted acquisitions
|
|
152
|
+
// are the ones least able to afford it: the pool's session open, `shutdownAll`, and a write
|
|
153
|
+
// drain whose batch spans budgets. (#417 later let a UNANIMOUS drain hint its own budget, so
|
|
154
|
+
// that case is no longer a barrier. A mixed batch still is, and must stay so: one entry's
|
|
155
|
+
// budget says nothing about the rest.) Reproduced on a SINGLE-budget setup, where affinity can never save a
|
|
156
|
+
// re-selection because only one budget is ever loaded: a drain enqueued first ran ninth,
|
|
157
|
+
// behind eight reads. So the majority deployment got the reordering with none of the benefit,
|
|
158
|
+
// a write could be overtaken by reads issued after it (impossible under the old FIFO, and
|
|
159
|
+
// visible to a client issuing parallel tool calls), and #412's teardown could be delayed by
|
|
160
|
+
// eight operations at exactly the moment it is racing SIGTERM.
|
|
161
|
+
//
|
|
162
|
+
// Skipping a same-budget waiter that is merely LATER in the queue is safe; skipping one that
|
|
163
|
+
// cannot express a preference is not, because "no hint" does not mean "no ordering
|
|
164
|
+
// requirement".
|
|
165
|
+
const barrier = waiters.findIndex((w) => w.budget === undefined);
|
|
166
|
+
if (match > 0 && (barrier === -1 || match < barrier))
|
|
167
|
+
index = match;
|
|
56
168
|
}
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
169
|
+
}
|
|
170
|
+
if (index > 0) {
|
|
171
|
+
consecutiveSkips++;
|
|
172
|
+
}
|
|
173
|
+
else {
|
|
174
|
+
// The head ran, so nobody is being starved right now.
|
|
175
|
+
consecutiveSkips = 0;
|
|
176
|
+
}
|
|
177
|
+
// GRANT FIRST, log after, and never let the log break the release.
|
|
178
|
+
//
|
|
179
|
+
// This runs in a `finally`, so anything that throws here leaves `held` true with a queue nobody
|
|
180
|
+
// will ever be handed. No timeout can rescue that: the waiters never enter the lock body, so
|
|
181
|
+
// `withOpTimeout` never applies to them. It is #278's signature, and CLAUDE.md's own line is
|
|
182
|
+
// that a hang has no timeout large enough. The previous chained-promise mutex released with a
|
|
183
|
+
// bare `resolve()`, which could not throw; this queue can, so the ordering is deliberate.
|
|
184
|
+
const skipped = index;
|
|
185
|
+
const [chosen] = waiters.splice(index, 1);
|
|
186
|
+
chosen.grant();
|
|
187
|
+
if (skipped > 0) {
|
|
188
|
+
try {
|
|
189
|
+
onSkipGrant({ skipped, consecutiveSkips, waiting: waiters.length });
|
|
190
|
+
}
|
|
191
|
+
catch { /* the release path must never throw */ }
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Run `fn` holding the api mutex.
|
|
196
|
+
*
|
|
197
|
+
* `budget` is an optional AFFINITY HINT, not an assertion: pass the syncId this acquisition will
|
|
198
|
+
* operate on when the caller can name one unambiguously. Callers that cannot (a write drain whose
|
|
199
|
+
* batch spans budgets) pass nothing and are treated as ordinary FIFO waiters, which is always
|
|
200
|
+
* correct and merely forgoes the optimisation.
|
|
201
|
+
*/
|
|
202
|
+
export function withApiLock(fn, opts) {
|
|
203
|
+
return acquire(opts?.budget).then(async () => {
|
|
204
|
+
try {
|
|
205
|
+
// #393: settling an abandoned budget load is part of ACQUIRING the lock, not something call
|
|
206
|
+
// sites remember to do. Every previous round of #390 guarded per call site and every round
|
|
207
|
+
// missed one. Putting it here makes the set of call sites stop mattering: nothing reaches
|
|
208
|
+
// @actual-app/api without this lock, so nothing reaches it without the wait.
|
|
209
|
+
//
|
|
210
|
+
// It is bounded and fails closed (see `awaitAbandonedBudgetLoad`): an unbounded wait here
|
|
211
|
+
// would wedge every session on one stuck download, which is what made #390's round-2
|
|
212
|
+
// version worse than the bug it fixed.
|
|
213
|
+
if (await awaitAbandonedBudgetLoad(withOpTimeout)) {
|
|
214
|
+
logger.warn('waited for an abandoned budget load before running this operation');
|
|
215
|
+
}
|
|
216
|
+
return await fn();
|
|
217
|
+
}
|
|
218
|
+
finally {
|
|
219
|
+
grantNext();
|
|
220
|
+
}
|
|
221
|
+
});
|
|
60
222
|
}
|
package/dist/src/lib/apiState.js
CHANGED
|
@@ -127,6 +127,20 @@ export async function awaitAbandonedBudgetLoad(bound) {
|
|
|
127
127
|
await bound(() => outstanding, 'abandoned budget load');
|
|
128
128
|
return true;
|
|
129
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* Is a budget load currently outstanding?
|
|
132
|
+
*
|
|
133
|
+
* Production predicate (#414). The write drain uses it to FAIL FAST rather than pay the abandoned
|
|
134
|
+
* load bound once per remaining operation: a stuck load makes every one of them fail identically,
|
|
135
|
+
* so N operations cost N times ACTUAL_OP_TIMEOUT_MS holding the process-global mutex.
|
|
136
|
+
*
|
|
137
|
+
* Deliberately re-checked per operation rather than latched: a load can land just after a bound
|
|
138
|
+
* expires, and the whole point of keeping the registration is that a late landing is still waited
|
|
139
|
+
* for. Fail fast only while the condition still holds.
|
|
140
|
+
*/
|
|
141
|
+
export function hasPendingBudgetLoad() {
|
|
142
|
+
return _pendingBudgetLoads.size > 0;
|
|
143
|
+
}
|
|
130
144
|
/** Test hook: is a load currently outstanding? */
|
|
131
145
|
export function _hasPendingBudgetLoadForTests() {
|
|
132
146
|
return _pendingBudgetLoads.size > 0;
|
|
@@ -328,10 +328,49 @@ async function trackBudgetMutation(start, describeLoaded, label, verify, onRecor
|
|
|
328
328
|
// "first" untrue.
|
|
329
329
|
setLoadedBudgetSyncId(null);
|
|
330
330
|
const raw = start();
|
|
331
|
+
// #403: was this load ABANDONED by the caller's bound?
|
|
332
|
+
//
|
|
333
|
+
// The post-condition probe lives inside this chain so an abandoned load is verified when it
|
|
334
|
+
// lands rather than recording a success nobody checked (#396). The cost, raised in review, is
|
|
335
|
+
// that a LATE landing runs that probe with no lock held, against the single SQLite connection,
|
|
336
|
+
// while another session may be mid-operation inside the lock.
|
|
337
|
+
//
|
|
338
|
+
// The resolution is not to probe on that path at all. A landing whose caller has already given
|
|
339
|
+
// up leaves the record INDETERMINATE, which forces the next operation to re-select. That is the
|
|
340
|
+
// safe direction #390 established, it costs one extra re-selection after an event that is rare
|
|
341
|
+
// by construction (a load that outran its bound), and it removes the unsynchronised read
|
|
342
|
+
// entirely rather than arguing about whether it is observable.
|
|
343
|
+
let abandoned = false;
|
|
331
344
|
const tracked = raw
|
|
332
345
|
.then(async (result) => {
|
|
346
|
+
if (abandoned) {
|
|
347
|
+
log.warn('an abandoned budget load landed after its caller gave up; leaving the record indeterminate', {
|
|
348
|
+
label,
|
|
349
|
+
});
|
|
350
|
+
// The load itself SUCCEEDED upstream, so any cached failure reason for this budget is
|
|
351
|
+
// stale even though we are not recording the syncId. Clearing here keeps the hook's stated
|
|
352
|
+
// contract true on the abandoned path as well (see the note on `onRecorded` below).
|
|
353
|
+
try {
|
|
354
|
+
onRecorded?.();
|
|
355
|
+
}
|
|
356
|
+
catch { /* a hook cannot change the outcome */ }
|
|
357
|
+
return result;
|
|
358
|
+
}
|
|
333
359
|
if (verify)
|
|
334
360
|
await verify();
|
|
361
|
+
// #403 review: re-check. `abandoned` can flip DURING `verify()`, when the load resolves just
|
|
362
|
+
// before its bound expires. Without this the probe runs and the record is written on a load
|
|
363
|
+
// whose caller has gone, which contradicts the contract stated above. The value written
|
|
364
|
+
// would be true rather than false, so this is about honesty of the invariant rather than
|
|
365
|
+
// safety, but a contract the code only honours on the common path is not a contract.
|
|
366
|
+
if (abandoned) {
|
|
367
|
+
log.warn('a budget load was abandoned while its post-condition ran; leaving the record indeterminate', { label });
|
|
368
|
+
try {
|
|
369
|
+
onRecorded?.();
|
|
370
|
+
}
|
|
371
|
+
catch { /* a hook cannot change the outcome */ }
|
|
372
|
+
return result;
|
|
373
|
+
}
|
|
335
374
|
setLoadedBudgetSyncId(describeLoaded(result));
|
|
336
375
|
// #392 review (finding 5): this runs INSIDE the chain, not after the awaited call. On the
|
|
337
376
|
// abandoned-then-eventually-successful path the caller has already thrown, so a hook placed
|
|
@@ -357,9 +396,18 @@ async function trackBudgetMutation(start, describeLoaded, label, verify, onRecor
|
|
|
357
396
|
throw err;
|
|
358
397
|
});
|
|
359
398
|
registerBudgetLoad(tracked);
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
399
|
+
try {
|
|
400
|
+
// On timeout the tracked promise is still running. Leave it REGISTERED: the next operation
|
|
401
|
+
// waits for it inside the lock rather than racing it.
|
|
402
|
+
return await withOpTimeout(() => tracked, label, timeoutMs);
|
|
403
|
+
}
|
|
404
|
+
catch (err) {
|
|
405
|
+
// #403: from here the caller has given up, so anything still in flight must not touch the api
|
|
406
|
+
// with no lock held. The chain checks this before probing. Set on ANY rejection, which is
|
|
407
|
+
// harmless when the chain itself already failed: it has settled, so nothing reads the flag.
|
|
408
|
+
abandoned = true;
|
|
409
|
+
throw err;
|
|
410
|
+
}
|
|
363
411
|
}
|
|
364
412
|
/**
|
|
365
413
|
* #394: the same discipline for an IMPORT.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "actual-mcp-server",
|
|
3
3
|
"displayName": "Actual MCP Server",
|
|
4
|
-
"version": "0.16.
|
|
4
|
+
"version": "0.16.9",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22.0.0",
|
|
7
7
|
"npm": ">=10.0.0"
|
|
@@ -30,7 +30,7 @@
|
|
|
30
30
|
"verify-tools": "npm run build && node scripts/verify-tools.js",
|
|
31
31
|
"check:coverage": "node scripts/list-actual-api-methods.mjs",
|
|
32
32
|
"direct-sync": "node scripts/direct-sync/bank-sync-direct.mjs",
|
|
33
|
-
"test:unit-js": "node tests/unit/transactions_create.test.js && node tests/unit/generated_tools.smoke.test.js && node tests/unit/schema_validation.test.js && node tests/unit/schema_json_openai_compat.test.js && node tests/unit/config_https_validation.test.js && node tests/unit/config_data_dir.test.js && node tests/unit/config_drift.test.js && node tests/unit/port_alignment.test.js && node tests/unit/dockerfile_data_dir_alignment.test.js && node tests/unit/unraid_template_alignment.test.js && node tests/unit/entrypoint_invariants.test.js && node tests/unit/e2e_spec_collection.test.js && node tests/unit/compose_profile_sync.test.js && node tests/unit/advertised_tools_sync.test.js && node tests/unit/knip_config.test.js && node tests/unit/readme_stats_sync.test.js && node tests/unit/config_insecure_upstream.test.js && node tests/unit/workflow_release_guards.test.js && node tests/unit/report_train_failure.test.js && node tests/unit/unit_chain_membership.test.js && node tests/unit/api_surface_drift.test.js && node tests/unit/write_effect_audit_staleness.test.js && node tests/unit/write_effect_audit_membership.test.js && node tests/unit/tool_annotations.test.js && node tests/unit/tool_id_schema_drift.test.js && node tests/unit/api_coverage_baseline.test.js && node tests/unit/train_preflight.test.js && node tests/unit/verify_published_artifacts.test.js && node tests/unit/report_train_stale.test.js && node tests/unit/stdio_framing_check.test.js && node tests/unit/server_info.test.js && node tests/unit/transfers_create.test.js && node tests/unit/bot_target_branch.test.js && node tests/unit/auth_posture.test.js && node tests/unit/auth_posture_wiring.test.js && node tests/unit/auth-acl.test.js && node tests/unit/bug76.test.js && node tests/unit/budgets_setAmount.test.js && node tests/unit/budgets_hold_for_next_month.test.js && node tests/unit/budgets_transfer.test.js && node tests/unit/transactions_uncategorized.test.js && node tests/unit/httpServer_session_init.test.js && node tests/unit/httpServer_session_not_found.test.js && node tests/unit/manual_mcp_client_retry.test.js && node tests/unit/manual_mcp_client_session.test.js && node tests/unit/manual_mcp_client_circuit.test.js && node tests/unit/manual_runner_killswitch.test.js && node tests/unit/adapter_auth_rate_limit.test.js && node tests/unit/adapter_session_reuse.test.js && node tests/unit/retry_classifier.test.js && node tests/unit/adapter_module_surface.test.js && node tests/unit/adapter_nonidempotent_no_retry.test.js && node tests/unit/pool_liveness.test.js && node tests/unit/pool_shutdown_all.test.js && node tests/unit/query_where_operators.test.js && node tests/unit/query_run_validation.test.js && node tests/unit/adapter_with_write_session.test.js && node tests/unit/category_groups_delete.test.js && node tests/unit/rules_delete.test.js && node tests/unit/schedules_delete.test.js && node tests/unit/accounts_delete.test.js && node tests/unit/update_tools_not_found.test.js && node tests/unit/accounts_close.test.js && node tests/unit/accounts_reopen.test.js && node tests/unit/payees_delete.test.js && node tests/unit/payees_merge.test.js && node tests/unit/rules_create_or_update.test.js && node tests/unit/unhandled-rejection.test.js && node tests/unit/rejection-allowlist-purity.test.js && node tests/unit/httpServer_bearer_auth.test.js && node tests/unit/httpServer_oidc_audience.test.js && node tests/unit/httpServer_oidc_auth_verification.test.js && node tests/unit/httpServer_oidc_jwks_discovery.test.js && node tests/unit/httpServer_body_limit.test.js && node tests/unit/httpServer_bind_host.test.js && node tests/unit/adapter_write_pool_cooperation.test.js && node tests/unit/adapter_drain_listing_cache.test.js && node tests/unit/budget_acl_enforcement.test.js && node tests/unit/budget_selection_precondition.test.js && node tests/unit/budget_load_abandonment.test.js && node tests/unit/budget_loader_postcondition.test.js && node tests/unit/budget_loader_import_allowlist.test.js && node tests/unit/budget_import_tracking.test.js && node tests/unit/pool_shutdown_under_lock.test.js && node tests/unit/budget_preference_store.test.js && node tests/unit/budget_preference_restore.test.js && node tests/unit/tags_list.test.js && node tests/unit/tags_create.test.js && node tests/unit/tags_update.test.js && node tests/unit/tags_delete.test.js && node tests/unit/notes_get.test.js && node tests/unit/notes_update.test.js && node tests/unit/payees_common_list.test.js && node tests/unit/transactions_update_guard.test.js && node tests/unit/transactions_update_splits.test.js && node tests/unit/entities_search.test.js && node tests/unit/transactions_import_schema.test.js && node tests/unit/tool_schema_shape.test.js && node tests/unit/zod_error_format.test.js && node tests/unit/tool_count_sync.test.js && node tests/unit/stdio_transport.test.js && node tests/unit/stdio_budget_switch.test.js && node tests/unit/logger_structured.test.js && node tests/unit/logger_redaction.test.js && node tests/unit/logger_correlation.test.js && node tests/unit/check_coverage.test.js && node tests/unit/adapter_op_timeout.test.js && node tests/unit/adapter_write_queue_wakeup.test.js && node tests/unit/ci_workflow_guards.test.js && node tests/unit/node_version_guard.test.js && node tests/unit/node_version_drift.test.js && node tests/unit/observability_smoke.test.js && node tests/unit/residue_pattern.test.js && node tests/unit/dual_transport_gate.test.js && node tests/unit/manual_assertions_use_helper.test.js && node tests/unit/server_version_guard.test.js && node tests/unit/payee_rules_reference_filter.test.js && node tests/unit/oidc_metadata_discovery.test.js && node tests/unit/httpServer_oauth_metadata_route.test.js && node tests/unit/httpServer_mcp_info_route.test.js && node tests/unit/residue_sweep_balance_account.test.js && node tests/unit/tool_factory.test.js && node tests/unit/budget_registry_parse.test.js && node tests/unit/preferences_get.test.js && node tests/unit/budgets_export.test.js && node tests/unit/budgets_import.test.js && node tests/unit/budgets_import_syncid.test.js && node tests/unit/budget_acl_dynamic.test.js && node tests/unit/budget_acl_identity_map.test.js && node tests/unit/budget_acl_userinfo.test.js && node tests/unit/acl_e2e_fixture_contract.test.js && node tests/unit/budget_acl_middleware.test.js && node tests/unit/rules_stage.test.js",
|
|
33
|
+
"test:unit-js": "node tests/unit/transactions_create.test.js && node tests/unit/generated_tools.smoke.test.js && node tests/unit/schema_validation.test.js && node tests/unit/schema_json_openai_compat.test.js && node tests/unit/config_https_validation.test.js && node tests/unit/config_data_dir.test.js && node tests/unit/config_drift.test.js && node tests/unit/port_alignment.test.js && node tests/unit/dockerfile_data_dir_alignment.test.js && node tests/unit/unraid_template_alignment.test.js && node tests/unit/entrypoint_invariants.test.js && node tests/unit/e2e_spec_collection.test.js && node tests/unit/compose_profile_sync.test.js && node tests/unit/advertised_tools_sync.test.js && node tests/unit/knip_config.test.js && node tests/unit/readme_stats_sync.test.js && node tests/unit/config_insecure_upstream.test.js && node tests/unit/workflow_release_guards.test.js && node tests/unit/report_train_failure.test.js && node tests/unit/unit_chain_membership.test.js && node tests/unit/api_surface_drift.test.js && node tests/unit/write_effect_audit_staleness.test.js && node tests/unit/write_effect_audit_membership.test.js && node tests/unit/tool_annotations.test.js && node tests/unit/tool_id_schema_drift.test.js && node tests/unit/api_coverage_baseline.test.js && node tests/unit/train_preflight.test.js && node tests/unit/verify_published_artifacts.test.js && node tests/unit/report_train_stale.test.js && node tests/unit/stdio_framing_check.test.js && node tests/unit/server_info.test.js && node tests/unit/transfers_create.test.js && node tests/unit/bot_target_branch.test.js && node tests/unit/auth_posture.test.js && node tests/unit/auth_posture_wiring.test.js && node tests/unit/auth-acl.test.js && node tests/unit/bug76.test.js && node tests/unit/budgets_setAmount.test.js && node tests/unit/budgets_hold_for_next_month.test.js && node tests/unit/budgets_transfer.test.js && node tests/unit/transactions_uncategorized.test.js && node tests/unit/httpServer_session_init.test.js && node tests/unit/httpServer_session_not_found.test.js && node tests/unit/manual_mcp_client_retry.test.js && node tests/unit/manual_mcp_client_session.test.js && node tests/unit/manual_mcp_client_circuit.test.js && node tests/unit/manual_runner_killswitch.test.js && node tests/unit/adapter_auth_rate_limit.test.js && node tests/unit/adapter_session_reuse.test.js && node tests/unit/retry_classifier.test.js && node tests/unit/adapter_module_surface.test.js && node tests/unit/adapter_nonidempotent_no_retry.test.js && node tests/unit/pool_liveness.test.js && node tests/unit/pool_shutdown_all.test.js && node tests/unit/query_where_operators.test.js && node tests/unit/query_run_validation.test.js && node tests/unit/adapter_with_write_session.test.js && node tests/unit/category_groups_delete.test.js && node tests/unit/rules_delete.test.js && node tests/unit/schedules_delete.test.js && node tests/unit/accounts_delete.test.js && node tests/unit/update_tools_not_found.test.js && node tests/unit/accounts_close.test.js && node tests/unit/accounts_reopen.test.js && node tests/unit/payees_delete.test.js && node tests/unit/payees_merge.test.js && node tests/unit/rules_create_or_update.test.js && node tests/unit/unhandled-rejection.test.js && node tests/unit/rejection-allowlist-purity.test.js && node tests/unit/httpServer_bearer_auth.test.js && node tests/unit/httpServer_oidc_audience.test.js && node tests/unit/httpServer_oidc_auth_verification.test.js && node tests/unit/httpServer_oidc_jwks_discovery.test.js && node tests/unit/httpServer_body_limit.test.js && node tests/unit/httpServer_bind_host.test.js && node tests/unit/adapter_write_pool_cooperation.test.js && node tests/unit/adapter_drain_listing_cache.test.js && node tests/unit/budget_acl_enforcement.test.js && node tests/unit/budget_selection_precondition.test.js && node tests/unit/budget_load_abandonment.test.js && node tests/unit/budget_loader_postcondition.test.js && node tests/unit/budget_loader_import_allowlist.test.js && node tests/unit/budget_import_tracking.test.js && node tests/unit/pool_shutdown_under_lock.test.js && node tests/unit/api_lock_affinity.test.js && node tests/unit/commit_message_closing_keyword.test.js && node tests/unit/write_queue_stranding.test.js && node tests/unit/write_queue_stranded_bound.test.js && node tests/unit/budget_preference_store.test.js && node tests/unit/budget_preference_restore.test.js && node tests/unit/tags_list.test.js && node tests/unit/tags_create.test.js && node tests/unit/tags_update.test.js && node tests/unit/tags_delete.test.js && node tests/unit/notes_get.test.js && node tests/unit/notes_update.test.js && node tests/unit/payees_common_list.test.js && node tests/unit/transactions_update_guard.test.js && node tests/unit/transactions_update_splits.test.js && node tests/unit/entities_search.test.js && node tests/unit/transactions_import_schema.test.js && node tests/unit/tool_schema_shape.test.js && node tests/unit/zod_error_format.test.js && node tests/unit/tool_count_sync.test.js && node tests/unit/stdio_transport.test.js && node tests/unit/stdio_budget_switch.test.js && node tests/unit/logger_structured.test.js && node tests/unit/logger_redaction.test.js && node tests/unit/logger_correlation.test.js && node tests/unit/check_coverage.test.js && node tests/unit/adapter_op_timeout.test.js && node tests/unit/adapter_write_queue_wakeup.test.js && node tests/unit/ci_workflow_guards.test.js && node tests/unit/node_version_guard.test.js && node tests/unit/node_version_drift.test.js && node tests/unit/observability_smoke.test.js && node tests/unit/residue_pattern.test.js && node tests/unit/dual_transport_gate.test.js && node tests/unit/manual_assertions_use_helper.test.js && node tests/unit/server_version_guard.test.js && node tests/unit/payee_rules_reference_filter.test.js && node tests/unit/oidc_metadata_discovery.test.js && node tests/unit/httpServer_oauth_metadata_route.test.js && node tests/unit/httpServer_mcp_info_route.test.js && node tests/unit/residue_sweep_balance_account.test.js && node tests/unit/tool_factory.test.js && node tests/unit/budget_registry_parse.test.js && node tests/unit/preferences_get.test.js && node tests/unit/budgets_export.test.js && node tests/unit/budgets_import.test.js && node tests/unit/budgets_import_syncid.test.js && node tests/unit/budget_acl_dynamic.test.js && node tests/unit/budget_acl_identity_map.test.js && node tests/unit/budget_acl_userinfo.test.js && node tests/unit/acl_e2e_fixture_contract.test.js && node tests/unit/budget_acl_middleware.test.js && node tests/unit/rules_stage.test.js",
|
|
34
34
|
"test:adapter": "npm run build && node dist/src/tests_adapter_runner.js",
|
|
35
35
|
"test:e2e": "npx playwright test",
|
|
36
36
|
"test:e2e:docker": "./tests/e2e/run-docker-e2e.sh",
|