dexbot 1.7.1 → 1.7.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/claw/package.json +1 -1
- package/claw/runtimes/openclaw-plugin/openclaw.plugin.json +1 -1
- package/claw/runtimes/openclaw-plugin/package.json +1 -1
- package/dist/analysis/grid_correction_check.d.ts +60 -1
- package/dist/analysis/grid_correction_check.js +268 -18
- package/dist/analysis/pivot_timeline.d.ts +78 -0
- package/dist/analysis/pivot_timeline.js +360 -0
- package/dist/market_adapter/utils/file_lock.js +16 -4
- package/dist/modules/account_bots.d.ts +7 -0
- package/dist/modules/account_bots.js +7 -5
- package/dist/modules/bot_defaults.js +6 -1
- package/dist/modules/constants.d.ts +14 -0
- package/dist/modules/constants.js +82 -2
- package/dist/modules/dexbot_class.d.ts +39 -0
- package/dist/modules/dexbot_class.js +86 -1
- package/dist/modules/dexbot_cow_runtime.d.ts +31 -1
- package/dist/modules/dexbot_cow_runtime.js +299 -75
- package/dist/modules/dexbot_maintenance_runtime.d.ts +32 -2
- package/dist/modules/dexbot_maintenance_runtime.js +196 -16
- package/dist/modules/dexbot_state_recovery.js +15 -1
- package/dist/modules/launcher/instance_lock.d.ts +27 -0
- package/dist/modules/launcher/instance_lock.js +131 -0
- package/dist/modules/node_flap_score.d.ts +6 -6
- package/dist/modules/node_flap_score.js +6 -6
- package/dist/modules/order/accounting.d.ts +9 -4
- package/dist/modules/order/accounting.js +89 -26
- package/dist/modules/order/async_lock.d.ts +7 -1
- package/dist/modules/order/async_lock.js +7 -1
- package/dist/modules/order/export.js +25 -18
- package/dist/modules/order/fill_ledger.d.ts +99 -0
- package/dist/modules/order/fill_ledger.js +103 -0
- package/dist/modules/order/grid.d.ts +94 -18
- package/dist/modules/order/grid.js +247 -75
- package/dist/modules/order/grid_reconcile_internal.js +16 -4
- package/dist/modules/order/index.d.ts +29 -6
- package/dist/modules/order/index.js +5 -5
- package/dist/modules/order/lifecycle.js +3 -3
- package/dist/modules/order/manager.d.ts +36 -24
- package/dist/modules/order/manager.js +159 -39
- package/dist/modules/order/placement_lock.d.ts +30 -0
- package/dist/modules/order/placement_lock.js +48 -0
- package/dist/modules/order/sync_engine.d.ts +66 -118
- package/dist/modules/order/sync_engine.js +118 -162
- package/dist/modules/order/utils/math.d.ts +7 -3
- package/dist/modules/order/utils/math.js +7 -3
- package/dist/modules/order/utils/order.d.ts +79 -42
- package/dist/modules/order/utils/order.js +183 -86
- package/dist/modules/order/utils/system.d.ts +192 -8
- package/dist/modules/order/utils/system.js +534 -34
- package/dist/modules/paths.d.ts +1 -0
- package/dist/modules/paths.js +1 -0
- package/dist/modules/timing_policy.js +4 -2
- package/dist/modules/types.d.ts +49 -27
- package/dist/pm2.js +126 -0
- package/dist/unlock.js +26 -1
- package/docs/COW_INVARIANTS.md +5 -1
- package/docs/DEXBOT_COMPARISON.md +5 -5
- package/docs/EVOLUTION.md +10 -8
- package/docs/FUND_MOVEMENT_AND_ACCOUNTING.md +1 -1
- package/docs/GRID_RECONCILE.md +3 -3
- package/docs/ORDER_ENGINE_POST_1.0_RETROSPECTIVE.md +192 -31
- package/docs/README.md +1 -1
- package/docs/architecture.md +26 -8
- package/modules/README.md +4 -0
- package/package.json +4 -3
|
@@ -265,7 +265,7 @@ declare function stopBotsConfigPollInterval(bot: BotLike): void;
|
|
|
265
265
|
/**
|
|
266
266
|
* Release the market adapter runtime for a bot.
|
|
267
267
|
* In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
|
|
268
|
-
* @param {
|
|
268
|
+
* @param {unknown} _bot - Unused; release is keyed by botId.
|
|
269
269
|
* @param {string} botId - Bot identifier
|
|
270
270
|
* @param {string} [context='shutdown'] - Context label for logging
|
|
271
271
|
* @returns {Promise<any>}
|
|
@@ -324,6 +324,27 @@ declare function checkGridLockHoldDuration(bot: BotLike): number;
|
|
|
324
324
|
* @returns {Promise<void>}
|
|
325
325
|
*/
|
|
326
326
|
declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise<void>;
|
|
327
|
+
/**
|
|
328
|
+
* Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
|
|
329
|
+
*
|
|
330
|
+
* Retrospective R5.5: "Make deferred work level-triggered — never let a retry
|
|
331
|
+
* depend on a future event." The spread check has no driver of its own: it runs
|
|
332
|
+
* post-fill and on the blockchain-fetch tick (240 min by default, and the
|
|
333
|
+
* open-orders loop that would shorten it is OFF by default). Every watchdog
|
|
334
|
+
* window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
|
|
335
|
+
* therefore unreachable while the grid sits out of tolerance, which is how a
|
|
336
|
+
* live bot went 86 minutes without a single check. While out of tolerance the
|
|
337
|
+
* watchdog now schedules its own retry, so the clocks run at any base cadence.
|
|
338
|
+
*
|
|
339
|
+
* Deduped and shutdown-safe: one timer at a time, never while a recheck is in
|
|
340
|
+
* flight, never while shutting down. `performPeriodicGridChecks` keeps its own
|
|
341
|
+
* pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
|
|
342
|
+
*
|
|
343
|
+
* @param {Object} bot - DEXBot instance
|
|
344
|
+
* @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
|
|
345
|
+
* @returns {boolean} True when a recheck was armed (or already pending)
|
|
346
|
+
*/
|
|
347
|
+
export declare function scheduleSpreadRecheck(bot: BotLike, delayOverrideMs?: number): boolean;
|
|
327
348
|
/**
|
|
328
349
|
* Out-of-spread persistence watchdog: the never-run-stale backstop.
|
|
329
350
|
*
|
|
@@ -342,9 +363,12 @@ declare function executeMaintenanceLogic(bot: BotLike, context: string): Promise
|
|
|
342
363
|
* @param {number} ordersPlaced - Correction orders placed this tick
|
|
343
364
|
* @param {boolean} [transient=false] - True when the correction was refused for
|
|
344
365
|
* a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
|
|
366
|
+
* @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
|
|
367
|
+
* after the placement, or undefined when the correction placed nothing /
|
|
368
|
+
* could not be measured
|
|
345
369
|
* @returns {{staleMs: number, escalated: boolean}}
|
|
346
370
|
*/
|
|
347
|
-
export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean): {
|
|
371
|
+
export declare function trackOutOfSpreadStaleness(bot: BotLike, spreadChecked: boolean, ordersPlaced: number, transient?: boolean, excessAfter?: number): {
|
|
348
372
|
staleMs: number;
|
|
349
373
|
escalated: boolean;
|
|
350
374
|
};
|
|
@@ -461,6 +485,11 @@ declare function getMetrics(bot: BotLike): {
|
|
|
461
485
|
spreadCorrectionRefusedCount: number;
|
|
462
486
|
spreadCorrectionRefusalCauses: Record<string, number>;
|
|
463
487
|
spreadCorrectionRetryCount: number;
|
|
488
|
+
stalePlanDroppedCount: number;
|
|
489
|
+
stalePlanProceedCount: number;
|
|
490
|
+
clampedNoopUpdateCount: number;
|
|
491
|
+
chunkAbortedOpCount: number;
|
|
492
|
+
reloadReplacedRailCount: number;
|
|
464
493
|
};
|
|
465
494
|
/**
|
|
466
495
|
* Read open orders from chain, sync with local state, and process any fills found.
|
|
@@ -492,6 +521,7 @@ declare const _default: {
|
|
|
492
521
|
handlePendingTriggerReset: typeof handlePendingTriggerReset;
|
|
493
522
|
setupTriggerFileDetection: typeof setupTriggerFileDetection;
|
|
494
523
|
performPeriodicGridChecks: typeof performPeriodicGridChecks;
|
|
524
|
+
scheduleSpreadRecheck: typeof scheduleSpreadRecheck;
|
|
495
525
|
isOpenOrdersSyncLoopEnabled: typeof isOpenOrdersSyncLoopEnabled;
|
|
496
526
|
startOpenOrdersSyncLoop: typeof startOpenOrdersSyncLoop;
|
|
497
527
|
stopOpenOrdersSyncLoop: typeof stopOpenOrdersSyncLoop;
|
|
@@ -8,7 +8,7 @@ const require = createRequire(import.meta.url);
|
|
|
8
8
|
import { lazyShim } from './lazy_shim.js';
|
|
9
9
|
import fs from 'node:fs';
|
|
10
10
|
import { spawn } from 'node:child_process';
|
|
11
|
-
import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills } from './order/utils/system.js';
|
|
11
|
+
import { nowIso, fireStructuralResync, resolveLastFillGuardReference, hasUnsettledFills, resolveGridUpdateBlockers, noteGridUpdateBlockers, expireLastFillGuardReferences } from './order/utils/system.js';
|
|
12
12
|
import { withPlacement } from './order/placement_lock.js';
|
|
13
13
|
import { path } from './path_api.js';
|
|
14
14
|
import * as chainOrders from './chain_orders.js';
|
|
@@ -740,8 +740,19 @@ function getTargetedSyncReason(bot) {
|
|
|
740
740
|
}
|
|
741
741
|
async function maybeRunTargetedDriftReconciliation(bot, context) {
|
|
742
742
|
const trigger = getTargetedSyncReason(bot);
|
|
743
|
-
if (!trigger)
|
|
743
|
+
if (!trigger) {
|
|
744
|
+
// CONSECUTIVE means consecutive: a tick that never evaluates the gate
|
|
745
|
+
// must clear the counter. Without this reset a counter parked at
|
|
746
|
+
// GRID_BLOCKED_ESCALATE_CYCLES survives an arbitrarily long
|
|
747
|
+
// trigger-less stretch, and the first unreadable cycle afterwards fires
|
|
748
|
+
// a structural resync instead of waiting the full window.
|
|
749
|
+
// (docs/FILL_PIPELINE.md B1; pinned by test_targeted_drift_reconcile.)
|
|
750
|
+
// Guarded: without a manager there is no counter to clear, and the
|
|
751
|
+
// helper would otherwise park a stray field on the bot.
|
|
752
|
+
if (bot.manager)
|
|
753
|
+
noteGridUpdateBlockers(bot, []);
|
|
744
754
|
return false;
|
|
755
|
+
}
|
|
745
756
|
// UNSETTLED-FILL GATE: a startup-style reconcile derives its window from the
|
|
746
757
|
// master grid. While fills are still queued / draining / in flight / owed a
|
|
747
758
|
// crawl, that master still holds the filled slots as ACTIVE and the boundary
|
|
@@ -752,9 +763,25 @@ async function maybeRunTargetedDriftReconciliation(bot, context) {
|
|
|
752
763
|
// resolves against the crawled rail. No cooldown stamp, so the retry is
|
|
753
764
|
// immediate once the fills settle.
|
|
754
765
|
const unsettled = hasUnsettledFills(bot);
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
766
|
+
const gridBlockers = resolveGridUpdateBlockers(bot);
|
|
767
|
+
// Escalation: only an UNREADABLE pipeline state escalates, and only after
|
|
768
|
+
// GRID_BLOCKED_ESCALATE_CYCLES consecutive attempts. A busy pipeline is
|
|
769
|
+
// normal work; the placement/broadcast watchdogs own their own bounds.
|
|
770
|
+
const blockedGate = noteGridUpdateBlockers(bot, gridBlockers);
|
|
771
|
+
if (blockedGate.escalate) {
|
|
772
|
+
const codes = gridBlockers.map((b) => b.code).join(',');
|
|
773
|
+
bot._log(`[GRID-BLOCKED] ${blockedGate.cycles} consecutive blocked cycle(s) with unreadable pipeline state ` +
|
|
774
|
+
`(${codes}); requesting a structural resync`, 'error');
|
|
775
|
+
fireStructuralResync(bot, 'grid update blocked by unreadable pipeline state', { cycles: blockedGate.cycles, codes }, '[GRID-BLOCKED]');
|
|
776
|
+
}
|
|
777
|
+
if (gridBlockers.length > 0) {
|
|
778
|
+
// Any blocker defers the reconcile: it is a chain-mutating update, and
|
|
779
|
+
// mutating while a fill batch, a broadcast or a placement region is
|
|
780
|
+
// unresolved is what built the stale-master plans this gate exists for.
|
|
781
|
+
const codes = gridBlockers.map((b) => b.code).join(',');
|
|
782
|
+
const reason = unsettled.reason || gridBlockers[0].detail;
|
|
783
|
+
bot._log(`[TARGETED-SYNC] Deferring ${context} reconciliation: ${reason} ` +
|
|
784
|
+
`[${codes}] — the rail must absorb the fills before it is topped up`, 'debug');
|
|
758
785
|
return false;
|
|
759
786
|
}
|
|
760
787
|
const now = Date.now();
|
|
@@ -2074,7 +2101,7 @@ function stopBotsConfigPollInterval(bot) {
|
|
|
2074
2101
|
/**
|
|
2075
2102
|
* Release the market adapter runtime for a bot.
|
|
2076
2103
|
* In PM2 mode this is a no-op; in direct mode it calls the shared runtime's releaseBot.
|
|
2077
|
-
* @param {
|
|
2104
|
+
* @param {unknown} _bot - Unused; release is keyed by botId.
|
|
2078
2105
|
* @param {string} botId - Bot identifier
|
|
2079
2106
|
* @param {string} [context='shutdown'] - Context label for logging
|
|
2080
2107
|
* @returns {Promise<any>}
|
|
@@ -2635,6 +2662,7 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2635
2662
|
}
|
|
2636
2663
|
else {
|
|
2637
2664
|
const spreadResult = await bot.manager.checkSpreadCondition(BitShares, bot.updateOrdersOnChainPlan.bind(bot));
|
|
2665
|
+
const spreadExcessAfter = Number(spreadResult?.excessAfter);
|
|
2638
2666
|
if (await bot._abortFlowIfIllegalState(`${context} spread check`))
|
|
2639
2667
|
return;
|
|
2640
2668
|
// Mirror the stall cause and arm the targeted-sync refresh for
|
|
@@ -2651,7 +2679,9 @@ async function executeMaintenanceLogic(bot, context) {
|
|
|
2651
2679
|
}
|
|
2652
2680
|
// Persistence watchdog: a correction that keeps placing nothing
|
|
2653
2681
|
// while the spread stays wide is a stale grid, not patience.
|
|
2654
|
-
|
|
2682
|
+
// `excessAfter` is the post-placement violation, so a placement
|
|
2683
|
+
// that did not reduce it no longer counts as progress.
|
|
2684
|
+
trackOutOfSpreadStaleness(bot, true, spreadPlaced, spreadResult?.transient === true, (spreadResult?.excessAfter !== undefined && Number.isFinite(spreadExcessAfter)) ? spreadExcessAfter : undefined);
|
|
2655
2685
|
}
|
|
2656
2686
|
}
|
|
2657
2687
|
catch (err) {
|
|
@@ -2679,6 +2709,75 @@ function clearGeometryResyncMemory(bot) {
|
|
|
2679
2709
|
bot._lastGeometryResyncPivotKey = '';
|
|
2680
2710
|
bot._geometryResyncWarnedPivotKey = '';
|
|
2681
2711
|
}
|
|
2712
|
+
/**
|
|
2713
|
+
* Arm a one-shot, level-triggered spread recheck (TIMING.SPREAD_STALE_RECHECK_MS).
|
|
2714
|
+
*
|
|
2715
|
+
* Retrospective R5.5: "Make deferred work level-triggered — never let a retry
|
|
2716
|
+
* depend on a future event." The spread check has no driver of its own: it runs
|
|
2717
|
+
* post-fill and on the blockchain-fetch tick (240 min by default, and the
|
|
2718
|
+
* open-orders loop that would shorten it is OFF by default). Every watchdog
|
|
2719
|
+
* window (warn 10 min, geometry 10 min, fund 5 min, pivot-expiry 20 min) is
|
|
2720
|
+
* therefore unreachable while the grid sits out of tolerance, which is how a
|
|
2721
|
+
* live bot went 86 minutes without a single check. While out of tolerance the
|
|
2722
|
+
* watchdog now schedules its own retry, so the clocks run at any base cadence.
|
|
2723
|
+
*
|
|
2724
|
+
* Deduped and shutdown-safe: one timer at a time, never while a recheck is in
|
|
2725
|
+
* flight, never while shutting down. `performPeriodicGridChecks` keeps its own
|
|
2726
|
+
* pipeline/broadcast deferrals, so this cannot interleave with a fill cycle.
|
|
2727
|
+
*
|
|
2728
|
+
* @param {Object} bot - DEXBot instance
|
|
2729
|
+
* @param {number} [delayOverrideMs] - Injectable delay (tests); production reads TIMING
|
|
2730
|
+
* @returns {boolean} True when a recheck was armed (or already pending)
|
|
2731
|
+
*/
|
|
2732
|
+
export function scheduleSpreadRecheck(bot, delayOverrideMs) {
|
|
2733
|
+
const configured = Number.isFinite(Number(delayOverrideMs)) && Number(delayOverrideMs) > 0
|
|
2734
|
+
? Number(delayOverrideMs)
|
|
2735
|
+
: (Number(TIMING?.SPREAD_STALE_RECHECK_MS) > 0 ? Number(TIMING.SPREAD_STALE_RECHECK_MS) : 0);
|
|
2736
|
+
const ms = configured > 0 ? configured : 0;
|
|
2737
|
+
if (!(ms > 0))
|
|
2738
|
+
return false;
|
|
2739
|
+
if (!bot || bot._shuttingDown)
|
|
2740
|
+
return false;
|
|
2741
|
+
if (bot._spreadRecheckTimer || bot._spreadRecheckInFlight)
|
|
2742
|
+
return true;
|
|
2743
|
+
try {
|
|
2744
|
+
bot._spreadRecheckTimer = setTimeout(async () => {
|
|
2745
|
+
bot._spreadRecheckTimer = null;
|
|
2746
|
+
if (bot._shuttingDown || bot._spreadRecheckInFlight)
|
|
2747
|
+
return;
|
|
2748
|
+
bot._spreadRecheckInFlight = true;
|
|
2749
|
+
try {
|
|
2750
|
+
bot._log(`[SPREAD-STALE] Level-triggered recheck (${Math.round(ms / 1000)}s cadence while out of tolerance)`, 'debug');
|
|
2751
|
+
await performPeriodicGridChecks(bot);
|
|
2752
|
+
}
|
|
2753
|
+
catch (err) {
|
|
2754
|
+
try {
|
|
2755
|
+
bot._warn(`Spread recheck failed: ${getErrorMessage(err)}`);
|
|
2756
|
+
}
|
|
2757
|
+
catch { /* logging is best-effort */ }
|
|
2758
|
+
}
|
|
2759
|
+
finally {
|
|
2760
|
+
bot._spreadRecheckInFlight = false;
|
|
2761
|
+
// `trackOutOfSpreadStaleness` runs inside the maintenance call,
|
|
2762
|
+
// while this flag is true, so its dedupe check intentionally
|
|
2763
|
+
// does not arm the next timer. Re-arm after the region ends when
|
|
2764
|
+
// the level is still active; otherwise a single retry would be
|
|
2765
|
+
// falsely documented as a recurring level-triggered repair.
|
|
2766
|
+
if (!bot._shuttingDown && Number(bot.manager?.outOfSpread) > 0) {
|
|
2767
|
+
scheduleSpreadRecheck(bot, ms);
|
|
2768
|
+
}
|
|
2769
|
+
}
|
|
2770
|
+
}, ms);
|
|
2771
|
+
const timer = bot._spreadRecheckTimer;
|
|
2772
|
+
if (typeof timer?.unref === 'function')
|
|
2773
|
+
timer.unref();
|
|
2774
|
+
return true;
|
|
2775
|
+
}
|
|
2776
|
+
catch {
|
|
2777
|
+
bot._spreadRecheckTimer = null;
|
|
2778
|
+
return false;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2682
2781
|
/**
|
|
2683
2782
|
* Out-of-spread persistence watchdog: the never-run-stale backstop.
|
|
2684
2783
|
*
|
|
@@ -2697,19 +2796,44 @@ function clearGeometryResyncMemory(bot) {
|
|
|
2697
2796
|
* @param {number} ordersPlaced - Correction orders placed this tick
|
|
2698
2797
|
* @param {boolean} [transient=false] - True when the correction was refused for
|
|
2699
2798
|
* a retryable reason (stale plan / duplicate CREATE / unclassified non-execution)
|
|
2799
|
+
* @param {number} [excessAfter] - Out-of-spread violation (grid steps) MEASURED
|
|
2800
|
+
* after the placement, or undefined when the correction placed nothing /
|
|
2801
|
+
* could not be measured
|
|
2700
2802
|
* @returns {{staleMs: number, escalated: boolean}}
|
|
2701
2803
|
*/
|
|
2702
|
-
export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false) {
|
|
2804
|
+
export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, transient = false, excessAfter) {
|
|
2703
2805
|
const outOfSpread = Number(bot?.manager?.outOfSpread) || 0;
|
|
2704
2806
|
const nowMs = Date.now();
|
|
2705
2807
|
const inToleranceResetMs = Number(TIMING?.SPREAD_STALE_IN_TOLERANCE_RESET_MS) > 0
|
|
2706
2808
|
? Number(TIMING.SPREAD_STALE_IN_TOLERANCE_RESET_MS)
|
|
2707
2809
|
: 10 * 60 * 1000;
|
|
2708
|
-
|
|
2810
|
+
// PLACED IS NOT PROGRESS (retrospective §9.11.3, "a repair must reduce the
|
|
2811
|
+
// violation"). Clearing the clock on ANY placement let a correction that
|
|
2812
|
+
// only rotated the side that was not blocking report a heal every cycle:
|
|
2813
|
+
// live, 6 buy rotations at a still-3.0% spread reset the clock AND the
|
|
2814
|
+
// geometry-recenter memory, so the pivot expiry and the one-recenter-per-pivot
|
|
2815
|
+
// budget could never fire while the sell side stayed unquoted for 86 minutes.
|
|
2816
|
+
// A placement now clears the clock only when the measured violation actually
|
|
2817
|
+
// shrank (excessAfter < outOfSpread, or 0). Unmeasurable => legacy behavior
|
|
2818
|
+
// (a placement counts), so a broken measurement can never churn a re-center.
|
|
2819
|
+
const measured = Number(excessAfter);
|
|
2820
|
+
const excessKnown = excessAfter !== undefined && excessAfter !== null && Number.isFinite(measured);
|
|
2821
|
+
const placedReducedViolation = ordersPlaced > 0 && (!excessKnown || measured < outOfSpread);
|
|
2822
|
+
if (ordersPlaced > 0 && !placedReducedViolation) {
|
|
2823
|
+
if (!bot._outOfSpreadNoProgressWarned) {
|
|
2824
|
+
bot._outOfSpreadNoProgressWarned = true;
|
|
2825
|
+
bot._log(`[SPREAD-STALE] ${ordersPlaced} order(s) placed but the spread is still ${measured} step(s) `
|
|
2826
|
+
+ `out of tolerance (was ${outOfSpread}) — counting as no progress, the stale clock keeps running`, 'warn');
|
|
2827
|
+
}
|
|
2828
|
+
// Fall through: the clock is NOT cleared, so the documented windows
|
|
2829
|
+
// (warn / geometry / fund / pivot-expiry) stay reachable.
|
|
2830
|
+
}
|
|
2831
|
+
if (placedReducedViolation) {
|
|
2709
2832
|
bot._outOfSpreadSince = 0;
|
|
2710
2833
|
bot._outOfSpreadStaleWarned = false;
|
|
2711
2834
|
bot._outOfSpreadTransientCount = 0;
|
|
2712
2835
|
bot._outOfSpreadToleranceSince = 0;
|
|
2836
|
+
bot._outOfSpreadNoProgressWarned = false;
|
|
2713
2837
|
clearGeometryResyncMemory(bot);
|
|
2714
2838
|
try {
|
|
2715
2839
|
bot.manager._spreadRailOversubscribed = null;
|
|
@@ -2736,6 +2860,7 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
|
|
|
2736
2860
|
bot._outOfSpreadStaleWarned = false;
|
|
2737
2861
|
bot._outOfSpreadTransientCount = 0;
|
|
2738
2862
|
bot._outOfSpreadToleranceSince = 0;
|
|
2863
|
+
bot._outOfSpreadNoProgressWarned = false;
|
|
2739
2864
|
clearGeometryResyncMemory(bot);
|
|
2740
2865
|
try {
|
|
2741
2866
|
bot.manager._spreadRailOversubscribed = null;
|
|
@@ -2744,6 +2869,13 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
|
|
|
2744
2869
|
return { staleMs: 0, escalated: false };
|
|
2745
2870
|
}
|
|
2746
2871
|
bot._outOfSpreadToleranceSince = 0;
|
|
2872
|
+
// LEVEL-TRIGGERED RECHECK. Every watchdog clock below assumes the spread
|
|
2873
|
+
// check is EVALUATED, but the base cadence is the blockchain fetch interval
|
|
2874
|
+
// (240 min by default) plus the post-fill path — so a grid that is out of
|
|
2875
|
+
// tolerance waits hours for the next look and every window becomes dead
|
|
2876
|
+
// code (live: 86 minutes with no check at all). While out of tolerance, arm
|
|
2877
|
+
// one short recheck; it re-arms itself until the spread heals.
|
|
2878
|
+
scheduleSpreadRecheck(bot);
|
|
2747
2879
|
if (!spreadChecked) {
|
|
2748
2880
|
const since = Number(bot._outOfSpreadSince) || 0;
|
|
2749
2881
|
return { staleMs: since > 0 ? Date.now() - since : 0, escalated: false };
|
|
@@ -2813,12 +2945,27 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
|
|
|
2813
2945
|
const geometryEscalateMs = Number(TIMING?.SPREAD_STALE_GEOMETRY_ESCALATE_MS) > 0
|
|
2814
2946
|
? Number(TIMING.SPREAD_STALE_GEOMETRY_ESCALATE_MS)
|
|
2815
2947
|
: 10 * 60 * 1000;
|
|
2948
|
+
// After the one re-center per pivot, the pivot gets a BOUNDED extra window
|
|
2949
|
+
// (SPREAD_STALE_PIVOT_EXPIRE_MS) instead of escalating to NOTHING ("a fill
|
|
2950
|
+
// or the pivot TTL is the only heal"). Live, a single off-market pivot buy
|
|
2951
|
+
// stranded the grid out of spread for the full 24h TTL because every
|
|
2952
|
+
// rebuild re-armed the same pivot. Past the window the guard references
|
|
2953
|
+
// are expired outright (expireLastFillGuardReferences) so spread
|
|
2954
|
+
// correction trades around the live market again; both re-arm on the next
|
|
2955
|
+
// real fill. 0 disables the expiry (legacy behavior).
|
|
2956
|
+
const pivotExpireConfigured = Number(TIMING?.SPREAD_STALE_PIVOT_EXPIRE_MS);
|
|
2957
|
+
const pivotExpireMs = pivotExpireConfigured > 0
|
|
2958
|
+
? pivotExpireConfigured
|
|
2959
|
+
: pivotExpireConfigured === 0
|
|
2960
|
+
? Number.POSITIVE_INFINITY
|
|
2961
|
+
: 20 * 60 * 1000;
|
|
2816
2962
|
const escalateMs = oversubscribedFresh
|
|
2817
2963
|
? fundEscalateMs
|
|
2818
2964
|
: geometryAlreadyRecentered
|
|
2819
|
-
// Do not re-center again for the same fill.
|
|
2820
|
-
//
|
|
2821
|
-
|
|
2965
|
+
// Do not re-center again for the same fill. Bounded pivot-expiry
|
|
2966
|
+
// window instead of Infinity: past it the guard references are
|
|
2967
|
+
// cleared and the market is traded around (see pivotExpireMs).
|
|
2968
|
+
? pivotExpireMs
|
|
2822
2969
|
: geometryBlocked
|
|
2823
2970
|
? geometryEscalateMs
|
|
2824
2971
|
: (Number(TIMING?.SPREAD_STALE_ESCALATE_MS) > 0
|
|
@@ -2850,6 +2997,19 @@ export function trackOutOfSpreadStaleness(bot, spreadChecked, ordersPlaced, tran
|
|
|
2850
2997
|
const lastAt = Number(bot._lastSpreadStaleResyncAt) || 0;
|
|
2851
2998
|
if (now - lastAt >= cooldownMs) {
|
|
2852
2999
|
bot._lastSpreadStaleResyncAt = now;
|
|
3000
|
+
if (geometryAlreadyRecentered) {
|
|
3001
|
+
// The re-center for this pivot already ran and cannot clear the
|
|
3002
|
+
// veto; the bounded pivot-expiry window is up. Clear the guard
|
|
3003
|
+
// references (live + persisted) so the market is traded around
|
|
3004
|
+
// again instead of staying stranded until the 24h TTL.
|
|
3005
|
+
expireLastFillGuardReferences(bot.manager, `out of tolerance ${staleMin}min after re-centering on fill pivot ${geometryPivotKey.split('|')[0]}`);
|
|
3006
|
+
clearGeometryResyncMemory(bot);
|
|
3007
|
+
bot._outOfSpreadSince = 0;
|
|
3008
|
+
bot._outOfSpreadStaleWarned = false;
|
|
3009
|
+
bot._outOfSpreadTransientCount = 0;
|
|
3010
|
+
bot._outOfSpreadToleranceSince = 0;
|
|
3011
|
+
return { staleMs, escalated: true };
|
|
3012
|
+
}
|
|
2853
3013
|
if (geometryBlocked && geometryPivotKey)
|
|
2854
3014
|
bot._lastGeometryResyncPivotKey = geometryPivotKey;
|
|
2855
3015
|
bot._log(`[SPREAD-STALE] Spread out of tolerance for ${staleMin}min despite corrections` +
|
|
@@ -3329,7 +3489,26 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3329
3489
|
// commit refusal + re-adoption cycle. Defer until the batch
|
|
3330
3490
|
// completes; the fill consumer is already gated on _batchInFlight,
|
|
3331
3491
|
// so this only blocks the recovery itself, never new fills.
|
|
3332
|
-
|
|
3492
|
+
//
|
|
3493
|
+
// SAME DEFER FOR AN UNSETTLED FILL PIPELINE: a fill batch mutates
|
|
3494
|
+
// the grid in memory long before its persistence flush reaches
|
|
3495
|
+
// disk, so the on-disk snapshot can still describe the PRE-fill
|
|
3496
|
+
// slot sizes. Reloading it rolls an applied fill back, and the next
|
|
3497
|
+
// fill for the same order is then subtracted from that rolled-back
|
|
3498
|
+
// baseline — leaving a phantom residual plus an orderId the chain
|
|
3499
|
+
// no longer has. Observed live: two partial fills of one order in
|
|
3500
|
+
// consecutive blocks; the reload between them turned a completed
|
|
3501
|
+
// order into a phantom parked in the gap band, which froze the
|
|
3502
|
+
// boundary crawl so the spread could not be closed.
|
|
3503
|
+
//
|
|
3504
|
+
// `hasUnsettledFills` is the same predicate the destructive
|
|
3505
|
+
// snapshot-reject gate uses: only a settled fill pipeline makes the
|
|
3506
|
+
// on-disk snapshot a safe base.
|
|
3507
|
+
const fillPipeline = hasUnsettledFills(bot);
|
|
3508
|
+
const deferCause = bot._batchInFlight > 0
|
|
3509
|
+
? `_batchInFlight=${bot._batchInFlight}`
|
|
3510
|
+
: (fillPipeline.unsettled ? `fill pipeline not settled: ${fillPipeline.reason}` : null);
|
|
3511
|
+
if (deferCause) {
|
|
3333
3512
|
const now = Date.now();
|
|
3334
3513
|
if (!bot._structuralGridResyncDeferStartedAt) {
|
|
3335
3514
|
bot._structuralGridResyncDeferStartedAt = now;
|
|
@@ -3338,7 +3517,7 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3338
3517
|
bot._structuralGridResyncDeferCount = (bot._structuralGridResyncDeferCount || 0) + 1;
|
|
3339
3518
|
const deferredMs = now - bot._structuralGridResyncDeferStartedAt;
|
|
3340
3519
|
if (deferredMs >= STRUCTURAL_RESYNC_MAX_DEFER_MS) {
|
|
3341
|
-
bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite
|
|
3520
|
+
bot._warn(`[RECOVERY] Structural resync max-defer reached (${deferredMs}ms, cap ${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms, deferred ${bot._structuralGridResyncDeferCount}x) — forcing despite ${deferCause}: ${reason}`);
|
|
3342
3521
|
bot._structuralGridResyncDeferStartedAt = null;
|
|
3343
3522
|
bot._structuralGridResyncDeferCount = 0;
|
|
3344
3523
|
}
|
|
@@ -3348,10 +3527,10 @@ function wireStructuralGridResyncRequest(bot) {
|
|
|
3348
3527
|
// defer lines in seconds); count the rest and surface them only in
|
|
3349
3528
|
// the final forced message.
|
|
3350
3529
|
if (bot._structuralGridResyncDeferCount === 1) {
|
|
3351
|
-
bot._warn(`[RECOVERY] Structural resync defer (
|
|
3530
|
+
bot._warn(`[RECOVERY] Structural resync defer (${deferCause}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`);
|
|
3352
3531
|
}
|
|
3353
3532
|
else {
|
|
3354
|
-
bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
|
|
3533
|
+
bot.manager?.logger?.log?.(`[RECOVERY] Structural resync defer (suppressed; ${deferCause}, count=${bot._structuralGridResyncDeferCount}, deferred ${deferredMs}ms/${STRUCTURAL_RESYNC_MAX_DEFER_MS}ms): ${reason}`, 'debug');
|
|
3355
3534
|
}
|
|
3356
3535
|
bot._structuralGridResyncTimer = setTimeout(runStructuralResync, TIMING.LOCK_REFRESH_MIN_MS);
|
|
3357
3536
|
return;
|
|
@@ -3588,6 +3767,7 @@ export default {
|
|
|
3588
3767
|
handlePendingTriggerReset,
|
|
3589
3768
|
setupTriggerFileDetection,
|
|
3590
3769
|
performPeriodicGridChecks,
|
|
3770
|
+
scheduleSpreadRecheck,
|
|
3591
3771
|
isOpenOrdersSyncLoopEnabled,
|
|
3592
3772
|
startOpenOrdersSyncLoop,
|
|
3593
3773
|
stopOpenOrdersSyncLoop,
|
|
@@ -11,6 +11,7 @@ import { restoreGapEvacStreaks, applyPersistedPendingCrawls, resetLastFillPivot,
|
|
|
11
11
|
import { blockchainToFloat, calculateGapSlots, resolveGapSlots, validatePersistedBoundary, isTransientInBandRejection } from './order/utils/math.js';
|
|
12
12
|
import { ON_MISSING_GENESIS, isMissingGenesisError } from './order/genesis_policy.js';
|
|
13
13
|
import { hasExecutableActions } from './order/utils/validate.js';
|
|
14
|
+
import { releasePlanLease } from './order/placement_lock.js';
|
|
14
15
|
import { getErrorMessage } from './utils/errors.js';
|
|
15
16
|
const { isGridBloated } = grid;
|
|
16
17
|
/**
|
|
@@ -643,7 +644,10 @@ async function _rebalanceAfterRecovery(bot, context) {
|
|
|
643
644
|
return;
|
|
644
645
|
}
|
|
645
646
|
try {
|
|
646
|
-
|
|
647
|
+
// holdPlanRegion: every result on this path is either executed below or
|
|
648
|
+
// released explicitly, so the plan→commit window is serialized against
|
|
649
|
+
// the fill-driven planner.
|
|
650
|
+
const rebalanceResult = await bot.manager.performSafeRebalance([], new Set(), { holdPlanRegion: true });
|
|
647
651
|
if (!rebalanceResult || rebalanceResult.aborted) {
|
|
648
652
|
// A deferred abort means owed work is still outstanding (broadcast
|
|
649
653
|
// held the plan, or the identical-held-plan suppression is waiting
|
|
@@ -661,9 +665,19 @@ async function _rebalanceAfterRecovery(bot, context) {
|
|
|
661
665
|
// rebalance stack stays balanced and _rebalanceState returns to
|
|
662
666
|
// NORMAL (mirrors _executeBatchIfNeeded's no-action pop). Aborted
|
|
663
667
|
// results are never pushed, so only this branch needs the pop.
|
|
668
|
+
// The result also owns the plan region acquired before planning;
|
|
669
|
+
// no executor will adopt it on this path, so release it here.
|
|
670
|
+
releasePlanLease(rebalanceResult);
|
|
664
671
|
bot.manager?._popWorkingGridRef?.(rebalanceResult);
|
|
665
672
|
return;
|
|
666
673
|
}
|
|
674
|
+
const railCount = Array.isArray(rebalanceResult.actions) ? rebalanceResult.actions.length : 0;
|
|
675
|
+
if (railCount > 0 && bot._metrics) {
|
|
676
|
+
// Cost of a recovery reload/repair in re-placed rails. Counted here
|
|
677
|
+
// (not derived from logs) so the reload→re-place amplification is
|
|
678
|
+
// observable next to the other churn counters.
|
|
679
|
+
bot._metrics.reloadReplacedRailCount = (bot._metrics.reloadReplacedRailCount || 0) + railCount;
|
|
680
|
+
}
|
|
667
681
|
bot.manager.logger?.log?.(`[RECOVERY] Re-placing ${(rebalanceResult.actions ?? []).length} missing rail(s) after ${context}`, 'info');
|
|
668
682
|
await bot.updateOrdersOnChainBatch(rebalanceResult);
|
|
669
683
|
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/** A held instance lock. release() is ownership-checked and idempotent. */
|
|
2
|
+
export interface BotInstanceLock {
|
|
3
|
+
release(): void;
|
|
4
|
+
/** Absolute path of the lock file (diagnostics/tests). */
|
|
5
|
+
lockPath: string;
|
|
6
|
+
}
|
|
7
|
+
export interface AcquireInstanceLockOptions {
|
|
8
|
+
/** Best-effort logger (manager logger or launcher console shim). */
|
|
9
|
+
log?: (message: string, level?: string) => void;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Acquire the single-controller lock for `account`.
|
|
13
|
+
*
|
|
14
|
+
* Throws (never waits) when another LIVE process holds the lock — the error
|
|
15
|
+
* message names the holder (pid, bot, start time) so the operator can stop the
|
|
16
|
+
* right instance. A DEAD holder (crashed process, recycled pid) is detected
|
|
17
|
+
* via /proc liveness + starttime match and its stale lock is taken over
|
|
18
|
+
* immediately, so a PM2 restart after a crash is not blocked by its own
|
|
19
|
+
* previous incarnation.
|
|
20
|
+
*
|
|
21
|
+
* @param {string} account - Chain account name (lock key)
|
|
22
|
+
* @param {string} botName - Configured bot name (diagnostics only)
|
|
23
|
+
* @param {Object} [options]
|
|
24
|
+
* @returns {BotInstanceLock} Handle with release()
|
|
25
|
+
*/
|
|
26
|
+
export declare function acquireBotInstanceLock(account: string, botName?: string, options?: AcquireInstanceLockOptions): BotInstanceLock;
|
|
27
|
+
//# sourceMappingURL=instance_lock.d.ts.map
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// node-only entry point — per-account bot instance lock (/proc liveness, fs)
|
|
3
|
+
/**
|
|
4
|
+
* instance_lock.ts - Single-controller lock per trading account
|
|
5
|
+
*
|
|
6
|
+
* Prevents two bot processes (e.g. one started via `dexbot pm2` and one via a
|
|
7
|
+
* normal `dexbot start` unlock) from controlling the SAME account at the same
|
|
8
|
+
* time. Observed live (2026-11-14): two controllers kept independent in-memory
|
|
9
|
+
* grids over the same 40 chain orders, cancelled each other's orders as
|
|
10
|
+
* "duplicate orphans", attributed the same fill to different slots, and
|
|
11
|
+
* finally stranded the grid out of spread for ~24h behind an off-market
|
|
12
|
+
* LAST-FILL-GUARD pivot. This lock makes the second controller fail fast with
|
|
13
|
+
* the holder's identity instead of silently double-trading.
|
|
14
|
+
*
|
|
15
|
+
* The lock is keyed by ACCOUNT, not bot name: the conflict domain is the chain
|
|
16
|
+
* account's open orders and fills, so even two differently named bot entries
|
|
17
|
+
* sharing an account are refused.
|
|
18
|
+
*
|
|
19
|
+
* Implementation reuses the market adapter's file-lock primitive (staleness +
|
|
20
|
+
* heartbeat + ownership-checked release) with a process-liveness aliveCheck
|
|
21
|
+
* that also compares /proc/<pid>/stat starttime, so a recycled pid can neither
|
|
22
|
+
* keep a dead holder's lock alive nor steal a live holder's lock.
|
|
23
|
+
*/
|
|
24
|
+
import { path } from '../path_api.js';
|
|
25
|
+
import { getStorage } from '../storage/index.js';
|
|
26
|
+
import { PATHS } from '../paths.js';
|
|
27
|
+
import { runtime } from '../runtime.js';
|
|
28
|
+
import { getProcessDiscovery } from '../process_discovery.js';
|
|
29
|
+
import { acquireFileLockSync, releaseFileLockSync } from '../../market_adapter/utils/file_lock.js';
|
|
30
|
+
import { getErrorMessage } from '../utils/errors.js';
|
|
31
|
+
const storage = getStorage();
|
|
32
|
+
function sanitizeLockKey(raw) {
|
|
33
|
+
return String(raw || 'unknown').toLowerCase().replace(/[^a-z0-9._-]+/g, '_').slice(0, 64) || 'unknown';
|
|
34
|
+
}
|
|
35
|
+
function instanceLocksDir() {
|
|
36
|
+
return PATHS.PROFILES.INSTANCE_LOCKS_DIR || path.join(PATHS.PROFILES_DIR, 'instance-locks');
|
|
37
|
+
}
|
|
38
|
+
function lockPathForAccount(account) {
|
|
39
|
+
return path.join(instanceLocksDir(), `${sanitizeLockKey(account)}.lock`);
|
|
40
|
+
}
|
|
41
|
+
function readLockPayload(lockPath) {
|
|
42
|
+
try {
|
|
43
|
+
const parsed = storage.readJSON(lockPath);
|
|
44
|
+
return parsed && typeof parsed === 'object' ? parsed : null;
|
|
45
|
+
}
|
|
46
|
+
catch (_) {
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Acquire the single-controller lock for `account`.
|
|
52
|
+
*
|
|
53
|
+
* Throws (never waits) when another LIVE process holds the lock — the error
|
|
54
|
+
* message names the holder (pid, bot, start time) so the operator can stop the
|
|
55
|
+
* right instance. A DEAD holder (crashed process, recycled pid) is detected
|
|
56
|
+
* via /proc liveness + starttime match and its stale lock is taken over
|
|
57
|
+
* immediately, so a PM2 restart after a crash is not blocked by its own
|
|
58
|
+
* previous incarnation.
|
|
59
|
+
*
|
|
60
|
+
* @param {string} account - Chain account name (lock key)
|
|
61
|
+
* @param {string} botName - Configured bot name (diagnostics only)
|
|
62
|
+
* @param {Object} [options]
|
|
63
|
+
* @returns {BotInstanceLock} Handle with release()
|
|
64
|
+
*/
|
|
65
|
+
export function acquireBotInstanceLock(account, botName = '', options = {}) {
|
|
66
|
+
const lockPath = lockPathForAccount(account);
|
|
67
|
+
const discovery = getProcessDiscovery();
|
|
68
|
+
try {
|
|
69
|
+
storage.ensureDir(instanceLocksDir());
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
throw new Error(`Cannot create instance-lock directory ${instanceLocksDir()}: ${getErrorMessage(err)}`);
|
|
73
|
+
}
|
|
74
|
+
const buildPayload = () => ({
|
|
75
|
+
pid: runtime.pid,
|
|
76
|
+
botName: String(botName || ''),
|
|
77
|
+
account: String(account || ''),
|
|
78
|
+
starttime: discovery.readStat(runtime.pid)?.starttime ?? 0,
|
|
79
|
+
at: new Date().toISOString(),
|
|
80
|
+
});
|
|
81
|
+
const aliveCheck = (pid) => {
|
|
82
|
+
if (!discovery.isAlive(pid))
|
|
83
|
+
return false;
|
|
84
|
+
// pid-recycle guard: the payload's starttime is the holder's boot
|
|
85
|
+
// identity. A recycled pid whose /proc starttime differs is NOT the
|
|
86
|
+
// holder — treat the lock as dead so a new controller can take over.
|
|
87
|
+
const info = readLockPayload(lockPath);
|
|
88
|
+
const holderStart = Number(info?.starttime);
|
|
89
|
+
if (Number(info?.pid) === pid && holderStart > 0) {
|
|
90
|
+
const stat = discovery.readStat(pid);
|
|
91
|
+
if (stat && stat.starttime !== holderStart)
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
return true;
|
|
95
|
+
};
|
|
96
|
+
const alreadyRunningMsg = () => {
|
|
97
|
+
const info = readLockPayload(lockPath);
|
|
98
|
+
const holderBot = info?.botName ? ` (bot ${info.botName})` : '';
|
|
99
|
+
const holderAt = info?.at ? `, started ${info.at}` : '';
|
|
100
|
+
return (`account '${account}' is already controlled by another bot process ` +
|
|
101
|
+
`(pid ${info?.pid ?? 'unknown'}${holderBot}${holderAt}, lock: ${lockPath}). ` +
|
|
102
|
+
`Stop that instance first (dexbot pm2 stop / dexbot stop) — two controllers ` +
|
|
103
|
+
`on one account corrupt each other's grids and fills.`);
|
|
104
|
+
};
|
|
105
|
+
const lock = acquireFileLockSync(lockPath, {
|
|
106
|
+
// Fail fast — the second controller must not wait out a live holder.
|
|
107
|
+
contention: 'throw',
|
|
108
|
+
// Only bounds an ALIVE-but-never-beating holder (heartbeat stopped
|
|
109
|
+
// without process death); a dead holder is stolen immediately by the
|
|
110
|
+
// aliveCheck above regardless of staleness.
|
|
111
|
+
staleMs: 10 * 60 * 1000,
|
|
112
|
+
aliveCheck,
|
|
113
|
+
buildPayload,
|
|
114
|
+
alreadyRunningMsg,
|
|
115
|
+
timeoutMsg: `cannot acquire instance lock for account '${account}': ${lockPath}`,
|
|
116
|
+
});
|
|
117
|
+
options.log?.(`Instance lock acquired for account '${account}' (bot ${botName || 'n/a'}, pid ${runtime.pid})`, 'info');
|
|
118
|
+
return {
|
|
119
|
+
lockPath,
|
|
120
|
+
release() {
|
|
121
|
+
try {
|
|
122
|
+
releaseFileLockSync({ fd: lock.fd, lockPath: lock.lockPath, heartbeat: lock.heartbeat, ownerId: lock.ownerId });
|
|
123
|
+
options.log?.(`Instance lock released for account '${account}'`, 'info');
|
|
124
|
+
}
|
|
125
|
+
catch (err) {
|
|
126
|
+
options.log?.(`Instance lock release failed for account '${account}': ${getErrorMessage(err)}`, 'warn');
|
|
127
|
+
}
|
|
128
|
+
},
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=instance_lock.js.map
|
|
@@ -11,13 +11,13 @@
|
|
|
11
11
|
*
|
|
12
12
|
* effectiveLatency = latencyMs + flapPenaltyMs
|
|
13
13
|
* flapScore = Σ 0.5 ^ (age_i / halfLife)
|
|
14
|
-
* flapPenaltyMs = min(flapScore *
|
|
14
|
+
* flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
|
|
15
15
|
*
|
|
16
|
-
* A node with no recent drops ranks on its ping alone. One drop costs
|
|
17
|
-
*
|
|
18
|
-
* node that flaps once and then behaves is re-admitted
|
|
19
|
-
* that keeps flapping stays at the back of the list, and
|
|
20
|
-
* "reset" by a lucky health probe (the old
|
|
16
|
+
* A node with no recent drops ranks on its ping alone. One undecayed drop costs
|
|
17
|
+
* DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
|
|
18
|
+
* half-life, so a node that flaps once and then behaves is re-admitted
|
|
19
|
+
* automatically, a node that keeps flapping stays at the back of the list, and
|
|
20
|
+
* nothing has to be "reset" by a lucky health probe (the old
|
|
21
21
|
* LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
|
|
22
22
|
*
|
|
23
23
|
* Everything here is pure: callers own persistence and logging. Drop history
|
|
@@ -12,13 +12,13 @@
|
|
|
12
12
|
*
|
|
13
13
|
* effectiveLatency = latencyMs + flapPenaltyMs
|
|
14
14
|
* flapScore = Σ 0.5 ^ (age_i / halfLife)
|
|
15
|
-
* flapPenaltyMs = min(flapScore *
|
|
15
|
+
* flapPenaltyMs = min(flapScore * DEFAULT_FLAP_PENALTY_MS_PER_DROP, DEFAULT_FLAP_PENALTY_MAX_MS)
|
|
16
16
|
*
|
|
17
|
-
* A node with no recent drops ranks on its ping alone. One drop costs
|
|
18
|
-
*
|
|
19
|
-
* node that flaps once and then behaves is re-admitted
|
|
20
|
-
* that keeps flapping stays at the back of the list, and
|
|
21
|
-
* "reset" by a lucky health probe (the old
|
|
17
|
+
* A node with no recent drops ranks on its ping alone. One undecayed drop costs
|
|
18
|
+
* DEFAULT_FLAP_PENALTY_MS_PER_DROP of effective latency and then fades with the
|
|
19
|
+
* half-life, so a node that flaps once and then behaves is re-admitted
|
|
20
|
+
* automatically, a node that keeps flapping stays at the back of the list, and
|
|
21
|
+
* nothing has to be "reset" by a lucky health probe (the old
|
|
22
22
|
* LIVE_FAILURE_HEALTH_SUCCESS_STREAK wipe-on-success counter).
|
|
23
23
|
*
|
|
24
24
|
* Everything here is pure: callers own persistence and logging. Drop history
|