@reefclaw/openclaw-plugin 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bridge/bridge.d.ts +20 -5
- package/bridge/bridge.js +29 -14
- package/bridge/config.js +6 -0
- package/bridge/gateway/gateway-config.d.ts +16 -5
- package/bridge/gateway/gateway-config.js +68 -12
- package/bridge/gateway/gateway-ws-client.d.ts +4 -1
- package/bridge/gateway/gateway-ws-client.js +41 -11
- package/bridge/gateway/poller.js +18 -8
- package/bridge/providers/emergency-commands.d.ts +9 -1
- package/bridge/providers/emergency-commands.js +38 -1
- package/bridge/providers/gateway.d.ts +51 -1
- package/bridge/providers/gateway.js +209 -22
- package/bridge/providers/onboarding-commands.d.ts +11 -0
- package/bridge/providers/onboarding-commands.js +5 -5
- package/bridge/providers/risk-calculator.d.ts +61 -2
- package/bridge/providers/risk-calculator.js +92 -20
- package/bridge/utils/skill-signing.js +8 -3
- package/ccxt/binance-public.d.ts +17 -5
- package/ccxt/binance-public.js +31 -3
- package/config/operator-provenance.d.ts +6 -0
- package/config/operator-provenance.js +50 -0
- package/config/plugin-config-io.d.ts +15 -1
- package/config/plugin-config-io.js +29 -0
- package/exchange-adapter.d.ts +13 -0
- package/index.js +230 -176
- package/ingest/event-loop-monitor.d.ts +22 -0
- package/ingest/event-loop-monitor.js +190 -0
- package/ingest/position-auto-capture.d.ts +5 -0
- package/ingest/position-auto-capture.js +14 -5
- package/ingest/readiness-reporter.d.ts +26 -6
- package/ingest/readiness-reporter.js +137 -9
- package/ingest/skill-version-reader.d.ts +16 -0
- package/ingest/skill-version-reader.js +64 -0
- package/live/approval-lifecycle.d.ts +30 -0
- package/live/approval-lifecycle.js +80 -0
- package/live/bracket-types.d.ts +9 -0
- package/live/live-adapter.d.ts +0 -1
- package/live/user-data-stream.js +10 -2
- package/onboarding/runtime.d.ts +34 -1
- package/onboarding/runtime.js +56 -5
- package/openclaw.plugin.json +1 -1
- package/package.json +6 -5
- package/risk/pre-trade-check.js +18 -5
- package/simulator/exchange-simulator.d.ts +45 -2
- package/simulator/exchange-simulator.js +96 -4
- package/simulator/types.d.ts +17 -0
- package/skills/reefclaw/SKILL.md +6 -11
- package/strategy/condition-registry.js +9 -2
- package/strategy/evaluator.d.ts +5 -0
- package/tools/attach-brackets.js +50 -1
- package/tools/cancel-all-orders.js +9 -1
- package/tools/create-order.js +18 -1
- package/tools/get-bracket-config.d.ts +21 -2
- package/tools/get-bracket-config.js +18 -2
- package/tools/set-trading-mode.js +6 -3
- package/venues/hyperliquid/hl-bracket-coordinator.d.ts +25 -1
- package/venues/hyperliquid/hl-bracket-coordinator.js +57 -0
- package/venues/hyperliquid/hl-brackets.d.ts +10 -0
- package/venues/hyperliquid/hl-brackets.js +45 -13
- package/venues/hyperliquid/hl-fill-ingest.d.ts +18 -0
- package/venues/hyperliquid/hl-fill-ingest.js +88 -0
- package/venues/hyperliquid/hl-live-adapter.d.ts +36 -0
- package/venues/hyperliquid/hl-live-adapter.js +116 -7
- package/venues/hyperliquid/hl-public.d.ts +12 -5
- package/venues/hyperliquid/hl-public.js +24 -3
- package/venues/hyperliquid/hl-user-stream.d.ts +13 -1
- package/venues/hyperliquid/hl-user-stream.js +4 -1
- package/venues/registry.js +8 -7
- package/wave9/paper-admission-guard.d.ts +12 -1
- package/wave9/paper-admission-guard.js +12 -1
- package/scripts/assemble.mjs +0 -130
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// Event-loop delay monitor — the signal that tells "this process was frozen"
|
|
2
|
+
// apart from "the venue was unreachable" (issue #265).
|
|
3
|
+
//
|
|
4
|
+
// Why this exists: the readiness probe wraps its request in a 10s abort timer.
|
|
5
|
+
// When the HOST starves the process (CPU contention, a neighbouring container
|
|
6
|
+
// leaking into swap), the loop stops running — the request never gets a fair
|
|
7
|
+
// chance and the abort fires late. The old code fell into one catch and blamed
|
|
8
|
+
// the venue, producing a false "cannot reach the API — check DNS/firewall"
|
|
9
|
+
// banner. Live evidence on the HL rig: a 10s timer that fired 86s late, with
|
|
10
|
+
// OpenClaw's own diagnostic recording loop delays up to 556s at 2-17% of one
|
|
11
|
+
// core (loop runnable, never scheduled).
|
|
12
|
+
//
|
|
13
|
+
// perf_hooks.monitorEventLoopDelay samples libuv timer lag into an
|
|
14
|
+
// IntervalHistogram. Its internal timer does NOT hold the process open, and
|
|
15
|
+
// every entry point is wrapped so a failure here can never break readiness
|
|
16
|
+
// reporting — an unavailable monitor reports `null` (⇒ 'unknown'), never a
|
|
17
|
+
// fabricated zero.
|
|
18
|
+
// ★ Static import, NOT require(): this package is ESM ("type":"module" +
|
|
19
|
+
// module:ES2022), so tsc emits a bare `require` verbatim and Node throws
|
|
20
|
+
// ReferenceError on it. The throw landed in the catch below, which set
|
|
21
|
+
// initFailed and disabled the monitor FOREVER — the check would have reported
|
|
22
|
+
// 'unknown' on every cycle in production while every test passed (vitest's
|
|
23
|
+
// CJS interop defines `require`). node:perf_hooks is a builtin with no
|
|
24
|
+
// CJS-shape problem, so there is nothing here to defer: the ccxt-style
|
|
25
|
+
// createRequire dance exists for that dependency's module shape, not for
|
|
26
|
+
// builtins. See docs/CLAUDE/plugin-integration.md.
|
|
27
|
+
import { monitorEventLoopDelay } from 'node:perf_hooks';
|
|
28
|
+
import { readFileSync } from 'node:fs';
|
|
29
|
+
import { logger } from '../logger.js';
|
|
30
|
+
const TAG = 'event-loop-monitor';
|
|
31
|
+
/** Sampling resolution. 20ms is fine-grained enough to catch a real stall
|
|
32
|
+
* without meaningful overhead — we only ever read the MAX. */
|
|
33
|
+
const RESOLUTION_MS = 20;
|
|
34
|
+
/** How long after enabling before a reading means anything.
|
|
35
|
+
*
|
|
36
|
+
* ★ The histogram is BLIND to a block that happens in the same tick as
|
|
37
|
+
* `enable()`: Node baselines it at its first internal timer fire, not at the
|
|
38
|
+
* enable call, so a stall that predates that fire is never attributed.
|
|
39
|
+
* Measured (node 22): identical 300ms block reads 30ms when it runs in the
|
|
40
|
+
* enable tick vs 300ms once the loop has turned. The floor it reports instead
|
|
41
|
+
* (~30ms on Windows, the timer-coalescing noise) looks exactly like a healthy
|
|
42
|
+
* host — a `pass` from a measurement that CANNOT fail, which is the same
|
|
43
|
+
* false-green this check exists to kill. So we withhold judgement until the
|
|
44
|
+
* histogram's own timer has fired: this delay is > 2× RESOLUTION_MS, and a
|
|
45
|
+
* starved loop only pushes it later, never earlier. */
|
|
46
|
+
const ARM_DELAY_MS = 50;
|
|
47
|
+
let histogram = null;
|
|
48
|
+
let initFailed = false;
|
|
49
|
+
/** False until the histogram can actually attribute a stall (see ARM_DELAY_MS).
|
|
50
|
+
* While false, sampling reports `null` ⇒ 'unknown' rather than the floor. */
|
|
51
|
+
let armed = false;
|
|
52
|
+
/** Lazily create + enable the histogram. Returns null if the runtime refuses
|
|
53
|
+
* one (never throws — readiness must survive a missing monitor). */
|
|
54
|
+
function getHistogram() {
|
|
55
|
+
if (histogram || initFailed)
|
|
56
|
+
return histogram;
|
|
57
|
+
try {
|
|
58
|
+
const h = monitorEventLoopDelay({ resolution: RESOLUTION_MS });
|
|
59
|
+
h.enable();
|
|
60
|
+
histogram = h;
|
|
61
|
+
// unref'd: arming must never be a reason the process stays alive.
|
|
62
|
+
const t = setTimeout(() => {
|
|
63
|
+
armed = true;
|
|
64
|
+
}, ARM_DELAY_MS);
|
|
65
|
+
t.unref?.();
|
|
66
|
+
return histogram;
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
initFailed = true;
|
|
70
|
+
logger.warn(TAG, `event-loop monitor unavailable (host-responsiveness check will report unknown): ${err instanceof Error ? err.message : String(err)}`);
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
/** Start sampling. Safe to call repeatedly (idempotent). */
|
|
75
|
+
export function startEventLoopMonitor() {
|
|
76
|
+
getHistogram();
|
|
77
|
+
}
|
|
78
|
+
/** Worst event-loop delay (ms) observed since the previous call, then reset —
|
|
79
|
+
* so consecutive calls partition the timeline into non-overlapping windows.
|
|
80
|
+
* Returns null when the monitor is unavailable or has no sample yet.
|
|
81
|
+
*
|
|
82
|
+
* The histogram stores nanoseconds; `max` is Infinity-safe but can read as a
|
|
83
|
+
* sentinel before the first sample lands, so anything non-finite → null. */
|
|
84
|
+
export function sampleEventLoopDelayMs() {
|
|
85
|
+
const h = getHistogram();
|
|
86
|
+
if (!h)
|
|
87
|
+
return null;
|
|
88
|
+
// Not yet able to measure — report 'unknown' rather than the idle floor, which
|
|
89
|
+
// would read as a healthy host on a boot cycle that never observed anything.
|
|
90
|
+
if (!armed)
|
|
91
|
+
return null;
|
|
92
|
+
try {
|
|
93
|
+
const maxNs = Number(h.max);
|
|
94
|
+
h.reset();
|
|
95
|
+
if (!Number.isFinite(maxNs) || maxNs < 0)
|
|
96
|
+
return null;
|
|
97
|
+
return maxNs / 1e6;
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
104
|
+
// Runqueue wait — the measurement that says WHO froze the loop.
|
|
105
|
+
//
|
|
106
|
+
// ★ Event-loop delay is a symptom with two very different causes: the host
|
|
107
|
+
// descheduled us (co-tenant load, an oversubscribed VPS) or we blocked our own
|
|
108
|
+
// loop (a synchronous burst — on a ReefClaw box, an LLM turn, which runs
|
|
109
|
+
// EMBEDDED in this same process alongside order handling). The histogram above
|
|
110
|
+
// cannot tell them apart, so the check used to assert the first and tell the
|
|
111
|
+
// operator to buy more CPU. Verified wrong on prod 2026-07-29: a "froze 5.9s"
|
|
112
|
+
// warn on a box at 0.05 load, 0 steal, 0 swap, whose cumulative runqueue wait
|
|
113
|
+
// was 1.95s across the whole process lifetime. Resizing would have done nothing.
|
|
114
|
+
//
|
|
115
|
+
// /proc/self/schedstat field 2 is the kernel's own count of nanoseconds this
|
|
116
|
+
// thread spent RUNNABLE but waiting for a CPU. It is exactly the discriminator:
|
|
117
|
+
// large ⇒ the host really is starving us; ~0 while the loop froze ⇒ the freeze
|
|
118
|
+
// came from inside. `/proc/self` is the thread-group leader = the main thread,
|
|
119
|
+
// which is where the event loop runs, so this is the right thread to ask.
|
|
120
|
+
//
|
|
121
|
+
// Same evidence rule as everything else on this path (issue #265): unavailable
|
|
122
|
+
// ⇒ null ⇒ 'unknown' attribution and the cautious copy — never a fabricated
|
|
123
|
+
// zero, which would read as "definitely self-inflicted" and is the same
|
|
124
|
+
// false-confidence bug pointed the other way.
|
|
125
|
+
const SCHEDSTAT_PATH = '/proc/self/schedstat';
|
|
126
|
+
/** Cumulative ns at the previous sample; null until the first read establishes
|
|
127
|
+
* a baseline. A delta needs two points — the first call can only arm. */
|
|
128
|
+
let lastRunqueueWaitNs = null;
|
|
129
|
+
/** Latched after the first failed read: not Linux, or a kernel built without
|
|
130
|
+
* CONFIG_SCHEDSTATS. Stops us re-reading a file that will never exist. */
|
|
131
|
+
let schedstatUnavailable = false;
|
|
132
|
+
/** Read the cumulative runqueue-wait counter (ns). Null when unreadable or
|
|
133
|
+
* malformed — never a guess. */
|
|
134
|
+
function readRunqueueWaitNs() {
|
|
135
|
+
if (schedstatUnavailable)
|
|
136
|
+
return null;
|
|
137
|
+
try {
|
|
138
|
+
// Three space-separated numbers: cpu_time_ns, runqueue_wait_ns, timeslices.
|
|
139
|
+
const parts = readFileSync(SCHEDSTAT_PATH, 'utf8').trim().split(/\s+/);
|
|
140
|
+
const ns = Number(parts[1]);
|
|
141
|
+
if (!Number.isFinite(ns) || ns < 0) {
|
|
142
|
+
schedstatUnavailable = true;
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
return ns;
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
schedstatUnavailable = true;
|
|
149
|
+
logger.info(TAG, 'runqueue-wait unavailable (not Linux, or kernel without CONFIG_SCHEDSTATS) — host-stall attribution will report unknown');
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/** Time (ms) this process spent waiting for a CPU since the previous call, then
|
|
154
|
+
* re-baselines — so consecutive calls partition the timeline into the SAME
|
|
155
|
+
* non-overlapping windows as `sampleEventLoopDelayMs`, and the two readings of
|
|
156
|
+
* one cycle describe one window.
|
|
157
|
+
*
|
|
158
|
+
* ★ Call this EVERY cycle, not only when a stall was observed: the counter is
|
|
159
|
+
* cumulative since process start, so a first read taken at the moment of a
|
|
160
|
+
* freeze would report hours of ordinary scheduling as if it were the freeze.
|
|
161
|
+
*
|
|
162
|
+
* Returns null when unavailable or on the first (baseline-establishing) call. */
|
|
163
|
+
export function sampleRunqueueWaitMs() {
|
|
164
|
+
const ns = readRunqueueWaitNs();
|
|
165
|
+
if (ns == null)
|
|
166
|
+
return null;
|
|
167
|
+
const prev = lastRunqueueWaitNs;
|
|
168
|
+
lastRunqueueWaitNs = ns;
|
|
169
|
+
if (prev == null)
|
|
170
|
+
return null;
|
|
171
|
+
// Counters only climb; a decrease means we are not reading what we think we
|
|
172
|
+
// are (or it wrapped). Report unknown rather than a negative/garbage window.
|
|
173
|
+
if (ns < prev)
|
|
174
|
+
return null;
|
|
175
|
+
return (ns - prev) / 1e6;
|
|
176
|
+
}
|
|
177
|
+
/** Test-only — drop the singleton so a fresh histogram is created. */
|
|
178
|
+
export function __resetEventLoopMonitorForTests() {
|
|
179
|
+
try {
|
|
180
|
+
histogram?.disable();
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
/* ignore */
|
|
184
|
+
}
|
|
185
|
+
histogram = null;
|
|
186
|
+
initFailed = false;
|
|
187
|
+
armed = false;
|
|
188
|
+
lastRunqueueWaitNs = null;
|
|
189
|
+
schedstatUnavailable = false;
|
|
190
|
+
}
|
|
@@ -105,6 +105,11 @@ export interface StopWatcherCloseInputs {
|
|
|
105
105
|
symbol: string;
|
|
106
106
|
stopPrice: number;
|
|
107
107
|
markPrice: number;
|
|
108
|
+
/** Which protective leg fired. Defaults to the stop (the historical caller).
|
|
109
|
+
* 'exchange_target' is the paper take-profit leg — the same journal path,
|
|
110
|
+
* because a target close bypasses close_position exactly like a stop does
|
|
111
|
+
* and would otherwise leave a phantom-open row (issue #199). */
|
|
112
|
+
leg?: 'stop_watcher' | 'exchange_target';
|
|
108
113
|
/** The executed close order from adapter.closePosition (may be absent if the
|
|
109
114
|
* close resolved through a path that didn't surface it). */
|
|
110
115
|
order?: CcxtOrder;
|
|
@@ -22,6 +22,7 @@ import { logger } from '../logger.js';
|
|
|
22
22
|
import { isBracketCid } from '../live/bracket-id.js';
|
|
23
23
|
import { normalizeBracketSymbol } from '../live/bracket-ledger.js';
|
|
24
24
|
import { fillPriceFromOrder } from '../live/fill-price.js';
|
|
25
|
+
import { getSkillVersionCached, withSkillVersion } from './skill-version-reader.js';
|
|
25
26
|
const TAG = 'position-auto-capture';
|
|
26
27
|
/** Flatness tolerance for remaining-contracts tracking. Reduce-only fills sum
|
|
27
28
|
* exactly to the position size on Binance, so any residual below this is noise
|
|
@@ -428,14 +429,19 @@ export async function onStopWatcherClose(ctx, inputs) {
|
|
|
428
429
|
dropState();
|
|
429
430
|
return;
|
|
430
431
|
}
|
|
432
|
+
const leg = inputs.leg ?? 'stop_watcher';
|
|
431
433
|
const close = {
|
|
432
434
|
positionId: stateEntry.webappPositionId,
|
|
433
435
|
closeAt: closeAtMs,
|
|
434
|
-
closeReason:
|
|
436
|
+
closeReason: leg,
|
|
435
437
|
closeAssessment: {
|
|
436
|
-
note:
|
|
437
|
-
'
|
|
438
|
-
|
|
438
|
+
note: leg === 'exchange_target'
|
|
439
|
+
? 'Position auto-closed by the paper take-profit leg: price reached the pinned targetPrice. ' +
|
|
440
|
+
'The live analog is an exchange-native TAKE_PROFIT_MARKET fill. Journaled from the close ' +
|
|
441
|
+
'fill — no close_position call.'
|
|
442
|
+
: 'Position auto-closed by the stop-watcher: mark crossed the pinned stopPrice. ' +
|
|
443
|
+
'Journaled from the watcher close fill — no close_position call (issue #199).',
|
|
444
|
+
observedFrom: leg,
|
|
439
445
|
stop_price: inputs.stopPrice,
|
|
440
446
|
mark_price: inputs.markPrice,
|
|
441
447
|
...(paperTrade
|
|
@@ -669,7 +675,10 @@ function buildEntryPayload(args) {
|
|
|
669
675
|
fillPrice: args.fillPrice,
|
|
670
676
|
fillSize: args.fillSize,
|
|
671
677
|
exchangeTradeId: args.exchangeTradeId,
|
|
672
|
-
|
|
678
|
+
// skill_version stamp: makes every SKILL.md release measurable against
|
|
679
|
+
// outcomes (capture ratio / scratch rate per doctrine version). Fail-soft —
|
|
680
|
+
// unreadable/absent SKILL.md just omits the key.
|
|
681
|
+
metadata: withSkillVersion(buildEntryPlanMetadata(md), getSkillVersionCached()),
|
|
673
682
|
};
|
|
674
683
|
}
|
|
675
684
|
/** Pinned-plan subset persisted to position_entries.metadata (free-form JSONB).
|
|
@@ -1,12 +1,23 @@
|
|
|
1
|
-
import { type ReadinessReport, type VenueId } from '@reefclaw/shared';
|
|
1
|
+
import { type ReadinessReport, type VenueId, type VenueReachabilityResult } from '@reefclaw/shared';
|
|
2
|
+
/** Who froze the loop. 'unknown' when the kernel counter is unreadable (not
|
|
3
|
+
* Linux / no CONFIG_SCHEDSTATS / first cycle) — attribution is evidence, and
|
|
4
|
+
* absent evidence stays absent rather than defaulting to a blame. */
|
|
5
|
+
export type StallAttribution = 'host' | 'self' | 'unknown';
|
|
6
|
+
/** Compare the freeze against the CPU the host actually denied us. Exported for
|
|
7
|
+
* unit tests — this is the whole discrimination rule in one place. */
|
|
8
|
+
export declare function attributeStall(stallMs: number, runqueueWaitMs: number | null): StallAttribution;
|
|
2
9
|
/** Venue-agnostic reachability probe — BinancePublicApi.probeReachability and
|
|
3
10
|
* HyperliquidPublicApi.probeReachability both return exactly this shape. */
|
|
4
11
|
export interface VenueReachabilityProbe {
|
|
5
|
-
probeReachability(): Promise<
|
|
6
|
-
outcome: 'reachable' | 'geo_blocked' | 'unreachable' | 'unknown';
|
|
7
|
-
driftMs: number | null;
|
|
8
|
-
}>;
|
|
12
|
+
probeReachability(): Promise<VenueReachabilityResult>;
|
|
9
13
|
}
|
|
14
|
+
/** Cross-cycle memory for the debounced rungs. Held by the interval loop and
|
|
15
|
+
* passed in explicitly so `collectReadiness` stays a pure function of its
|
|
16
|
+
* inputs — a module-global counter would leak between unit tests. */
|
|
17
|
+
export interface ReadinessCycleState {
|
|
18
|
+
consecutiveReachFailures: number;
|
|
19
|
+
}
|
|
20
|
+
export declare function createReadinessCycleState(): ReadinessCycleState;
|
|
10
21
|
export interface ReadinessReporterOptions {
|
|
11
22
|
apiBaseUrl: string;
|
|
12
23
|
token: string;
|
|
@@ -35,7 +46,16 @@ export interface ReadinessReporterOptions {
|
|
|
35
46
|
* warn/fail drift is reported 'unknown' (not amber/red) so the readiness
|
|
36
47
|
* banner doesn't cry-wolf for ~5 min after every restart; a genuinely
|
|
37
48
|
* skewed clock still surfaces on cycle 2. */
|
|
38
|
-
export declare function collectReadiness(opts: Pick<ReadinessReporterOptions, 'venue' | 'publicApi' | 'toolCount'>, bootWarmup?: boolean
|
|
49
|
+
export declare function collectReadiness(opts: Pick<ReadinessReporterOptions, 'venue' | 'publicApi' | 'toolCount'>, bootWarmup?: boolean, deps?: {
|
|
50
|
+
/** Debounce memory. Omitted → a fresh state, so a lone unreachable reads
|
|
51
|
+
* `unknown`; only a caller that persists state across cycles can ever
|
|
52
|
+
* reach the warn rung. */
|
|
53
|
+
state?: ReadinessCycleState;
|
|
54
|
+
/** Injected for tests; production reads the real event-loop histogram. */
|
|
55
|
+
sampleHostStallMs?: () => number | null;
|
|
56
|
+
/** Injected for tests; production reads /proc/self/schedstat. */
|
|
57
|
+
sampleRunqueueWaitMs?: () => number | null;
|
|
58
|
+
}): Promise<ReadinessReport>;
|
|
39
59
|
/** Test-only — reset the singleton guard between unit tests. */
|
|
40
60
|
export declare function __resetReadinessReporterForTests(): void;
|
|
41
61
|
/** Fire the readiness report once at boot then on an unref'd interval. */
|
|
@@ -8,13 +8,44 @@
|
|
|
8
8
|
// module-level singleton guard mirrors `pluginInitialised` in index.ts — OpenClaw
|
|
9
9
|
// calls register() multiple times per process and we must never spawn a second
|
|
10
10
|
// timer. The interval is unref()'d so it never holds the process open.
|
|
11
|
-
import { makeReadinessCheck, deriveOverallReadiness, } from '@reefclaw/shared';
|
|
11
|
+
import { makeReadinessCheck, deriveOverallReadiness, GEO_BLOCK_FIX_HINT, HOST_STALL_FIX_HINT, } from '@reefclaw/shared';
|
|
12
12
|
import { logger, formatError } from '../logger.js';
|
|
13
|
+
import { startEventLoopMonitor, sampleEventLoopDelayMs, sampleRunqueueWaitMs, } from './event-loop-monitor.js';
|
|
13
14
|
const TAG = 'readiness';
|
|
14
15
|
const DEFAULT_INTERVAL_MS = 300_000; // 5 min — geo/clock state changes rarely.
|
|
15
16
|
const MIN_INTERVAL_MS = 60_000;
|
|
16
17
|
/** Best-effort display fact; kept in sync with the register() banner in index.ts. */
|
|
17
18
|
const PLUGIN_VERSION = '3.8.0';
|
|
19
|
+
/** Consecutive non-pass reachability probes required before the banner goes
|
|
20
|
+
* amber. One 5-min sample is not enough evidence to send an operator hunting a
|
|
21
|
+
* network fault — the intel-health AMBER rung already debounces the same way
|
|
22
|
+
* (docs/CLAUDE/intel-health.md), readiness had no equivalent (issue #265). */
|
|
23
|
+
const REACH_WARN_AFTER_FAILURES = 2;
|
|
24
|
+
/** Worst event-loop delay in a cycle that still counts as a responsive host.
|
|
25
|
+
* Well above ordinary GC/boot jitter (tens to low hundreds of ms) and far
|
|
26
|
+
* below the freezes worth alarming on — the live incident produced 89s-556s. */
|
|
27
|
+
const HOST_STALL_WARN_MS = 5_000;
|
|
28
|
+
/** How much of a freeze the kernel must attribute to runqueue wait before we
|
|
29
|
+
* blame the HOST for it. Half is deliberately generous to the host-starved
|
|
30
|
+
* reading: a host that froze us for N seconds necessarily made us wait most of
|
|
31
|
+
* those N seconds, while a self-blocked loop accrues ~nothing (prod: 1.95s
|
|
32
|
+
* across an entire process lifetime, against a 5.9s freeze in one cycle). */
|
|
33
|
+
const HOST_WAIT_SHARE_OF_STALL = 0.5;
|
|
34
|
+
/** Absolute floor under the share test. Sub-second waiting is ordinary
|
|
35
|
+
* scheduling on any box and must never read as starvation, however short the
|
|
36
|
+
* freeze it is compared against. */
|
|
37
|
+
const HOST_WAIT_FLOOR_MS = 1_000;
|
|
38
|
+
/** Compare the freeze against the CPU the host actually denied us. Exported for
|
|
39
|
+
* unit tests — this is the whole discrimination rule in one place. */
|
|
40
|
+
export function attributeStall(stallMs, runqueueWaitMs) {
|
|
41
|
+
if (runqueueWaitMs == null)
|
|
42
|
+
return 'unknown';
|
|
43
|
+
const hostThreshold = Math.max(HOST_WAIT_FLOOR_MS, stallMs * HOST_WAIT_SHARE_OF_STALL);
|
|
44
|
+
return runqueueWaitMs >= hostThreshold ? 'host' : 'self';
|
|
45
|
+
}
|
|
46
|
+
export function createReadinessCycleState() {
|
|
47
|
+
return { consecutiveReachFailures: 0 };
|
|
48
|
+
}
|
|
18
49
|
function resolveIntervalMs(explicit) {
|
|
19
50
|
if (explicit && explicit > 0)
|
|
20
51
|
return Math.max(MIN_INTERVAL_MS, explicit);
|
|
@@ -34,9 +65,12 @@ function resolveIntervalMs(explicit) {
|
|
|
34
65
|
* warn/fail drift is reported 'unknown' (not amber/red) so the readiness
|
|
35
66
|
* banner doesn't cry-wolf for ~5 min after every restart; a genuinely
|
|
36
67
|
* skewed clock still surfaces on cycle 2. */
|
|
37
|
-
export async function collectReadiness(opts, bootWarmup = false) {
|
|
68
|
+
export async function collectReadiness(opts, bootWarmup = false, deps = {}) {
|
|
38
69
|
const now = Date.now();
|
|
39
70
|
const checks = [];
|
|
71
|
+
const state = deps.state ?? createReadinessCycleState();
|
|
72
|
+
const sampleStall = deps.sampleHostStallMs ?? sampleEventLoopDelayMs;
|
|
73
|
+
const sampleWait = deps.sampleRunqueueWaitMs ?? sampleRunqueueWaitMs;
|
|
40
74
|
// The venue decides which reachability check this report carries; the copy
|
|
41
75
|
// for both ids lives in shared/src/readiness.ts. Clock drift comes from the
|
|
42
76
|
// same probe on both venues (Binance fapi serverTime / Hyperliquid
|
|
@@ -50,9 +84,71 @@ export async function collectReadiness(opts, bootWarmup = false) {
|
|
|
50
84
|
detail: `${opts.toolCount} tools`,
|
|
51
85
|
checkedAt: now,
|
|
52
86
|
}));
|
|
53
|
-
//
|
|
87
|
+
// ★ Probe FIRST, sample the loop delay AFTER. The freeze that makes a probe
|
|
88
|
+
// abort happens *during* the probe, so sampling first would file the evidence
|
|
89
|
+
// in the NEXT cycle's window — reachability would read 'stalled' this cycle
|
|
90
|
+
// while host_responsive still read 'pass', putting the two halves of one
|
|
91
|
+
// incident 5 minutes apart. The checks are still pushed in display order.
|
|
54
92
|
const probe = await opts.publicApi.probeReachability();
|
|
93
|
+
// host_responsive — the signal that actually matters on a trading rig, and the
|
|
94
|
+
// one whose absence let a starved host masquerade as an unreachable venue.
|
|
95
|
+
// A null sample means the monitor is unavailable or has no window yet: report
|
|
96
|
+
// 'unknown', never a fabricated healthy zero — except that a probe which
|
|
97
|
+
// measured its own overshoot IS a stall measurement, so it stands in.
|
|
98
|
+
const stallMs = sampleStall() ?? (probe.outcome === 'stalled' ? (probe.stallMs ?? null) : null);
|
|
99
|
+
// ★ Sampled unconditionally, even on a passing cycle: the counter behind it is
|
|
100
|
+
// cumulative, so it must be re-baselined every cycle for its window to line up
|
|
101
|
+
// with the freeze window. Reading it only when a stall appeared would charge
|
|
102
|
+
// that one freeze with every millisecond of ordinary scheduling since boot and
|
|
103
|
+
// score a self-blocked loop as a starved host.
|
|
104
|
+
const runqueueWaitMs = sampleWait();
|
|
105
|
+
if (stallMs == null) {
|
|
106
|
+
checks.push(makeReadinessCheck('host_responsive', 'unknown', { checkedAt: now }));
|
|
107
|
+
}
|
|
108
|
+
else {
|
|
109
|
+
const rawStatus = stallMs < HOST_STALL_WARN_MS ? 'pass' : 'warn';
|
|
110
|
+
// Boot congestion (WS start, snapshot, seed all racing) legitimately stalls
|
|
111
|
+
// the loop for seconds — same warm-up rule as the clock check below.
|
|
112
|
+
const status = bootWarmup && rawStatus !== 'pass' ? 'unknown' : rawStatus;
|
|
113
|
+
// WHO froze it — computed only for a real freeze, since the whole question
|
|
114
|
+
// is what to tell the operator to go fix.
|
|
115
|
+
const attribution = rawStatus === 'pass' ? 'unknown' : attributeStall(stallMs, runqueueWaitMs);
|
|
116
|
+
const frozeSecs = (stallMs / 1000).toFixed(1);
|
|
117
|
+
const waitedSecs = runqueueWaitMs != null ? (runqueueWaitMs / 1000).toFixed(1) : null;
|
|
118
|
+
// The detail line carries the evidence, not just the verdict: an operator
|
|
119
|
+
// who disagrees with the attribution can see the number it rests on.
|
|
120
|
+
const stallDetail = attribution === 'host'
|
|
121
|
+
? `froze ${frozeSecs}s (waited ${waitedSecs}s for CPU)`
|
|
122
|
+
: attribution === 'self'
|
|
123
|
+
? `froze ${frozeSecs}s (host had CPU free — blocked inside the agent process)`
|
|
124
|
+
: `froze ${frozeSecs}s`;
|
|
125
|
+
checks.push(makeReadinessCheck('host_responsive', status, {
|
|
126
|
+
detail: rawStatus === 'pass'
|
|
127
|
+
? `max loop delay ${Math.round(stallMs)}ms`
|
|
128
|
+
: bootWarmup
|
|
129
|
+
? `froze ${frozeSecs}s (boot warm-up — rechecking)`
|
|
130
|
+
: stallDetail,
|
|
131
|
+
// Only an attributed stall may name its cause; 'unknown' falls through to
|
|
132
|
+
// the cautious default in the copy map (issue #265 rule).
|
|
133
|
+
...(attribution === 'host'
|
|
134
|
+
? { fixHint: HOST_STALL_FIX_HINT.host_starved }
|
|
135
|
+
: attribution === 'self'
|
|
136
|
+
? { fixHint: HOST_STALL_FIX_HINT.self_inflicted }
|
|
137
|
+
: {}),
|
|
138
|
+
checkedAt: now,
|
|
139
|
+
}));
|
|
140
|
+
if (rawStatus !== 'pass' && !bootWarmup) {
|
|
141
|
+
const cause = attribution === 'host'
|
|
142
|
+
? `host starved: waited ${waitedSecs}s for CPU`
|
|
143
|
+
: attribution === 'self'
|
|
144
|
+
? `blocked inside this process (host had CPU free — waited only ${waitedSecs}s)`
|
|
145
|
+
: 'cause unattributable (runqueue wait unreadable)';
|
|
146
|
+
logger.warn(TAG, `event loop froze ${frozeSecs}s this cycle — order handling and bracket resync can lag by that much; ${cause}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// venue reachability (+ clock drift from the same probe response)
|
|
55
150
|
if (probe.outcome === 'reachable') {
|
|
151
|
+
state.consecutiveReachFailures = 0;
|
|
56
152
|
checks.push(makeReadinessCheck(reachId, 'pass', { checkedAt: now }));
|
|
57
153
|
const drift = probe.driftMs;
|
|
58
154
|
if (drift == null) {
|
|
@@ -74,21 +170,47 @@ export async function collectReadiness(opts, bootWarmup = false) {
|
|
|
74
170
|
}
|
|
75
171
|
}
|
|
76
172
|
else if (probe.outcome === 'geo_blocked') {
|
|
173
|
+
// The venue ANSWERED (451/403) — so the host reached it, and this is the one
|
|
174
|
+
// reachability verdict we may state as a cause. Clears the debounce counter
|
|
175
|
+
// and carries the assertive copy rather than the cautious default.
|
|
176
|
+
state.consecutiveReachFailures = 0;
|
|
77
177
|
checks.push(makeReadinessCheck(reachId, 'fail', {
|
|
78
178
|
detail: opts.venue === 'binance' ? 'HTTP 451' : 'blocked (HTTP 451/403)',
|
|
179
|
+
fixHint: GEO_BLOCK_FIX_HINT[opts.venue === 'binance' ? 'binance' : 'hyperliquid'],
|
|
180
|
+
checkedAt: now,
|
|
181
|
+
}));
|
|
182
|
+
checks.push(makeReadinessCheck('clock_in_sync', 'unknown', { checkedAt: now }));
|
|
183
|
+
}
|
|
184
|
+
else if (probe.outcome === 'stalled') {
|
|
185
|
+
// ★ Our own process was starved, so the probe proves nothing about the venue
|
|
186
|
+
// (issue #265). Report 'unknown' — which never turns the banner amber — and
|
|
187
|
+
// leave the debounce counter untouched: a non-observation is not evidence
|
|
188
|
+
// either way. host_responsive above carries the signal that IS actionable.
|
|
189
|
+
const secs = probe.stallMs != null ? (probe.stallMs / 1000).toFixed(1) : '?';
|
|
190
|
+
checks.push(makeReadinessCheck(reachId, 'unknown', {
|
|
191
|
+
detail: `not measurable — agent host froze ${secs}s`,
|
|
79
192
|
checkedAt: now,
|
|
80
193
|
}));
|
|
81
194
|
checks.push(makeReadinessCheck('clock_in_sync', 'unknown', { checkedAt: now }));
|
|
82
195
|
}
|
|
83
196
|
else if (probe.outcome === 'unreachable') {
|
|
84
|
-
// Network/DNS/timeout
|
|
85
|
-
//
|
|
86
|
-
// re-checks; only the
|
|
87
|
-
|
|
197
|
+
// Network/DNS/timeout. Debounced: one 5-min sample is not enough to alarm,
|
|
198
|
+
// so the first failure reports 'unknown' and only a SECOND consecutive one
|
|
199
|
+
// goes amber. A persistent problem stays amber across re-checks; only the
|
|
200
|
+
// geo-block is a hard red.
|
|
201
|
+
state.consecutiveReachFailures += 1;
|
|
202
|
+
const confirmed = state.consecutiveReachFailures >= REACH_WARN_AFTER_FAILURES;
|
|
203
|
+
checks.push(makeReadinessCheck(reachId, confirmed ? 'warn' : 'unknown', {
|
|
204
|
+
detail: confirmed
|
|
205
|
+
? `unreachable (${state.consecutiveReachFailures} consecutive checks)`
|
|
206
|
+
: 'one failed check — rechecking',
|
|
207
|
+
checkedAt: now,
|
|
208
|
+
}));
|
|
88
209
|
checks.push(makeReadinessCheck('clock_in_sync', 'unknown', { checkedAt: now }));
|
|
89
210
|
}
|
|
90
211
|
else {
|
|
91
|
-
// 'unknown' — the ban/weight gate paused the probe; don't assert anything
|
|
212
|
+
// 'unknown' — the ban/weight gate paused the probe; don't assert anything,
|
|
213
|
+
// and don't let a non-observation move the debounce counter.
|
|
92
214
|
checks.push(makeReadinessCheck(reachId, 'unknown', { checkedAt: now }));
|
|
93
215
|
checks.push(makeReadinessCheck('clock_in_sync', 'unknown', { checkedAt: now }));
|
|
94
216
|
}
|
|
@@ -142,9 +264,15 @@ export function startReadinessReporter(opts) {
|
|
|
142
264
|
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
143
265
|
const intervalMs = resolveIntervalMs(opts.intervalMs);
|
|
144
266
|
const timeoutMs = opts.requestTimeoutMs ?? 10_000;
|
|
267
|
+
// Start sampling loop delay now so the first cycle measures a real window.
|
|
268
|
+
// Never throws — an unavailable monitor just reports host_responsive unknown.
|
|
269
|
+
startEventLoopMonitor();
|
|
270
|
+
// One state object for the process: the debounce rungs are only meaningful
|
|
271
|
+
// across cycles.
|
|
272
|
+
const state = createReadinessCycleState();
|
|
145
273
|
const cycle = async (bootWarmup) => {
|
|
146
274
|
try {
|
|
147
|
-
const report = await collectReadiness({ venue: opts.venue, publicApi: opts.publicApi, toolCount: opts.toolCount }, bootWarmup);
|
|
275
|
+
const report = await collectReadiness({ venue: opts.venue, publicApi: opts.publicApi, toolCount: opts.toolCount }, bootWarmup, { state });
|
|
148
276
|
await postReadiness(opts.apiBaseUrl, opts.token, report, fetchImpl, timeoutMs);
|
|
149
277
|
if (report.overall === 'fail') {
|
|
150
278
|
const failing = report.checks.filter((c) => c.status === 'fail').map((c) => c.id).join(', ');
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Pure parse — exported for tests. Only the first 4 KB is considered. */
|
|
2
|
+
export declare function parseSkillVersion(content: string): string | undefined;
|
|
3
|
+
/**
|
|
4
|
+
* Read the running SKILL.md version, cached until the file changes.
|
|
5
|
+
* Returns undefined when the file is absent, unreadable, or has no parseable
|
|
6
|
+
* frontmatter version — callers omit the stamp in that case.
|
|
7
|
+
*/
|
|
8
|
+
export declare function getSkillVersionCached(filePath?: string): string | undefined;
|
|
9
|
+
/**
|
|
10
|
+
* Merge the skill-version stamp into an entry-metadata object.
|
|
11
|
+
* Key is snake_case (`skill_version`) per the canonical-JSONB-key rule
|
|
12
|
+
* (CLAUDE.md learning-loop: a camelCase mismatch previously broke matching).
|
|
13
|
+
* No version → base returned untouched (possibly undefined), so rows never
|
|
14
|
+
* gain an empty/noise object just for a failed read.
|
|
15
|
+
*/
|
|
16
|
+
export declare function withSkillVersion(meta: Record<string, unknown> | undefined, version: string | undefined): Record<string, unknown> | undefined;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Best-effort reader for the SKILL.md version the agent is actually running.
|
|
2
|
+
//
|
|
3
|
+
// Purpose: stamp `skill_version` into position_entries.metadata so every
|
|
4
|
+
// doctrine (SKILL.md) release becomes measurable against trade outcomes —
|
|
5
|
+
// capture ratio / scratch rate per version — the same shadow→observe→enforce
|
|
6
|
+
// evidence discipline the repo applies to code gates, applied to the prose.
|
|
7
|
+
// Without this stamp a SKILL.md change is fleet-wide AND unmeasurable: journal
|
|
8
|
+
// rows from before and after the change are indistinguishable.
|
|
9
|
+
//
|
|
10
|
+
// Source of truth: ~/.openclaw/workspace/SKILL.md — the copy the bridge's
|
|
11
|
+
// applySkillUpdate maintains and the agent's OpenClaw loads (same path as
|
|
12
|
+
// skill/src/utils/skill-version.ts SKILL_MD_PATH; duplicated here because
|
|
13
|
+
// plugin and skill are separate deliverables with no shared package).
|
|
14
|
+
//
|
|
15
|
+
// Fail-soft by design: absent/unreadable/unparseable file → undefined → the
|
|
16
|
+
// metadata key is simply omitted. Journal capture must never block or throw
|
|
17
|
+
// on this read (same rule as the rest of the capture path).
|
|
18
|
+
import { readFileSync, statSync } from 'node:fs';
|
|
19
|
+
import { homedir } from 'node:os';
|
|
20
|
+
import { join } from 'node:path';
|
|
21
|
+
const DEFAULT_SKILL_MD_PATH = join(homedir(), '.openclaw', 'workspace', 'SKILL.md');
|
|
22
|
+
/** Frontmatter version line, e.g. `version: 2.20.11`. Anchored to the head of
|
|
23
|
+
* the file (frontmatter is lines 1-10) so a `version:` string later in the
|
|
24
|
+
* body can never shadow it. */
|
|
25
|
+
const VERSION_LINE = /^version:\s*(\d+\.\d+\.\d+)\s*$/m;
|
|
26
|
+
const HEAD_BYTES = 4096;
|
|
27
|
+
/** Pure parse — exported for tests. Only the first 4 KB is considered. */
|
|
28
|
+
export function parseSkillVersion(content) {
|
|
29
|
+
return VERSION_LINE.exec(content.slice(0, HEAD_BYTES))?.[1];
|
|
30
|
+
}
|
|
31
|
+
/** mtime+size-keyed cache per path. Entries are rare (a handful a day), but a
|
|
32
|
+
* cached hit avoids re-reading 110 KB on every fill during a busy burst. */
|
|
33
|
+
const cache = new Map();
|
|
34
|
+
/**
|
|
35
|
+
* Read the running SKILL.md version, cached until the file changes.
|
|
36
|
+
* Returns undefined when the file is absent, unreadable, or has no parseable
|
|
37
|
+
* frontmatter version — callers omit the stamp in that case.
|
|
38
|
+
*/
|
|
39
|
+
export function getSkillVersionCached(filePath = DEFAULT_SKILL_MD_PATH) {
|
|
40
|
+
try {
|
|
41
|
+
const st = statSync(filePath);
|
|
42
|
+
const hit = cache.get(filePath);
|
|
43
|
+
if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size)
|
|
44
|
+
return hit.v;
|
|
45
|
+
const v = parseSkillVersion(readFileSync(filePath, 'utf8'));
|
|
46
|
+
cache.set(filePath, { v, mtimeMs: st.mtimeMs, size: st.size });
|
|
47
|
+
return v;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Merge the skill-version stamp into an entry-metadata object.
|
|
55
|
+
* Key is snake_case (`skill_version`) per the canonical-JSONB-key rule
|
|
56
|
+
* (CLAUDE.md learning-loop: a camelCase mismatch previously broke matching).
|
|
57
|
+
* No version → base returned untouched (possibly undefined), so rows never
|
|
58
|
+
* gain an empty/noise object just for a failed read.
|
|
59
|
+
*/
|
|
60
|
+
export function withSkillVersion(meta, version) {
|
|
61
|
+
if (!version)
|
|
62
|
+
return meta;
|
|
63
|
+
return { ...(meta ?? {}), skill_version: version };
|
|
64
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { IExchangeAdapter } from '../exchange-adapter.js';
|
|
2
|
+
/** The slice of ProposalDecisionListener this class needs — injected so the
|
|
3
|
+
* lifecycle is testable without gateway credentials. */
|
|
4
|
+
export interface StartableListener {
|
|
5
|
+
start(): void;
|
|
6
|
+
stop(): Promise<void>;
|
|
7
|
+
}
|
|
8
|
+
export interface ApprovalLifecycleDeps {
|
|
9
|
+
/** Re-read config+env — NOT a boot snapshot; a config edit applies on the
|
|
10
|
+
* next swap without a restart. */
|
|
11
|
+
resolveApprovalMode: () => 'off' | 'shadow' | 'per_trade';
|
|
12
|
+
/** Ingest credentials + proposal manager present? Without them a listener
|
|
13
|
+
* cannot claim or report — refuse to start rather than half-run. */
|
|
14
|
+
hasWiring: () => boolean;
|
|
15
|
+
/** Build a listener bound to THIS adapter. Called only when starting. */
|
|
16
|
+
buildListener: (adapter: IExchangeAdapter) => StartableListener;
|
|
17
|
+
}
|
|
18
|
+
export declare class ApprovalListenerLifecycle {
|
|
19
|
+
private readonly deps;
|
|
20
|
+
private listener;
|
|
21
|
+
private chain;
|
|
22
|
+
constructor(deps: ApprovalLifecycleDeps);
|
|
23
|
+
get active(): boolean;
|
|
24
|
+
/** Apply the lifecycle for a freshly published adapter. Serialized. */
|
|
25
|
+
onAdapterSwapped(adapter: IExchangeAdapter): Promise<void>;
|
|
26
|
+
/** Terminal stop (shutdown drain) — also serialized behind pending swaps. */
|
|
27
|
+
stop(): Promise<void>;
|
|
28
|
+
private apply;
|
|
29
|
+
private stopCurrent;
|
|
30
|
+
}
|