@reefclaw/openclaw-plugin 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bridge/connector.d.ts +3 -1
- package/bridge/connector.js +37 -2
- package/bridge/gateway/gateway-ws-client.d.ts +2 -0
- package/bridge/gateway/gateway-ws-client.js +6 -0
- package/bridge/gateway/heartbeat-cron.js +2 -1
- package/bridge/heartbeat-runs-state.d.ts +16 -0
- package/bridge/heartbeat-runs-state.js +58 -0
- package/bridge/heartbeat-runs.d.ts +99 -0
- package/bridge/heartbeat-runs.js +300 -0
- package/bridge/heartbeat-transcript.d.ts +209 -0
- package/bridge/heartbeat-transcript.js +688 -0
- package/bridge/index.js +35 -0
- package/bridge/model-health.d.ts +37 -0
- package/bridge/model-health.js +97 -0
- package/bridge/provider.d.ts +5 -1
- package/bridge/providers/gateway.d.ts +25 -1
- package/bridge/providers/gateway.js +167 -2
- package/bridge/providers/mock.js +1 -0
- package/bridge/shock-wake.d.ts +80 -0
- package/bridge/shock-wake.js +291 -0
- package/bridge/types.d.ts +45 -0
- package/bridge/utils/instance-id.d.ts +3 -0
- package/bridge/utils/instance-id.js +48 -0
- package/config/agent-config-client.d.ts +3 -1
- package/config/agent-config-client.js +4 -0
- package/config/brackets-config.d.ts +2 -1
- package/config/brackets-config.js +25 -3
- package/config/gate-store.d.ts +3 -0
- package/config/gate-store.js +11 -2
- package/config/loss-streak-config.d.ts +2 -0
- package/config/loss-streak-config.js +33 -0
- package/config/plugin-config-io.d.ts +19 -0
- package/config/reentry-cooldown-config.d.ts +7 -0
- package/config/reentry-cooldown-config.js +59 -0
- package/index.js +29 -2
- package/ingest/position-auto-capture.js +49 -4
- package/ingest/readiness-reporter.d.ts +23 -2
- package/ingest/readiness-reporter.js +56 -1
- package/onboarding/runtime.js +4 -0
- package/openclaw.plugin.json +1 -1
- package/package.json +2 -2
- package/portfolio/directional-scoreboard.d.ts +17 -0
- package/portfolio/directional-scoreboard.js +71 -0
- package/portfolio/reentry-tracker.d.ts +38 -1
- package/portfolio/reentry-tracker.js +49 -0
- package/signals/change-of-character.d.ts +38 -0
- package/signals/change-of-character.js +93 -0
- package/signals/types.js +1 -1
- package/simulator/exchange-simulator.d.ts +5 -1
- package/simulator/exchange-simulator.js +24 -6
- package/simulator/types.d.ts +11 -0
- package/skills/reefclaw/SKILL.md +2 -2
- package/strategy/evaluator.d.ts +4 -0
- package/tools/close-position.js +10 -1
- package/tools/create-order.js +72 -2
- package/tools/hl-provision-agent-wallet.js +29 -11
- package/tools/reentry-cooldown.d.ts +33 -0
- package/tools/reentry-cooldown.js +74 -0
- package/tools/scan-pairs.d.ts +7 -0
- package/tools/scan-pairs.js +47 -0
- package/tools/set-exchange-credentials.js +19 -0
- package/tools/set-trading-mode.d.ts +6 -0
- package/tools/set-trading-mode.js +48 -1
- package/venues/hyperliquid/hl-agent-wallet.d.ts +26 -0
- package/venues/hyperliquid/hl-agent-wallet.js +32 -0
- package/venues/hyperliquid/hl-live-adapter.d.ts +27 -2
- package/venues/hyperliquid/hl-live-adapter.js +101 -13
package/bridge/index.js
CHANGED
|
@@ -16,7 +16,10 @@ import { MockProvider } from './providers/mock.js';
|
|
|
16
16
|
import { GatewayProvider } from './providers/gateway.js';
|
|
17
17
|
import { resolveConfig } from './config.js';
|
|
18
18
|
import { resolveGatewayConfig, validateGatewayConfig } from './gateway/gateway-config.js';
|
|
19
|
+
import { ShockWakePoller } from './shock-wake.js';
|
|
20
|
+
import { HeartbeatRunRecorder } from './heartbeat-runs.js';
|
|
19
21
|
import { runSetup } from './setup.js';
|
|
22
|
+
import { resolveRelayInstanceId } from './utils/instance-id.js';
|
|
20
23
|
const TAG = 'main';
|
|
21
24
|
// ---- Load .env file (skill/.env only) ----
|
|
22
25
|
function loadEnvFile() {
|
|
@@ -191,6 +194,7 @@ async function main() {
|
|
|
191
194
|
}
|
|
192
195
|
// Create provider
|
|
193
196
|
let provider;
|
|
197
|
+
let intelligenceUrlForWakes;
|
|
194
198
|
if (args.provider === 'mock') {
|
|
195
199
|
logger.info(TAG, 'Using MockProvider');
|
|
196
200
|
provider = new MockProvider();
|
|
@@ -217,14 +221,41 @@ async function main() {
|
|
|
217
221
|
gwConfig.connectionToken = token;
|
|
218
222
|
logger.info(TAG, `Using GatewayProvider (url=${gwConfig.gatewayUrl}, symbol=${gwConfig.symbol})`);
|
|
219
223
|
provider = new GatewayProvider(gwConfig);
|
|
224
|
+
intelligenceUrlForWakes = gwConfig.intelligenceUrl;
|
|
220
225
|
}
|
|
221
226
|
// Create bridge
|
|
222
227
|
const connectorConfig = {
|
|
223
228
|
relayUrl,
|
|
224
229
|
userId,
|
|
225
230
|
token,
|
|
231
|
+
// Stable per-install id → the relay's skill-slot admission recognises a
|
|
232
|
+
// same-box restart (instant takeover) vs a second box (rejected 4011).
|
|
233
|
+
instanceId: resolveRelayInstanceId(),
|
|
226
234
|
};
|
|
227
235
|
const bridge = new Bridge(provider, connectorConfig);
|
|
236
|
+
// Shock-wake poller (WS3, docs/MARKET_ADAPTIVITY_PLAN.md) — watches intel's
|
|
237
|
+
// /api/shocks (the WS2 change-of-character flags) and delivers an
|
|
238
|
+
// out-of-band re-evaluation turn to the agent on a genuine market shift.
|
|
239
|
+
// Inert until the central gate agent_config.gates.shockWake leaves 'off'.
|
|
240
|
+
const shockWake = intelligenceUrlForWakes
|
|
241
|
+
? new ShockWakePoller({ provider, token, intelligenceUrl: intelligenceUrlForWakes })
|
|
242
|
+
: null;
|
|
243
|
+
if (shockWake) {
|
|
244
|
+
// Delivery confirmation: any agent turn STARTING proves the main-session
|
|
245
|
+
// lane is alive — the gateway ack alone cannot (swallowed-turn class).
|
|
246
|
+
provider.on('chatMessageStart', () => shockWake.noteAssistantActivity());
|
|
247
|
+
}
|
|
248
|
+
// Heartbeat flight recorder — records what every beat actually DID (tool
|
|
249
|
+
// calls + results, report, model, raw token counts) from the isolated cron
|
|
250
|
+
// session's transcript, posts it to the webapp journal, and feeds the
|
|
251
|
+
// dashboard's last-beat summary + model-health dot. Gateway provider only
|
|
252
|
+
// (the mock has no OpenClaw sessions dir). Kill-switch: RC_HEARTBEAT_RECORDER=off.
|
|
253
|
+
const recorderOff = (process.env.RC_HEARTBEAT_RECORDER ?? 'on').trim().toLowerCase() === 'off';
|
|
254
|
+
const heartbeatRecorder = provider instanceof GatewayProvider && !recorderOff
|
|
255
|
+
? new HeartbeatRunRecorder({ source: provider, token })
|
|
256
|
+
: null;
|
|
257
|
+
if (recorderOff)
|
|
258
|
+
logger.info(TAG, 'Heartbeat flight recorder disabled (RC_HEARTBEAT_RECORDER=off)');
|
|
228
259
|
// Handle graceful shutdown
|
|
229
260
|
let shuttingDown = false;
|
|
230
261
|
async function shutdown(signal) {
|
|
@@ -239,6 +270,8 @@ async function main() {
|
|
|
239
270
|
}
|
|
240
271
|
catch { /* best-effort */ }
|
|
241
272
|
}
|
|
273
|
+
shockWake?.stop();
|
|
274
|
+
heartbeatRecorder?.stop();
|
|
242
275
|
bridge.stop();
|
|
243
276
|
// Allow 1s for final cleanup before force exit
|
|
244
277
|
setTimeout(() => process.exit(0), 1000);
|
|
@@ -248,5 +281,7 @@ async function main() {
|
|
|
248
281
|
// Start
|
|
249
282
|
logger.info(TAG, `Starting ReefClaw skill (provider=${args.provider}, relay=${relayUrl})`);
|
|
250
283
|
bridge.start();
|
|
284
|
+
shockWake?.start();
|
|
285
|
+
heartbeatRecorder?.start();
|
|
251
286
|
}
|
|
252
287
|
main();
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export type ModelHealthLevel = 'ok' | 'degraded' | 'error';
|
|
2
|
+
export interface ModelHealth {
|
|
3
|
+
level: ModelHealthLevel;
|
|
4
|
+
/** Machine reason (a FailoverReason, 'model_fallback', 'consecutive_errors'). */
|
|
5
|
+
reason: string | null;
|
|
6
|
+
/** Human detail (error text, fallback description), redacted upstream. */
|
|
7
|
+
detail: string | null;
|
|
8
|
+
/** Epoch ms of the evidence this verdict rests on. */
|
|
9
|
+
atMs: number;
|
|
10
|
+
model: string | null;
|
|
11
|
+
}
|
|
12
|
+
/** FailoverReasons that do not self-heal — the operator must act. */
|
|
13
|
+
export declare const HARD_REASONS: ReadonlySet<string>;
|
|
14
|
+
/** FailoverReasons that usually clear on their own (quota windows, outages). */
|
|
15
|
+
export declare const SOFT_REASONS: ReadonlySet<string>;
|
|
16
|
+
export interface ModelHealthInputs {
|
|
17
|
+
/** The most recently recorded heartbeat run (flight recorder). */
|
|
18
|
+
lastRun?: {
|
|
19
|
+
status: string;
|
|
20
|
+
errorReason?: string | null;
|
|
21
|
+
error?: string | null;
|
|
22
|
+
model?: string | null;
|
|
23
|
+
endedAtMs?: number | null;
|
|
24
|
+
fallbackSteps?: number;
|
|
25
|
+
modelChanges?: number;
|
|
26
|
+
} | null;
|
|
27
|
+
/** The heartbeat job's cron state (`cron.list`). */
|
|
28
|
+
cron?: {
|
|
29
|
+
consecutiveErrors: number;
|
|
30
|
+
lastErrorReason?: string | null;
|
|
31
|
+
lastError?: string | null;
|
|
32
|
+
lastRunStatus?: string | null;
|
|
33
|
+
lastRunAtMs?: number | null;
|
|
34
|
+
} | null;
|
|
35
|
+
now: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function deriveModelHealth(i: ModelHealthInputs): ModelHealth;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Model health — is the LLM behind the agent actually answering? (pure)
|
|
2
|
+
//
|
|
3
|
+
// Two incident classes motivated this: a Codex quota death (2026-06-07) and a
|
|
4
|
+
// self-inflicted harness/provider mismatch (2026-07-26) both took the agent
|
|
5
|
+
// dark for hours-to-days while the dashboard looked healthy. The gateway
|
|
6
|
+
// classifies every failed model call with a FailoverReason (vendored
|
|
7
|
+
// `embedded-agent-helpers/types.ts`) and persists it on the cron job state
|
|
8
|
+
// (`lastErrorReason`) and the run log (`errorReason`); auth failures fall back
|
|
9
|
+
// SILENTLY to the next model (docs/CLAUDE/agent-runtime.md §4), format
|
|
10
|
+
// failures kill every turn with no fallback (§1). This derives one dot from
|
|
11
|
+
// those signals so the operator sees "model degraded/erroring" instead of
|
|
12
|
+
// inferring it from a stalled last-decision clock.
|
|
13
|
+
//
|
|
14
|
+
// Levels:
|
|
15
|
+
// error — a reason that will not self-heal (auth_permanent, format,
|
|
16
|
+
// model_not_found, billing), or ≥2 consecutive model-classified
|
|
17
|
+
// failures.
|
|
18
|
+
// degraded — a recoverable model reason on the last run (auth, rate_limit,
|
|
19
|
+
// overloaded, server_error, timeout, …) or a fallback / model
|
|
20
|
+
// change observed inside the last beat.
|
|
21
|
+
// ok — nothing model-related is wrong. A run error WITHOUT a
|
|
22
|
+
// FailoverReason (e.g. a delivery-target error) stays 'ok' here:
|
|
23
|
+
// the heartbeat banner owns non-model failures.
|
|
24
|
+
/** FailoverReasons that do not self-heal — the operator must act. */
|
|
25
|
+
export const HARD_REASONS = new Set(['auth_permanent', 'format', 'model_not_found', 'billing']);
|
|
26
|
+
/** FailoverReasons that usually clear on their own (quota windows, outages). */
|
|
27
|
+
export const SOFT_REASONS = new Set([
|
|
28
|
+
'auth',
|
|
29
|
+
'rate_limit',
|
|
30
|
+
'overloaded',
|
|
31
|
+
'server_error',
|
|
32
|
+
'timeout',
|
|
33
|
+
'session_expired',
|
|
34
|
+
'empty_response',
|
|
35
|
+
'no_error_details',
|
|
36
|
+
'unclassified',
|
|
37
|
+
'unknown',
|
|
38
|
+
]);
|
|
39
|
+
function clip(s, n = 200) {
|
|
40
|
+
if (typeof s !== 'string')
|
|
41
|
+
return null;
|
|
42
|
+
const t = s.trim();
|
|
43
|
+
if (!t)
|
|
44
|
+
return null;
|
|
45
|
+
return t.length > n ? `${t.slice(0, n)}…` : t;
|
|
46
|
+
}
|
|
47
|
+
export function deriveModelHealth(i) {
|
|
48
|
+
const run = i.lastRun ?? null;
|
|
49
|
+
const cron = i.cron ?? null;
|
|
50
|
+
const model = run?.model ?? null;
|
|
51
|
+
const cronReason = cron?.lastRunStatus === 'error' ? cron.lastErrorReason ?? null : null;
|
|
52
|
+
const runReason = run?.status === 'error' ? run.errorReason ?? null : null;
|
|
53
|
+
const consecutive = cron?.consecutiveErrors ?? 0;
|
|
54
|
+
// Prefer whichever piece of evidence is newer.
|
|
55
|
+
const cronAt = cron?.lastRunAtMs ?? 0;
|
|
56
|
+
const runAt = run?.endedAtMs ?? 0;
|
|
57
|
+
const atMs = Math.max(cronAt, runAt) || i.now;
|
|
58
|
+
const hard = [cronReason, runReason].find((r) => r && HARD_REASONS.has(r)) ?? null;
|
|
59
|
+
if (hard) {
|
|
60
|
+
return {
|
|
61
|
+
level: 'error',
|
|
62
|
+
reason: hard,
|
|
63
|
+
detail: clip(cronReason === hard ? cron?.lastError : run?.error) ?? `model ${hard}`,
|
|
64
|
+
atMs,
|
|
65
|
+
model,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
const anyReason = cronReason ?? runReason;
|
|
69
|
+
if (anyReason && consecutive >= 2) {
|
|
70
|
+
return {
|
|
71
|
+
level: 'error',
|
|
72
|
+
reason: 'consecutive_errors',
|
|
73
|
+
detail: clip(cron?.lastError ?? run?.error) ?? `${consecutive} consecutive model failures (${anyReason})`,
|
|
74
|
+
atMs,
|
|
75
|
+
model,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (anyReason && (SOFT_REASONS.has(anyReason) || !HARD_REASONS.has(anyReason))) {
|
|
79
|
+
return {
|
|
80
|
+
level: 'degraded',
|
|
81
|
+
reason: anyReason,
|
|
82
|
+
detail: clip(cronReason ? cron?.lastError : run?.error) ?? `model ${anyReason}`,
|
|
83
|
+
atMs,
|
|
84
|
+
model,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (run && ((run.fallbackSteps ?? 0) > 0 || (run.modelChanges ?? 0) > 0)) {
|
|
88
|
+
return {
|
|
89
|
+
level: 'degraded',
|
|
90
|
+
reason: 'model_fallback',
|
|
91
|
+
detail: `${run.fallbackSteps ?? 0} fallback step(s), ${run.modelChanges ?? 0} model change(s) in the last beat`,
|
|
92
|
+
atMs,
|
|
93
|
+
model,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
return { level: 'ok', reason: null, detail: null, atMs, model };
|
|
97
|
+
}
|
package/bridge/provider.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData } from './types.js';
|
|
1
|
+
import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData, AgentRunLifecycleData } from './types.js';
|
|
2
2
|
import type { TradingMode } from '@reefclaw/shared';
|
|
3
3
|
import type { ConnectorUpdateOutcome } from './providers/connector-update.js';
|
|
4
4
|
/** Per-venue credential shapes for the operator set/test RPCs. Binance is an
|
|
@@ -126,6 +126,10 @@ export interface ProviderEvents {
|
|
|
126
126
|
tradingModeUpdate: (data: TradingModeData) => void;
|
|
127
127
|
/** Decision trace update (Phase 9c) */
|
|
128
128
|
decisionTraceUpdate: (data: DecisionTraceData) => void;
|
|
129
|
+
/** An agent turn started/ended/errored, with the gateway's session key so a
|
|
130
|
+
* heartbeat (cron) turn is distinguishable from a chat turn. Local-only
|
|
131
|
+
* (not forwarded to the relay); consumed by the heartbeat flight recorder. */
|
|
132
|
+
agentRunLifecycle: (data: AgentRunLifecycleData) => void;
|
|
129
133
|
}
|
|
130
134
|
export type ProviderEventName = keyof ProviderEvents;
|
|
131
135
|
/**
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { OpenClawProvider, ProviderEvents, ProviderEventName, ExchangeCredentialsRequest, HlProvisionOutcome, HlAgentWalletStatusOutcome, HlSubmitApprovalOutcome } from '../provider.js';
|
|
2
|
-
import type { EmergencyAction, ReconciliationSnapshot, TradingMode } from '../types.js';
|
|
2
|
+
import type { EmergencyAction, ReconciliationSnapshot, TradingMode, AgentRunLifecycleData } from '../types.js';
|
|
3
|
+
import type { CronRunLogLite, HeartbeatRunRecord } from '../heartbeat-transcript.js';
|
|
3
4
|
import type { GatewayConfig } from '../gateway/gateway-config.js';
|
|
4
5
|
import { type GetBracketConfigOutcome, type SetBracketRequirementOutcome, type BracketRequirementFlag, type SetTradingModeOutcome, type SetExchangeCredentialsOutcome, type TestExchangeCredentialsOutcome, type ClearExchangeCredentialsOutcome } from './onboarding-commands.js';
|
|
5
6
|
import { type ConnectorUpdateOutcome } from './connector-update.js';
|
|
@@ -56,6 +57,14 @@ export declare class GatewayProvider implements OpenClawProvider {
|
|
|
56
57
|
private heartbeatHealth?;
|
|
57
58
|
private heartbeatHealthReadAtMs;
|
|
58
59
|
private heartbeatHealthRefreshing;
|
|
60
|
+
/** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
|
|
61
|
+
private heartbeatJobId?;
|
|
62
|
+
/** Last recorded heartbeat run (flight recorder) — dashboard summary. */
|
|
63
|
+
private lastHeartbeat?;
|
|
64
|
+
/** Model-health inputs from the last recorded run (kept apart from the
|
|
65
|
+
* summary so the verdict can be re-derived when cron state changes). */
|
|
66
|
+
private lastHeartbeatRunForHealth?;
|
|
67
|
+
private modelHealth?;
|
|
59
68
|
private lastTicker;
|
|
60
69
|
private positions;
|
|
61
70
|
private balance;
|
|
@@ -395,6 +404,21 @@ export declare class GatewayProvider implements OpenClawProvider {
|
|
|
395
404
|
* refresh is in flight. Fire-and-forget: the RPC completion re-emits
|
|
396
405
|
* agent_state, so callers stay synchronous. */
|
|
397
406
|
private maybeRefreshHeartbeatHealth;
|
|
407
|
+
/** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
|
|
408
|
+
* first), normalised to the recorder's shape. Returns null when the RPC or
|
|
409
|
+
* the job id is unavailable (older gateway / scope) — the recorder then
|
|
410
|
+
* falls back to transcript-only records. Token counts here are RAW counts. */
|
|
411
|
+
fetchCronRuns(limit?: number): Promise<CronRunLogLite[] | null>;
|
|
412
|
+
private loggedCronRunsProbe;
|
|
413
|
+
/** A heartbeat run was recorded — keep a compact summary for the dashboard
|
|
414
|
+
* header, re-derive model health, and re-emit agent_state. */
|
|
415
|
+
noteHeartbeatRun(record: HeartbeatRunRecord): void;
|
|
416
|
+
/** Active trading book for the record's mode tag (matches the webapp's
|
|
417
|
+
* modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
|
|
418
|
+
getTradingBook(): 'paper' | 'live';
|
|
419
|
+
/** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
|
|
420
|
+
onAgentRunLifecycle(listener: (d: AgentRunLifecycleData) => void): () => void;
|
|
421
|
+
private recomputeModelHealth;
|
|
398
422
|
/** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
|
|
399
423
|
private mergeAgentIdentity;
|
|
400
424
|
/**
|
|
@@ -6,6 +6,7 @@ import { join } from 'path';
|
|
|
6
6
|
import { homedir } from 'os';
|
|
7
7
|
import { logger, formatError } from '../logger.js';
|
|
8
8
|
import { isTradingMode } from '../types.js';
|
|
9
|
+
import { deriveModelHealth } from '../model-health.js';
|
|
9
10
|
import { handshakeLacksWriteScope, relaxGatewayDeviceAuth } from '../config.js';
|
|
10
11
|
import { toIntelSymbol } from '@reefclaw/shared';
|
|
11
12
|
import { GatewayHttpClient } from '../gateway/gateway-http-client.js';
|
|
@@ -175,6 +176,14 @@ export class GatewayProvider {
|
|
|
175
176
|
heartbeatHealth;
|
|
176
177
|
heartbeatHealthReadAtMs = 0;
|
|
177
178
|
heartbeatHealthRefreshing = false;
|
|
179
|
+
/** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
|
|
180
|
+
heartbeatJobId;
|
|
181
|
+
/** Last recorded heartbeat run (flight recorder) — dashboard summary. */
|
|
182
|
+
lastHeartbeat;
|
|
183
|
+
/** Model-health inputs from the last recorded run (kept apart from the
|
|
184
|
+
* summary so the verdict can be re-derived when cron state changes). */
|
|
185
|
+
lastHeartbeatRunForHealth;
|
|
186
|
+
modelHealth;
|
|
178
187
|
lastTicker = null;
|
|
179
188
|
positions = [];
|
|
180
189
|
balance = { currency: 'USDT', total: 0, available: 0, locked: 0 };
|
|
@@ -318,6 +327,7 @@ export class GatewayProvider {
|
|
|
318
327
|
shadowComparison: new Set(),
|
|
319
328
|
tradingModeUpdate: new Set(),
|
|
320
329
|
decisionTraceUpdate: new Set(),
|
|
330
|
+
agentRunLifecycle: new Set(),
|
|
321
331
|
};
|
|
322
332
|
constructor(config) {
|
|
323
333
|
this.config = config;
|
|
@@ -1368,6 +1378,21 @@ export class GatewayProvider {
|
|
|
1368
1378
|
onAgentEvent(payload) {
|
|
1369
1379
|
if (!this.started || !this.eventParser)
|
|
1370
1380
|
return;
|
|
1381
|
+
// Turn lifecycle WITH the session key — the heartbeat flight recorder
|
|
1382
|
+
// keys off `agent:main:cron:…` to scan the beat's transcript right after
|
|
1383
|
+
// it ends. Fired before parsing so it never depends on parser state.
|
|
1384
|
+
if (payload.stream === 'lifecycle') {
|
|
1385
|
+
const phase = payload.data?.phase;
|
|
1386
|
+
if (phase === 'start' || phase === 'end' || phase === 'error') {
|
|
1387
|
+
this.fire('agentRunLifecycle', {
|
|
1388
|
+
runId: payload.runId,
|
|
1389
|
+
phase,
|
|
1390
|
+
...(payload.sessionKey ? { sessionKey: payload.sessionKey } : {}),
|
|
1391
|
+
...(payload.sessionId ? { sessionId: payload.sessionId } : {}),
|
|
1392
|
+
at: Date.now(),
|
|
1393
|
+
});
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1371
1396
|
const parsed = this.eventParser.parseAgentEvent(payload);
|
|
1372
1397
|
for (const evt of parsed) {
|
|
1373
1398
|
// Track order/cancel rates for risk computation
|
|
@@ -2118,6 +2143,9 @@ export class GatewayProvider {
|
|
|
2118
2143
|
this.heartbeatSeconds = secs;
|
|
2119
2144
|
this.heartbeatReadAtMs = Date.now();
|
|
2120
2145
|
}
|
|
2146
|
+
const jobId = hb?.id;
|
|
2147
|
+
if (typeof jobId === 'string' && jobId)
|
|
2148
|
+
this.heartbeatJobId = jobId;
|
|
2121
2149
|
const state = hb?.state;
|
|
2122
2150
|
if (!state || typeof state !== 'object')
|
|
2123
2151
|
return;
|
|
@@ -2126,18 +2154,23 @@ export class GatewayProvider {
|
|
|
2126
2154
|
consecutiveErrors: typeof s.consecutiveErrors === 'number' && s.consecutiveErrors >= 0 ? s.consecutiveErrors : 0,
|
|
2127
2155
|
lastRunAtMs: typeof s.lastRunAtMs === 'number' ? s.lastRunAtMs : undefined,
|
|
2128
2156
|
lastRunStatus: typeof s.lastRunStatus === 'string' ? s.lastRunStatus : undefined,
|
|
2157
|
+
lastErrorReason: typeof s.lastErrorReason === 'string' ? s.lastErrorReason : undefined,
|
|
2158
|
+
lastError: typeof s.lastError === 'string' ? s.lastError.slice(0, 300) : undefined,
|
|
2129
2159
|
};
|
|
2130
2160
|
const prev = this.heartbeatHealth;
|
|
2131
2161
|
const changed = !prev ||
|
|
2132
2162
|
prev.consecutiveErrors !== next.consecutiveErrors ||
|
|
2133
2163
|
prev.lastRunAtMs !== next.lastRunAtMs ||
|
|
2134
|
-
prev.lastRunStatus !== next.lastRunStatus
|
|
2164
|
+
prev.lastRunStatus !== next.lastRunStatus ||
|
|
2165
|
+
prev.lastErrorReason !== next.lastErrorReason;
|
|
2135
2166
|
this.heartbeatHealth = next;
|
|
2136
2167
|
this.heartbeatHealthReadAtMs = Date.now();
|
|
2137
2168
|
if (changed) {
|
|
2138
2169
|
if (next.consecutiveErrors > 0) {
|
|
2139
|
-
logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}`
|
|
2170
|
+
logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}` +
|
|
2171
|
+
(next.lastErrorReason ? ` reason=${next.lastErrorReason}` : ''));
|
|
2140
2172
|
}
|
|
2173
|
+
this.recomputeModelHealth();
|
|
2141
2174
|
this.emitAgentState();
|
|
2142
2175
|
}
|
|
2143
2176
|
}
|
|
@@ -2158,6 +2191,134 @@ export class GatewayProvider {
|
|
|
2158
2191
|
return;
|
|
2159
2192
|
void this.refreshHeartbeatHealth();
|
|
2160
2193
|
}
|
|
2194
|
+
// ---- Heartbeat flight recorder hooks (skill/src/heartbeat-runs.ts) ----
|
|
2195
|
+
/** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
|
|
2196
|
+
* first), normalised to the recorder's shape. Returns null when the RPC or
|
|
2197
|
+
* the job id is unavailable (older gateway / scope) — the recorder then
|
|
2198
|
+
* falls back to transcript-only records. Token counts here are RAW counts. */
|
|
2199
|
+
async fetchCronRuns(limit = 25) {
|
|
2200
|
+
const ws = this.wsClient;
|
|
2201
|
+
if (!ws)
|
|
2202
|
+
return null;
|
|
2203
|
+
if (!this.heartbeatJobId) {
|
|
2204
|
+
await this.refreshHeartbeatHealth();
|
|
2205
|
+
if (!this.heartbeatJobId)
|
|
2206
|
+
return null;
|
|
2207
|
+
}
|
|
2208
|
+
let resp;
|
|
2209
|
+
try {
|
|
2210
|
+
resp = await ws.sendRpc('cron.runs', { jobId: this.heartbeatJobId, limit, sortDir: 'desc' });
|
|
2211
|
+
}
|
|
2212
|
+
catch (err) {
|
|
2213
|
+
logger.debug(TAG, `cron.runs RPC failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2214
|
+
return null;
|
|
2215
|
+
}
|
|
2216
|
+
const entries = resp?.entries;
|
|
2217
|
+
if (!Array.isArray(entries)) {
|
|
2218
|
+
if (!this.loggedCronRunsProbe) {
|
|
2219
|
+
this.loggedCronRunsProbe = true;
|
|
2220
|
+
const keys = resp && typeof resp === 'object' ? Object.keys(resp).join(',') : typeof resp;
|
|
2221
|
+
logger.info(TAG, `cron.runs probe: unexpected shape keys=[${keys}] — token counts fall back to transcripts`);
|
|
2222
|
+
}
|
|
2223
|
+
return null;
|
|
2224
|
+
}
|
|
2225
|
+
if (!this.loggedCronRunsProbe) {
|
|
2226
|
+
this.loggedCronRunsProbe = true;
|
|
2227
|
+
const first = entries[0] && typeof entries[0] === 'object' ? Object.keys(entries[0]).join(',') : '(empty)';
|
|
2228
|
+
logger.info(TAG, `cron.runs probe: ${entries.length} entries, keys=[${first}]`);
|
|
2229
|
+
}
|
|
2230
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
|
|
2231
|
+
const str = (v) => (typeof v === 'string' && v ? v : null);
|
|
2232
|
+
return entries.map((raw) => {
|
|
2233
|
+
const e = (raw && typeof raw === 'object' ? raw : {});
|
|
2234
|
+
const u = (e.usage && typeof e.usage === 'object' ? e.usage : null);
|
|
2235
|
+
return {
|
|
2236
|
+
sessionId: str(e.sessionId),
|
|
2237
|
+
runId: str(e.runId),
|
|
2238
|
+
status: str(e.status),
|
|
2239
|
+
error: str(e.error),
|
|
2240
|
+
errorReason: str(e.errorReason),
|
|
2241
|
+
runAtMs: num(e.runAtMs) ?? num(e.ts),
|
|
2242
|
+
durationMs: num(e.durationMs),
|
|
2243
|
+
model: str(e.model),
|
|
2244
|
+
provider: str(e.provider),
|
|
2245
|
+
usage: u
|
|
2246
|
+
? {
|
|
2247
|
+
input: num(u.input_tokens) ?? num(u.input),
|
|
2248
|
+
output: num(u.output_tokens) ?? num(u.output),
|
|
2249
|
+
cacheRead: num(u.cache_read_tokens) ?? num(u.cacheRead),
|
|
2250
|
+
cacheWrite: num(u.cache_write_tokens) ?? num(u.cacheWrite),
|
|
2251
|
+
total: num(u.total_tokens) ?? num(u.total) ?? num(u.totalTokens),
|
|
2252
|
+
}
|
|
2253
|
+
: null,
|
|
2254
|
+
};
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2257
|
+
loggedCronRunsProbe = false;
|
|
2258
|
+
/** A heartbeat run was recorded — keep a compact summary for the dashboard
|
|
2259
|
+
* header, re-derive model health, and re-emit agent_state. */
|
|
2260
|
+
noteHeartbeatRun(record) {
|
|
2261
|
+
this.lastHeartbeat = {
|
|
2262
|
+
runId: record.runId,
|
|
2263
|
+
startedAtMs: record.startedAtMs,
|
|
2264
|
+
endedAtMs: record.endedAtMs,
|
|
2265
|
+
durationMs: record.durationMs,
|
|
2266
|
+
status: record.status,
|
|
2267
|
+
toolCallCount: record.toolCallCount,
|
|
2268
|
+
toolErrorCount: record.toolErrorCount,
|
|
2269
|
+
tokensTotal: record.tokens.total,
|
|
2270
|
+
tokensInput: record.tokens.input,
|
|
2271
|
+
tokensOutput: record.tokens.output,
|
|
2272
|
+
model: record.model,
|
|
2273
|
+
};
|
|
2274
|
+
this.lastHeartbeatRunForHealth = {
|
|
2275
|
+
status: record.status,
|
|
2276
|
+
errorReason: record.errorReason,
|
|
2277
|
+
error: record.error,
|
|
2278
|
+
model: record.model,
|
|
2279
|
+
endedAtMs: record.endedAtMs,
|
|
2280
|
+
fallbackSteps: record.modelEvents.filter((e) => e.kind === 'fallback_step').length,
|
|
2281
|
+
modelChanges: record.modelEvents.filter((e) => e.kind === 'model_change').length,
|
|
2282
|
+
};
|
|
2283
|
+
this.recomputeModelHealth();
|
|
2284
|
+
this.emitAgentState();
|
|
2285
|
+
}
|
|
2286
|
+
/** Active trading book for the record's mode tag (matches the webapp's
|
|
2287
|
+
* modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
|
|
2288
|
+
getTradingBook() {
|
|
2289
|
+
return this.tradingMode === 'LIVE' || this.tradingMode === 'MICRO_LIVE' ? 'live' : 'paper';
|
|
2290
|
+
}
|
|
2291
|
+
/** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
|
|
2292
|
+
onAgentRunLifecycle(listener) {
|
|
2293
|
+
this.on('agentRunLifecycle', listener);
|
|
2294
|
+
return () => this.off('agentRunLifecycle', listener);
|
|
2295
|
+
}
|
|
2296
|
+
recomputeModelHealth() {
|
|
2297
|
+
const hb = this.heartbeatHealth;
|
|
2298
|
+
const next = deriveModelHealth({
|
|
2299
|
+
lastRun: this.lastHeartbeatRunForHealth ?? null,
|
|
2300
|
+
cron: hb
|
|
2301
|
+
? {
|
|
2302
|
+
consecutiveErrors: hb.consecutiveErrors,
|
|
2303
|
+
lastErrorReason: hb.lastErrorReason ?? null,
|
|
2304
|
+
lastError: hb.lastError ?? null,
|
|
2305
|
+
lastRunStatus: hb.lastRunStatus ?? null,
|
|
2306
|
+
lastRunAtMs: hb.lastRunAtMs ?? null,
|
|
2307
|
+
}
|
|
2308
|
+
: null,
|
|
2309
|
+
now: Date.now(),
|
|
2310
|
+
});
|
|
2311
|
+
const prev = this.modelHealth;
|
|
2312
|
+
if (!prev || prev.level !== next.level || prev.reason !== next.reason) {
|
|
2313
|
+
if (next.level !== 'ok') {
|
|
2314
|
+
logger.warn(TAG, `Model health ${next.level}: ${next.reason ?? ''} ${next.detail ?? ''}`.trim());
|
|
2315
|
+
}
|
|
2316
|
+
else if (prev && prev.level !== 'ok') {
|
|
2317
|
+
logger.info(TAG, 'Model health recovered: ok');
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
this.modelHealth = next;
|
|
2321
|
+
}
|
|
2161
2322
|
/** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
|
|
2162
2323
|
mergeAgentIdentity(found) {
|
|
2163
2324
|
let changed = false;
|
|
@@ -2349,6 +2510,10 @@ export class GatewayProvider {
|
|
|
2349
2510
|
},
|
|
2350
2511
|
}
|
|
2351
2512
|
: {}),
|
|
2513
|
+
// Flight recorder: absent until the first beat is recorded → payload
|
|
2514
|
+
// stays byte-identical to the pre-feature shape until then.
|
|
2515
|
+
...(this.lastHeartbeat ? { lastHeartbeat: this.lastHeartbeat } : {}),
|
|
2516
|
+
...(this.modelHealth ? { modelHealth: this.modelHealth } : {}),
|
|
2352
2517
|
},
|
|
2353
2518
|
};
|
|
2354
2519
|
}
|
package/bridge/providers/mock.js
CHANGED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
export type ShockWakeMode = 'off' | 'shadow' | 'on';
|
|
2
|
+
export interface ShockEntry {
|
|
3
|
+
symbol: string;
|
|
4
|
+
time: string;
|
|
5
|
+
flags: string[];
|
|
6
|
+
shock30mAtr: number;
|
|
7
|
+
range30mAtr: number;
|
|
8
|
+
return1hPct: number;
|
|
9
|
+
return4hPct: number;
|
|
10
|
+
tape4hAtr: number;
|
|
11
|
+
regime: string;
|
|
12
|
+
summary?: string;
|
|
13
|
+
}
|
|
14
|
+
/** 'BTCUSDT' / 'HL_BTC' / 'BTC/USDC' → 'BTC'. Best-effort market vocabulary
|
|
15
|
+
* for the wake message — never used to address an order. */
|
|
16
|
+
export declare function coinOf(intelSymbol: string): string;
|
|
17
|
+
/** Parse gates.shockWake off the raw /api/internal/config payload. Central-only
|
|
18
|
+
* gate — absent/garbage → 'off'. */
|
|
19
|
+
export declare function parseShockWakeGate(raw: unknown): ShockWakeMode;
|
|
20
|
+
export interface WakeAssessment {
|
|
21
|
+
wake: boolean;
|
|
22
|
+
reason?: 'leader_shock' | 'broad_shock';
|
|
23
|
+
/** Fresh 'shock'-flagged entries, extreme-first (as served). */
|
|
24
|
+
shocked: ShockEntry[];
|
|
25
|
+
}
|
|
26
|
+
/** Pure wake-worthiness rule. `nowMs` gates staleness. Broad rule (2026-09-06):
|
|
27
|
+
* ≥`minBroadCount` distinct coins shocked, OR ≥3 when a major participates. */
|
|
28
|
+
export declare function assessWakeWorthiness(shocks: ShockEntry[], nowMs: number, minBroadCount?: number): WakeAssessment;
|
|
29
|
+
/** The out-of-band agent turn. Clearly machine-labeled (never impersonates the
|
|
30
|
+
* operator), mandates re-evaluation, orders nothing. */
|
|
31
|
+
export declare function buildWakeMessage(a: WakeAssessment): string;
|
|
32
|
+
export interface ShockWakeOpts {
|
|
33
|
+
provider: {
|
|
34
|
+
handleChat(content: string): Promise<{
|
|
35
|
+
received: boolean;
|
|
36
|
+
error?: string;
|
|
37
|
+
}>;
|
|
38
|
+
};
|
|
39
|
+
token: string;
|
|
40
|
+
intelligenceUrl: string;
|
|
41
|
+
webappUrl?: string;
|
|
42
|
+
fetchImpl?: typeof fetch;
|
|
43
|
+
now?: () => number;
|
|
44
|
+
}
|
|
45
|
+
export declare class ShockWakePoller {
|
|
46
|
+
private readonly opts;
|
|
47
|
+
private mode;
|
|
48
|
+
private pollTimer;
|
|
49
|
+
private gateTimer;
|
|
50
|
+
private lastWakeAtMs;
|
|
51
|
+
private wakesToday;
|
|
52
|
+
private wakeDayUtc;
|
|
53
|
+
private polling;
|
|
54
|
+
private readonly pollMs;
|
|
55
|
+
private readonly gatePollMs;
|
|
56
|
+
private readonly cooldownMs;
|
|
57
|
+
private readonly dailyCap;
|
|
58
|
+
private readonly minBroad;
|
|
59
|
+
private readonly replyTimeoutMs;
|
|
60
|
+
private lastAssistantAtMs;
|
|
61
|
+
private awaitingSinceMs;
|
|
62
|
+
private awaitingDeadlineMs;
|
|
63
|
+
private suspectedSwallows;
|
|
64
|
+
constructor(opts: ShockWakeOpts);
|
|
65
|
+
start(): void;
|
|
66
|
+
stop(): void;
|
|
67
|
+
getMode(): ShockWakeMode;
|
|
68
|
+
/** Fed by the bridge from provider `chatMessageStart` — any agent turn
|
|
69
|
+
* beginning counts (the wake's reply, or an operator chat's; either proves
|
|
70
|
+
* the lane is alive, which is what the swallow check needs). */
|
|
71
|
+
noteAssistantActivity(atMs?: number): void;
|
|
72
|
+
/** Suspected-swallow count this process (test/telemetry seam). */
|
|
73
|
+
getSuspectedSwallows(): number;
|
|
74
|
+
private checkPendingReply;
|
|
75
|
+
private refreshGate;
|
|
76
|
+
/** One poll cycle. Exposed for tests. */
|
|
77
|
+
tick(): Promise<void>;
|
|
78
|
+
/** Test seam. */
|
|
79
|
+
__setModeForTest(mode: ShockWakeMode): void;
|
|
80
|
+
}
|