@reefclaw/connect 0.1.37 → 0.1.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/bridge/gateway/gateway-ws-client.d.ts +2 -0
- package/assets/bridge/gateway/gateway-ws-client.js +6 -0
- package/assets/bridge/heartbeat-runs-state.d.ts +16 -0
- package/assets/bridge/heartbeat-runs-state.js +58 -0
- package/assets/bridge/heartbeat-runs.d.ts +99 -0
- package/assets/bridge/heartbeat-runs.js +300 -0
- package/assets/bridge/heartbeat-transcript.d.ts +209 -0
- package/assets/bridge/heartbeat-transcript.js +688 -0
- package/assets/bridge/index.js +14 -0
- package/assets/bridge/model-health.d.ts +37 -0
- package/assets/bridge/model-health.js +97 -0
- package/assets/bridge/provider.d.ts +5 -1
- package/assets/bridge/providers/gateway.d.ts +25 -1
- package/assets/bridge/providers/gateway.js +167 -2
- package/assets/bridge/providers/mock.js +1 -0
- package/assets/bridge/types.d.ts +41 -0
- package/assets/plugin/signals/types.js +1 -1
- package/assets/plugin/simulator/types.d.ts +1 -1
- package/assets/plugin/tools/create-order.js +1 -1
- package/assets/plugin/tools/reentry-cooldown.js +2 -2
- package/package.json +1 -1
package/assets/bridge/index.js
CHANGED
|
@@ -17,6 +17,7 @@ import { GatewayProvider } from './providers/gateway.js';
|
|
|
17
17
|
import { resolveConfig } from './config.js';
|
|
18
18
|
import { resolveGatewayConfig, validateGatewayConfig } from './gateway/gateway-config.js';
|
|
19
19
|
import { ShockWakePoller } from './shock-wake.js';
|
|
20
|
+
import { HeartbeatRunRecorder } from './heartbeat-runs.js';
|
|
20
21
|
import { runSetup } from './setup.js';
|
|
21
22
|
import { resolveRelayInstanceId } from './utils/instance-id.js';
|
|
22
23
|
const TAG = 'main';
|
|
@@ -244,6 +245,17 @@ async function main() {
|
|
|
244
245
|
// lane is alive — the gateway ack alone cannot (swallowed-turn class).
|
|
245
246
|
provider.on('chatMessageStart', () => shockWake.noteAssistantActivity());
|
|
246
247
|
}
|
|
248
|
+
// Heartbeat flight recorder — records what every beat actually DID (tool
|
|
249
|
+
// calls + results, report, model, raw token counts) from the isolated cron
|
|
250
|
+
// session's transcript, posts it to the webapp journal, and feeds the
|
|
251
|
+
// dashboard's last-beat summary + model-health dot. Gateway provider only
|
|
252
|
+
// (the mock has no OpenClaw sessions dir). Kill-switch: RC_HEARTBEAT_RECORDER=off.
|
|
253
|
+
const recorderOff = (process.env.RC_HEARTBEAT_RECORDER ?? 'on').trim().toLowerCase() === 'off';
|
|
254
|
+
const heartbeatRecorder = provider instanceof GatewayProvider && !recorderOff
|
|
255
|
+
? new HeartbeatRunRecorder({ source: provider, token })
|
|
256
|
+
: null;
|
|
257
|
+
if (recorderOff)
|
|
258
|
+
logger.info(TAG, 'Heartbeat flight recorder disabled (RC_HEARTBEAT_RECORDER=off)');
|
|
247
259
|
// Handle graceful shutdown
|
|
248
260
|
let shuttingDown = false;
|
|
249
261
|
async function shutdown(signal) {
|
|
@@ -259,6 +271,7 @@ async function main() {
|
|
|
259
271
|
catch { /* best-effort */ }
|
|
260
272
|
}
|
|
261
273
|
shockWake?.stop();
|
|
274
|
+
heartbeatRecorder?.stop();
|
|
262
275
|
bridge.stop();
|
|
263
276
|
// Allow 1s for final cleanup before force exit
|
|
264
277
|
setTimeout(() => process.exit(0), 1000);
|
|
@@ -269,5 +282,6 @@ async function main() {
|
|
|
269
282
|
logger.info(TAG, `Starting ReefClaw skill (provider=${args.provider}, relay=${relayUrl})`);
|
|
270
283
|
bridge.start();
|
|
271
284
|
shockWake?.start();
|
|
285
|
+
heartbeatRecorder?.start();
|
|
272
286
|
}
|
|
273
287
|
main();
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export type ModelHealthLevel = 'ok' | 'degraded' | 'error';
|
|
2
|
+
export interface ModelHealth {
|
|
3
|
+
level: ModelHealthLevel;
|
|
4
|
+
/** Machine reason (a FailoverReason, 'model_fallback', 'consecutive_errors'). */
|
|
5
|
+
reason: string | null;
|
|
6
|
+
/** Human detail (error text, fallback description), redacted upstream. */
|
|
7
|
+
detail: string | null;
|
|
8
|
+
/** Epoch ms of the evidence this verdict rests on. */
|
|
9
|
+
atMs: number;
|
|
10
|
+
model: string | null;
|
|
11
|
+
}
|
|
12
|
+
/** FailoverReasons that do not self-heal — the operator must act. */
|
|
13
|
+
export declare const HARD_REASONS: ReadonlySet<string>;
|
|
14
|
+
/** FailoverReasons that usually clear on their own (quota windows, outages). */
|
|
15
|
+
export declare const SOFT_REASONS: ReadonlySet<string>;
|
|
16
|
+
export interface ModelHealthInputs {
|
|
17
|
+
/** The most recently recorded heartbeat run (flight recorder). */
|
|
18
|
+
lastRun?: {
|
|
19
|
+
status: string;
|
|
20
|
+
errorReason?: string | null;
|
|
21
|
+
error?: string | null;
|
|
22
|
+
model?: string | null;
|
|
23
|
+
endedAtMs?: number | null;
|
|
24
|
+
fallbackSteps?: number;
|
|
25
|
+
modelChanges?: number;
|
|
26
|
+
} | null;
|
|
27
|
+
/** The heartbeat job's cron state (`cron.list`). */
|
|
28
|
+
cron?: {
|
|
29
|
+
consecutiveErrors: number;
|
|
30
|
+
lastErrorReason?: string | null;
|
|
31
|
+
lastError?: string | null;
|
|
32
|
+
lastRunStatus?: string | null;
|
|
33
|
+
lastRunAtMs?: number | null;
|
|
34
|
+
} | null;
|
|
35
|
+
now: number;
|
|
36
|
+
}
|
|
37
|
+
export declare function deriveModelHealth(i: ModelHealthInputs): ModelHealth;
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
// Model health — is the LLM behind the agent actually answering? (pure)
|
|
2
|
+
//
|
|
3
|
+
// Two incident classes motivated this: a Codex quota death (2026-06-07) and a
|
|
4
|
+
// self-inflicted harness/provider mismatch (2026-07-26) both took the agent
|
|
5
|
+
// dark for hours-to-days while the dashboard looked healthy. The gateway
|
|
6
|
+
// classifies every failed model call with a FailoverReason (vendored
|
|
7
|
+
// `embedded-agent-helpers/types.ts`) and persists it on the cron job state
|
|
8
|
+
// (`lastErrorReason`) and the run log (`errorReason`); auth failures fall back
|
|
9
|
+
// SILENTLY to the next model (docs/CLAUDE/agent-runtime.md §4), format
|
|
10
|
+
// failures kill every turn with no fallback (§1). This derives one dot from
|
|
11
|
+
// those signals so the operator sees "model degraded/erroring" instead of
|
|
12
|
+
// inferring it from a stalled last-decision clock.
|
|
13
|
+
//
|
|
14
|
+
// Levels:
|
|
15
|
+
// error — a reason that will not self-heal (auth_permanent, format,
|
|
16
|
+
// model_not_found, billing), or ≥2 consecutive model-classified
|
|
17
|
+
// failures.
|
|
18
|
+
// degraded — a recoverable model reason on the last run (auth, rate_limit,
|
|
19
|
+
// overloaded, server_error, timeout, …) or a fallback / model
|
|
20
|
+
// change observed inside the last beat.
|
|
21
|
+
// ok — nothing model-related is wrong. A run error WITHOUT a
|
|
22
|
+
// FailoverReason (e.g. a delivery-target error) stays 'ok' here:
|
|
23
|
+
// the heartbeat banner owns non-model failures.
|
|
24
|
+
/** FailoverReasons that do not self-heal — the operator must act. */
|
|
25
|
+
export const HARD_REASONS = new Set(['auth_permanent', 'format', 'model_not_found', 'billing']);
|
|
26
|
+
/** FailoverReasons that usually clear on their own (quota windows, outages). */
|
|
27
|
+
export const SOFT_REASONS = new Set([
|
|
28
|
+
'auth',
|
|
29
|
+
'rate_limit',
|
|
30
|
+
'overloaded',
|
|
31
|
+
'server_error',
|
|
32
|
+
'timeout',
|
|
33
|
+
'session_expired',
|
|
34
|
+
'empty_response',
|
|
35
|
+
'no_error_details',
|
|
36
|
+
'unclassified',
|
|
37
|
+
'unknown',
|
|
38
|
+
]);
|
|
39
|
+
function clip(s, n = 200) {
|
|
40
|
+
if (typeof s !== 'string')
|
|
41
|
+
return null;
|
|
42
|
+
const t = s.trim();
|
|
43
|
+
if (!t)
|
|
44
|
+
return null;
|
|
45
|
+
return t.length > n ? `${t.slice(0, n)}…` : t;
|
|
46
|
+
}
|
|
47
|
+
export function deriveModelHealth(i) {
|
|
48
|
+
const run = i.lastRun ?? null;
|
|
49
|
+
const cron = i.cron ?? null;
|
|
50
|
+
const model = run?.model ?? null;
|
|
51
|
+
const cronReason = cron?.lastRunStatus === 'error' ? cron.lastErrorReason ?? null : null;
|
|
52
|
+
const runReason = run?.status === 'error' ? run.errorReason ?? null : null;
|
|
53
|
+
const consecutive = cron?.consecutiveErrors ?? 0;
|
|
54
|
+
// Prefer whichever piece of evidence is newer.
|
|
55
|
+
const cronAt = cron?.lastRunAtMs ?? 0;
|
|
56
|
+
const runAt = run?.endedAtMs ?? 0;
|
|
57
|
+
const atMs = Math.max(cronAt, runAt) || i.now;
|
|
58
|
+
const hard = [cronReason, runReason].find((r) => r && HARD_REASONS.has(r)) ?? null;
|
|
59
|
+
if (hard) {
|
|
60
|
+
return {
|
|
61
|
+
level: 'error',
|
|
62
|
+
reason: hard,
|
|
63
|
+
detail: clip(cronReason === hard ? cron?.lastError : run?.error) ?? `model ${hard}`,
|
|
64
|
+
atMs,
|
|
65
|
+
model,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
const anyReason = cronReason ?? runReason;
|
|
69
|
+
if (anyReason && consecutive >= 2) {
|
|
70
|
+
return {
|
|
71
|
+
level: 'error',
|
|
72
|
+
reason: 'consecutive_errors',
|
|
73
|
+
detail: clip(cron?.lastError ?? run?.error) ?? `${consecutive} consecutive model failures (${anyReason})`,
|
|
74
|
+
atMs,
|
|
75
|
+
model,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
if (anyReason && (SOFT_REASONS.has(anyReason) || !HARD_REASONS.has(anyReason))) {
|
|
79
|
+
return {
|
|
80
|
+
level: 'degraded',
|
|
81
|
+
reason: anyReason,
|
|
82
|
+
detail: clip(cronReason ? cron?.lastError : run?.error) ?? `model ${anyReason}`,
|
|
83
|
+
atMs,
|
|
84
|
+
model,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (run && ((run.fallbackSteps ?? 0) > 0 || (run.modelChanges ?? 0) > 0)) {
|
|
88
|
+
return {
|
|
89
|
+
level: 'degraded',
|
|
90
|
+
reason: 'model_fallback',
|
|
91
|
+
detail: `${run.fallbackSteps ?? 0} fallback step(s), ${run.modelChanges ?? 0} model change(s) in the last beat`,
|
|
92
|
+
atMs,
|
|
93
|
+
model,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
return { level: 'ok', reason: null, detail: null, atMs, model };
|
|
97
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData } from './types.js';
|
|
1
|
+
import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData, AgentRunLifecycleData } from './types.js';
|
|
2
2
|
import type { TradingMode } from '@reefclaw/shared';
|
|
3
3
|
import type { ConnectorUpdateOutcome } from './providers/connector-update.js';
|
|
4
4
|
/** Per-venue credential shapes for the operator set/test RPCs. Binance is an
|
|
@@ -126,6 +126,10 @@ export interface ProviderEvents {
|
|
|
126
126
|
tradingModeUpdate: (data: TradingModeData) => void;
|
|
127
127
|
/** Decision trace update (Phase 9c) */
|
|
128
128
|
decisionTraceUpdate: (data: DecisionTraceData) => void;
|
|
129
|
+
/** An agent turn started/ended/errored, with the gateway's session key so a
|
|
130
|
+
* heartbeat (cron) turn is distinguishable from a chat turn. Local-only
|
|
131
|
+
* (not forwarded to the relay); consumed by the heartbeat flight recorder. */
|
|
132
|
+
agentRunLifecycle: (data: AgentRunLifecycleData) => void;
|
|
129
133
|
}
|
|
130
134
|
export type ProviderEventName = keyof ProviderEvents;
|
|
131
135
|
/**
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { OpenClawProvider, ProviderEvents, ProviderEventName, ExchangeCredentialsRequest, HlProvisionOutcome, HlAgentWalletStatusOutcome, HlSubmitApprovalOutcome } from '../provider.js';
|
|
2
|
-
import type { EmergencyAction, ReconciliationSnapshot, TradingMode } from '../types.js';
|
|
2
|
+
import type { EmergencyAction, ReconciliationSnapshot, TradingMode, AgentRunLifecycleData } from '../types.js';
|
|
3
|
+
import type { CronRunLogLite, HeartbeatRunRecord } from '../heartbeat-transcript.js';
|
|
3
4
|
import type { GatewayConfig } from '../gateway/gateway-config.js';
|
|
4
5
|
import { type GetBracketConfigOutcome, type SetBracketRequirementOutcome, type BracketRequirementFlag, type SetTradingModeOutcome, type SetExchangeCredentialsOutcome, type TestExchangeCredentialsOutcome, type ClearExchangeCredentialsOutcome } from './onboarding-commands.js';
|
|
5
6
|
import { type ConnectorUpdateOutcome } from './connector-update.js';
|
|
@@ -56,6 +57,14 @@ export declare class GatewayProvider implements OpenClawProvider {
|
|
|
56
57
|
private heartbeatHealth?;
|
|
57
58
|
private heartbeatHealthReadAtMs;
|
|
58
59
|
private heartbeatHealthRefreshing;
|
|
60
|
+
/** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
|
|
61
|
+
private heartbeatJobId?;
|
|
62
|
+
/** Last recorded heartbeat run (flight recorder) — dashboard summary. */
|
|
63
|
+
private lastHeartbeat?;
|
|
64
|
+
/** Model-health inputs from the last recorded run (kept apart from the
|
|
65
|
+
* summary so the verdict can be re-derived when cron state changes). */
|
|
66
|
+
private lastHeartbeatRunForHealth?;
|
|
67
|
+
private modelHealth?;
|
|
59
68
|
private lastTicker;
|
|
60
69
|
private positions;
|
|
61
70
|
private balance;
|
|
@@ -395,6 +404,21 @@ export declare class GatewayProvider implements OpenClawProvider {
|
|
|
395
404
|
* refresh is in flight. Fire-and-forget: the RPC completion re-emits
|
|
396
405
|
* agent_state, so callers stay synchronous. */
|
|
397
406
|
private maybeRefreshHeartbeatHealth;
|
|
407
|
+
/** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
|
|
408
|
+
* first), normalised to the recorder's shape. Returns null when the RPC or
|
|
409
|
+
* the job id is unavailable (older gateway / scope) — the recorder then
|
|
410
|
+
* falls back to transcript-only records. Token counts here are RAW counts. */
|
|
411
|
+
fetchCronRuns(limit?: number): Promise<CronRunLogLite[] | null>;
|
|
412
|
+
private loggedCronRunsProbe;
|
|
413
|
+
/** A heartbeat run was recorded — keep a compact summary for the dashboard
|
|
414
|
+
* header, re-derive model health, and re-emit agent_state. */
|
|
415
|
+
noteHeartbeatRun(record: HeartbeatRunRecord): void;
|
|
416
|
+
/** Active trading book for the record's mode tag (matches the webapp's
|
|
417
|
+
* modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
|
|
418
|
+
getTradingBook(): 'paper' | 'live';
|
|
419
|
+
/** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
|
|
420
|
+
onAgentRunLifecycle(listener: (d: AgentRunLifecycleData) => void): () => void;
|
|
421
|
+
private recomputeModelHealth;
|
|
398
422
|
/** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
|
|
399
423
|
private mergeAgentIdentity;
|
|
400
424
|
/**
|
|
@@ -6,6 +6,7 @@ import { join } from 'path';
|
|
|
6
6
|
import { homedir } from 'os';
|
|
7
7
|
import { logger, formatError } from '../logger.js';
|
|
8
8
|
import { isTradingMode } from '../types.js';
|
|
9
|
+
import { deriveModelHealth } from '../model-health.js';
|
|
9
10
|
import { handshakeLacksWriteScope, relaxGatewayDeviceAuth } from '../config.js';
|
|
10
11
|
import { toIntelSymbol } from '@reefclaw/shared';
|
|
11
12
|
import { GatewayHttpClient } from '../gateway/gateway-http-client.js';
|
|
@@ -175,6 +176,14 @@ export class GatewayProvider {
|
|
|
175
176
|
heartbeatHealth;
|
|
176
177
|
heartbeatHealthReadAtMs = 0;
|
|
177
178
|
heartbeatHealthRefreshing = false;
|
|
179
|
+
/** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
|
|
180
|
+
heartbeatJobId;
|
|
181
|
+
/** Last recorded heartbeat run (flight recorder) — dashboard summary. */
|
|
182
|
+
lastHeartbeat;
|
|
183
|
+
/** Model-health inputs from the last recorded run (kept apart from the
|
|
184
|
+
* summary so the verdict can be re-derived when cron state changes). */
|
|
185
|
+
lastHeartbeatRunForHealth;
|
|
186
|
+
modelHealth;
|
|
178
187
|
lastTicker = null;
|
|
179
188
|
positions = [];
|
|
180
189
|
balance = { currency: 'USDT', total: 0, available: 0, locked: 0 };
|
|
@@ -318,6 +327,7 @@ export class GatewayProvider {
|
|
|
318
327
|
shadowComparison: new Set(),
|
|
319
328
|
tradingModeUpdate: new Set(),
|
|
320
329
|
decisionTraceUpdate: new Set(),
|
|
330
|
+
agentRunLifecycle: new Set(),
|
|
321
331
|
};
|
|
322
332
|
constructor(config) {
|
|
323
333
|
this.config = config;
|
|
@@ -1368,6 +1378,21 @@ export class GatewayProvider {
|
|
|
1368
1378
|
onAgentEvent(payload) {
|
|
1369
1379
|
if (!this.started || !this.eventParser)
|
|
1370
1380
|
return;
|
|
1381
|
+
// Turn lifecycle WITH the session key — the heartbeat flight recorder
|
|
1382
|
+
// keys off `agent:main:cron:…` to scan the beat's transcript right after
|
|
1383
|
+
// it ends. Fired before parsing so it never depends on parser state.
|
|
1384
|
+
if (payload.stream === 'lifecycle') {
|
|
1385
|
+
const phase = payload.data?.phase;
|
|
1386
|
+
if (phase === 'start' || phase === 'end' || phase === 'error') {
|
|
1387
|
+
this.fire('agentRunLifecycle', {
|
|
1388
|
+
runId: payload.runId,
|
|
1389
|
+
phase,
|
|
1390
|
+
...(payload.sessionKey ? { sessionKey: payload.sessionKey } : {}),
|
|
1391
|
+
...(payload.sessionId ? { sessionId: payload.sessionId } : {}),
|
|
1392
|
+
at: Date.now(),
|
|
1393
|
+
});
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1371
1396
|
const parsed = this.eventParser.parseAgentEvent(payload);
|
|
1372
1397
|
for (const evt of parsed) {
|
|
1373
1398
|
// Track order/cancel rates for risk computation
|
|
@@ -2118,6 +2143,9 @@ export class GatewayProvider {
|
|
|
2118
2143
|
this.heartbeatSeconds = secs;
|
|
2119
2144
|
this.heartbeatReadAtMs = Date.now();
|
|
2120
2145
|
}
|
|
2146
|
+
const jobId = hb?.id;
|
|
2147
|
+
if (typeof jobId === 'string' && jobId)
|
|
2148
|
+
this.heartbeatJobId = jobId;
|
|
2121
2149
|
const state = hb?.state;
|
|
2122
2150
|
if (!state || typeof state !== 'object')
|
|
2123
2151
|
return;
|
|
@@ -2126,18 +2154,23 @@ export class GatewayProvider {
|
|
|
2126
2154
|
consecutiveErrors: typeof s.consecutiveErrors === 'number' && s.consecutiveErrors >= 0 ? s.consecutiveErrors : 0,
|
|
2127
2155
|
lastRunAtMs: typeof s.lastRunAtMs === 'number' ? s.lastRunAtMs : undefined,
|
|
2128
2156
|
lastRunStatus: typeof s.lastRunStatus === 'string' ? s.lastRunStatus : undefined,
|
|
2157
|
+
lastErrorReason: typeof s.lastErrorReason === 'string' ? s.lastErrorReason : undefined,
|
|
2158
|
+
lastError: typeof s.lastError === 'string' ? s.lastError.slice(0, 300) : undefined,
|
|
2129
2159
|
};
|
|
2130
2160
|
const prev = this.heartbeatHealth;
|
|
2131
2161
|
const changed = !prev ||
|
|
2132
2162
|
prev.consecutiveErrors !== next.consecutiveErrors ||
|
|
2133
2163
|
prev.lastRunAtMs !== next.lastRunAtMs ||
|
|
2134
|
-
prev.lastRunStatus !== next.lastRunStatus
|
|
2164
|
+
prev.lastRunStatus !== next.lastRunStatus ||
|
|
2165
|
+
prev.lastErrorReason !== next.lastErrorReason;
|
|
2135
2166
|
this.heartbeatHealth = next;
|
|
2136
2167
|
this.heartbeatHealthReadAtMs = Date.now();
|
|
2137
2168
|
if (changed) {
|
|
2138
2169
|
if (next.consecutiveErrors > 0) {
|
|
2139
|
-
logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}`
|
|
2170
|
+
logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}` +
|
|
2171
|
+
(next.lastErrorReason ? ` reason=${next.lastErrorReason}` : ''));
|
|
2140
2172
|
}
|
|
2173
|
+
this.recomputeModelHealth();
|
|
2141
2174
|
this.emitAgentState();
|
|
2142
2175
|
}
|
|
2143
2176
|
}
|
|
@@ -2158,6 +2191,134 @@ export class GatewayProvider {
|
|
|
2158
2191
|
return;
|
|
2159
2192
|
void this.refreshHeartbeatHealth();
|
|
2160
2193
|
}
|
|
2194
|
+
// ---- Heartbeat flight recorder hooks (skill/src/heartbeat-runs.ts) ----
|
|
2195
|
+
/** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
|
|
2196
|
+
* first), normalised to the recorder's shape. Returns null when the RPC or
|
|
2197
|
+
* the job id is unavailable (older gateway / scope) — the recorder then
|
|
2198
|
+
* falls back to transcript-only records. Token counts here are RAW counts. */
|
|
2199
|
+
async fetchCronRuns(limit = 25) {
|
|
2200
|
+
const ws = this.wsClient;
|
|
2201
|
+
if (!ws)
|
|
2202
|
+
return null;
|
|
2203
|
+
if (!this.heartbeatJobId) {
|
|
2204
|
+
await this.refreshHeartbeatHealth();
|
|
2205
|
+
if (!this.heartbeatJobId)
|
|
2206
|
+
return null;
|
|
2207
|
+
}
|
|
2208
|
+
let resp;
|
|
2209
|
+
try {
|
|
2210
|
+
resp = await ws.sendRpc('cron.runs', { jobId: this.heartbeatJobId, limit, sortDir: 'desc' });
|
|
2211
|
+
}
|
|
2212
|
+
catch (err) {
|
|
2213
|
+
logger.debug(TAG, `cron.runs RPC failed: ${err instanceof Error ? err.message : String(err)}`);
|
|
2214
|
+
return null;
|
|
2215
|
+
}
|
|
2216
|
+
const entries = resp?.entries;
|
|
2217
|
+
if (!Array.isArray(entries)) {
|
|
2218
|
+
if (!this.loggedCronRunsProbe) {
|
|
2219
|
+
this.loggedCronRunsProbe = true;
|
|
2220
|
+
const keys = resp && typeof resp === 'object' ? Object.keys(resp).join(',') : typeof resp;
|
|
2221
|
+
logger.info(TAG, `cron.runs probe: unexpected shape keys=[${keys}] — token counts fall back to transcripts`);
|
|
2222
|
+
}
|
|
2223
|
+
return null;
|
|
2224
|
+
}
|
|
2225
|
+
if (!this.loggedCronRunsProbe) {
|
|
2226
|
+
this.loggedCronRunsProbe = true;
|
|
2227
|
+
const first = entries[0] && typeof entries[0] === 'object' ? Object.keys(entries[0]).join(',') : '(empty)';
|
|
2228
|
+
logger.info(TAG, `cron.runs probe: ${entries.length} entries, keys=[${first}]`);
|
|
2229
|
+
}
|
|
2230
|
+
const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
|
|
2231
|
+
const str = (v) => (typeof v === 'string' && v ? v : null);
|
|
2232
|
+
return entries.map((raw) => {
|
|
2233
|
+
const e = (raw && typeof raw === 'object' ? raw : {});
|
|
2234
|
+
const u = (e.usage && typeof e.usage === 'object' ? e.usage : null);
|
|
2235
|
+
return {
|
|
2236
|
+
sessionId: str(e.sessionId),
|
|
2237
|
+
runId: str(e.runId),
|
|
2238
|
+
status: str(e.status),
|
|
2239
|
+
error: str(e.error),
|
|
2240
|
+
errorReason: str(e.errorReason),
|
|
2241
|
+
runAtMs: num(e.runAtMs) ?? num(e.ts),
|
|
2242
|
+
durationMs: num(e.durationMs),
|
|
2243
|
+
model: str(e.model),
|
|
2244
|
+
provider: str(e.provider),
|
|
2245
|
+
usage: u
|
|
2246
|
+
? {
|
|
2247
|
+
input: num(u.input_tokens) ?? num(u.input),
|
|
2248
|
+
output: num(u.output_tokens) ?? num(u.output),
|
|
2249
|
+
cacheRead: num(u.cache_read_tokens) ?? num(u.cacheRead),
|
|
2250
|
+
cacheWrite: num(u.cache_write_tokens) ?? num(u.cacheWrite),
|
|
2251
|
+
total: num(u.total_tokens) ?? num(u.total) ?? num(u.totalTokens),
|
|
2252
|
+
}
|
|
2253
|
+
: null,
|
|
2254
|
+
};
|
|
2255
|
+
});
|
|
2256
|
+
}
|
|
2257
|
+
loggedCronRunsProbe = false;
|
|
2258
|
+
/** A heartbeat run was recorded — keep a compact summary for the dashboard
|
|
2259
|
+
* header, re-derive model health, and re-emit agent_state. */
|
|
2260
|
+
noteHeartbeatRun(record) {
|
|
2261
|
+
this.lastHeartbeat = {
|
|
2262
|
+
runId: record.runId,
|
|
2263
|
+
startedAtMs: record.startedAtMs,
|
|
2264
|
+
endedAtMs: record.endedAtMs,
|
|
2265
|
+
durationMs: record.durationMs,
|
|
2266
|
+
status: record.status,
|
|
2267
|
+
toolCallCount: record.toolCallCount,
|
|
2268
|
+
toolErrorCount: record.toolErrorCount,
|
|
2269
|
+
tokensTotal: record.tokens.total,
|
|
2270
|
+
tokensInput: record.tokens.input,
|
|
2271
|
+
tokensOutput: record.tokens.output,
|
|
2272
|
+
model: record.model,
|
|
2273
|
+
};
|
|
2274
|
+
this.lastHeartbeatRunForHealth = {
|
|
2275
|
+
status: record.status,
|
|
2276
|
+
errorReason: record.errorReason,
|
|
2277
|
+
error: record.error,
|
|
2278
|
+
model: record.model,
|
|
2279
|
+
endedAtMs: record.endedAtMs,
|
|
2280
|
+
fallbackSteps: record.modelEvents.filter((e) => e.kind === 'fallback_step').length,
|
|
2281
|
+
modelChanges: record.modelEvents.filter((e) => e.kind === 'model_change').length,
|
|
2282
|
+
};
|
|
2283
|
+
this.recomputeModelHealth();
|
|
2284
|
+
this.emitAgentState();
|
|
2285
|
+
}
|
|
2286
|
+
/** Active trading book for the record's mode tag (matches the webapp's
|
|
2287
|
+
* modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
|
|
2288
|
+
getTradingBook() {
|
|
2289
|
+
return this.tradingMode === 'LIVE' || this.tradingMode === 'MICRO_LIVE' ? 'live' : 'paper';
|
|
2290
|
+
}
|
|
2291
|
+
/** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
|
|
2292
|
+
onAgentRunLifecycle(listener) {
|
|
2293
|
+
this.on('agentRunLifecycle', listener);
|
|
2294
|
+
return () => this.off('agentRunLifecycle', listener);
|
|
2295
|
+
}
|
|
2296
|
+
recomputeModelHealth() {
|
|
2297
|
+
const hb = this.heartbeatHealth;
|
|
2298
|
+
const next = deriveModelHealth({
|
|
2299
|
+
lastRun: this.lastHeartbeatRunForHealth ?? null,
|
|
2300
|
+
cron: hb
|
|
2301
|
+
? {
|
|
2302
|
+
consecutiveErrors: hb.consecutiveErrors,
|
|
2303
|
+
lastErrorReason: hb.lastErrorReason ?? null,
|
|
2304
|
+
lastError: hb.lastError ?? null,
|
|
2305
|
+
lastRunStatus: hb.lastRunStatus ?? null,
|
|
2306
|
+
lastRunAtMs: hb.lastRunAtMs ?? null,
|
|
2307
|
+
}
|
|
2308
|
+
: null,
|
|
2309
|
+
now: Date.now(),
|
|
2310
|
+
});
|
|
2311
|
+
const prev = this.modelHealth;
|
|
2312
|
+
if (!prev || prev.level !== next.level || prev.reason !== next.reason) {
|
|
2313
|
+
if (next.level !== 'ok') {
|
|
2314
|
+
logger.warn(TAG, `Model health ${next.level}: ${next.reason ?? ''} ${next.detail ?? ''}`.trim());
|
|
2315
|
+
}
|
|
2316
|
+
else if (prev && prev.level !== 'ok') {
|
|
2317
|
+
logger.info(TAG, 'Model health recovered: ok');
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
this.modelHealth = next;
|
|
2321
|
+
}
|
|
2161
2322
|
/** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
|
|
2162
2323
|
mergeAgentIdentity(found) {
|
|
2163
2324
|
let changed = false;
|
|
@@ -2349,6 +2510,10 @@ export class GatewayProvider {
|
|
|
2349
2510
|
},
|
|
2350
2511
|
}
|
|
2351
2512
|
: {}),
|
|
2513
|
+
// Flight recorder: absent until the first beat is recorded → payload
|
|
2514
|
+
// stays byte-identical to the pre-feature shape until then.
|
|
2515
|
+
...(this.lastHeartbeat ? { lastHeartbeat: this.lastHeartbeat } : {}),
|
|
2516
|
+
...(this.modelHealth ? { modelHealth: this.modelHealth } : {}),
|
|
2352
2517
|
},
|
|
2353
2518
|
};
|
|
2354
2519
|
}
|
package/assets/bridge/types.d.ts
CHANGED
|
@@ -133,10 +133,51 @@ export interface AgentStateData {
|
|
|
133
133
|
/** Last run outcome, e.g. "ok" | "error". */
|
|
134
134
|
lastRunStatus?: string;
|
|
135
135
|
};
|
|
136
|
+
/** Compact summary of the last recorded heartbeat run (flight recorder,
|
|
137
|
+
* skill/src/heartbeat-runs.ts). Absent until the first beat is
|
|
138
|
+
* recorded after start — byte-identical to pre-feature until then. */
|
|
139
|
+
lastHeartbeat?: LastHeartbeatSummary;
|
|
140
|
+
/** Is the LLM behind the agent answering? Derived skill-side from the
|
|
141
|
+
* cron job's FailoverReason + the last recorded beat
|
|
142
|
+
* (skill/src/model-health.ts). Absent until first derived. */
|
|
143
|
+
modelHealth?: ModelHealthSummary;
|
|
136
144
|
};
|
|
137
145
|
};
|
|
138
146
|
timestamp: string;
|
|
139
147
|
}
|
|
148
|
+
/** Dashboard-facing summary of one heartbeat run (the full record lives in
|
|
149
|
+
* the webapp's heartbeat_runs table). Token counts are RAW COUNTS — never
|
|
150
|
+
* translated to a currency here or downstream. */
|
|
151
|
+
export interface LastHeartbeatSummary {
|
|
152
|
+
runId: string;
|
|
153
|
+
startedAtMs: number;
|
|
154
|
+
endedAtMs: number | null;
|
|
155
|
+
durationMs: number | null;
|
|
156
|
+
status: 'ok' | 'error' | 'incomplete' | 'unknown';
|
|
157
|
+
toolCallCount: number;
|
|
158
|
+
toolErrorCount: number;
|
|
159
|
+
tokensTotal: number | null;
|
|
160
|
+
tokensInput: number | null;
|
|
161
|
+
tokensOutput: number | null;
|
|
162
|
+
model: string | null;
|
|
163
|
+
}
|
|
164
|
+
export interface ModelHealthSummary {
|
|
165
|
+
level: 'ok' | 'degraded' | 'error';
|
|
166
|
+
reason: string | null;
|
|
167
|
+
detail: string | null;
|
|
168
|
+
atMs: number;
|
|
169
|
+
model: string | null;
|
|
170
|
+
}
|
|
171
|
+
/** An agent turn's lifecycle as the gateway reports it, WITH the session key
|
|
172
|
+
* (an isolated cron beat runs under `agent:main:cron:<jobId>:run:<sid>`),
|
|
173
|
+
* so consumers can tell a heartbeat turn from a chat turn. */
|
|
174
|
+
export interface AgentRunLifecycleData {
|
|
175
|
+
runId: string;
|
|
176
|
+
phase: 'start' | 'end' | 'error';
|
|
177
|
+
sessionKey?: string;
|
|
178
|
+
sessionId?: string;
|
|
179
|
+
at: number;
|
|
180
|
+
}
|
|
140
181
|
export interface RiskLimits {
|
|
141
182
|
position: {
|
|
142
183
|
maxPositionSize: number;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Re-export shim for the plugin-side strategy evaluator (facts-out).
|
|
2
2
|
//
|
|
3
|
-
// The generated
|
|
3
|
+
// The generated evaluator-core copies (registry, strategy-adapter, direction/entry/
|
|
4
4
|
// stop-rules) import their types from `./types.js`. The canonical definitions
|
|
5
5
|
// live in @reefclaw/shared (type-only → erased at runtime), so this shim makes
|
|
6
6
|
// those relative imports resolve inside the plugin tree without pulling any
|
|
@@ -150,7 +150,7 @@ export interface PositionMetadata {
|
|
|
150
150
|
/** The agent's stated profit-realization plan, pinned at entry (indication,
|
|
151
151
|
* not an enforced mechanic). */
|
|
152
152
|
realizationRule?: RealizationRule;
|
|
153
|
-
/** Re-entry cooldown gate
|
|
153
|
+
/** Re-entry cooldown gate evaluation, present ONLY when the gate TRIGGERED on this
|
|
154
154
|
* entry (a same-symbol loss within the window) and the order fired anyway
|
|
155
155
|
* (shadow/observe). Journaled to position_entries.metadata.reentry_cooldown
|
|
156
156
|
* so the shadow soak can measure the would-block cohort's forward outcomes
|
|
@@ -1113,7 +1113,7 @@ export async function createOrderTool(args, deps) {
|
|
|
1113
1113
|
const n = num(v);
|
|
1114
1114
|
return n != null ? Math.max(min, Math.min(max, n)) : undefined;
|
|
1115
1115
|
};
|
|
1116
|
-
// Journal-tag a TRIGGERED cooldown
|
|
1116
|
+
// Journal-tag a TRIGGERED cooldown evaluation (shadow/observe fire-anyway cohort —
|
|
1117
1117
|
// the measurement rows the promote-to-enforce decision reads).
|
|
1118
1118
|
const reentryCooldownTag = reentryCooldownEval?.triggered
|
|
1119
1119
|
? {
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
// Mode semantics mirror the exit gate (docs/CLAUDE/exit-gate.md):
|
|
23
23
|
// off — not evaluated; byte-identical to the pre-gate path.
|
|
24
24
|
// shadow — evaluated + logged + journal-tagged; never affects the order.
|
|
25
|
-
// observe — as shadow, but a triggered
|
|
26
|
-
// enforce — a triggered
|
|
25
|
+
// observe — as shadow, but a triggered evaluation logs at WARN (operator-visible).
|
|
26
|
+
// enforce — a triggered evaluation hard-rejects create_order with a recovery hint.
|
|
27
27
|
export function evaluateReentryCooldown(inputs) {
|
|
28
28
|
const base = {
|
|
29
29
|
mode: inputs.mode,
|