@reefclaw/connect 0.1.37 → 0.1.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,6 +17,7 @@ import { GatewayProvider } from './providers/gateway.js';
17
17
  import { resolveConfig } from './config.js';
18
18
  import { resolveGatewayConfig, validateGatewayConfig } from './gateway/gateway-config.js';
19
19
  import { ShockWakePoller } from './shock-wake.js';
20
+ import { HeartbeatRunRecorder } from './heartbeat-runs.js';
20
21
  import { runSetup } from './setup.js';
21
22
  import { resolveRelayInstanceId } from './utils/instance-id.js';
22
23
  const TAG = 'main';
@@ -244,6 +245,17 @@ async function main() {
244
245
  // lane is alive — the gateway ack alone cannot (swallowed-turn class).
245
246
  provider.on('chatMessageStart', () => shockWake.noteAssistantActivity());
246
247
  }
248
+ // Heartbeat flight recorder — records what every beat actually DID (tool
249
+ // calls + results, report, model, raw token counts) from the isolated cron
250
+ // session's transcript, posts it to the webapp journal, and feeds the
251
+ // dashboard's last-beat summary + model-health dot. Gateway provider only
252
+ // (the mock has no OpenClaw sessions dir). Kill-switch: RC_HEARTBEAT_RECORDER=off.
253
+ const recorderOff = (process.env.RC_HEARTBEAT_RECORDER ?? 'on').trim().toLowerCase() === 'off';
254
+ const heartbeatRecorder = provider instanceof GatewayProvider && !recorderOff
255
+ ? new HeartbeatRunRecorder({ source: provider, token })
256
+ : null;
257
+ if (recorderOff)
258
+ logger.info(TAG, 'Heartbeat flight recorder disabled (RC_HEARTBEAT_RECORDER=off)');
247
259
  // Handle graceful shutdown
248
260
  let shuttingDown = false;
249
261
  async function shutdown(signal) {
@@ -259,6 +271,7 @@ async function main() {
259
271
  catch { /* best-effort */ }
260
272
  }
261
273
  shockWake?.stop();
274
+ heartbeatRecorder?.stop();
262
275
  bridge.stop();
263
276
  // Allow 1s for final cleanup before force exit
264
277
  setTimeout(() => process.exit(0), 1000);
@@ -269,5 +282,6 @@ async function main() {
269
282
  logger.info(TAG, `Starting ReefClaw skill (provider=${args.provider}, relay=${relayUrl})`);
270
283
  bridge.start();
271
284
  shockWake?.start();
285
+ heartbeatRecorder?.start();
272
286
  }
273
287
  main();
@@ -0,0 +1,37 @@
1
+ export type ModelHealthLevel = 'ok' | 'degraded' | 'error';
2
+ export interface ModelHealth {
3
+ level: ModelHealthLevel;
4
+ /** Machine reason (a FailoverReason, 'model_fallback', 'consecutive_errors'). */
5
+ reason: string | null;
6
+ /** Human detail (error text, fallback description), redacted upstream. */
7
+ detail: string | null;
8
+ /** Epoch ms of the evidence this verdict rests on. */
9
+ atMs: number;
10
+ model: string | null;
11
+ }
12
+ /** FailoverReasons that do not self-heal — the operator must act. */
13
+ export declare const HARD_REASONS: ReadonlySet<string>;
14
+ /** FailoverReasons that usually clear on their own (quota windows, outages). */
15
+ export declare const SOFT_REASONS: ReadonlySet<string>;
16
+ export interface ModelHealthInputs {
17
+ /** The most recently recorded heartbeat run (flight recorder). */
18
+ lastRun?: {
19
+ status: string;
20
+ errorReason?: string | null;
21
+ error?: string | null;
22
+ model?: string | null;
23
+ endedAtMs?: number | null;
24
+ fallbackSteps?: number;
25
+ modelChanges?: number;
26
+ } | null;
27
+ /** The heartbeat job's cron state (`cron.list`). */
28
+ cron?: {
29
+ consecutiveErrors: number;
30
+ lastErrorReason?: string | null;
31
+ lastError?: string | null;
32
+ lastRunStatus?: string | null;
33
+ lastRunAtMs?: number | null;
34
+ } | null;
35
+ now: number;
36
+ }
37
+ export declare function deriveModelHealth(i: ModelHealthInputs): ModelHealth;
@@ -0,0 +1,97 @@
1
+ // Model health — is the LLM behind the agent actually answering? (pure)
2
+ //
3
+ // Two incident classes motivated this: a Codex quota death (2026-06-07) and a
4
+ // self-inflicted harness/provider mismatch (2026-07-26) both took the agent
5
+ // dark for hours-to-days while the dashboard looked healthy. The gateway
6
+ // classifies every failed model call with a FailoverReason (vendored
7
+ // `embedded-agent-helpers/types.ts`) and persists it on the cron job state
8
+ // (`lastErrorReason`) and the run log (`errorReason`); auth failures fall back
9
+ // SILENTLY to the next model (docs/CLAUDE/agent-runtime.md §4), format
10
+ // failures kill every turn with no fallback (§1). This derives one dot from
11
+ // those signals so the operator sees "model degraded/erroring" instead of
12
+ // inferring it from a stalled last-decision clock.
13
+ //
14
+ // Levels:
15
+ // error — a reason that will not self-heal (auth_permanent, format,
16
+ // model_not_found, billing), or ≥2 consecutive model-classified
17
+ // failures.
18
+ // degraded — a recoverable model reason on the last run (auth, rate_limit,
19
+ // overloaded, server_error, timeout, …) or a fallback / model
20
+ // change observed inside the last beat.
21
+ // ok — nothing model-related is wrong. A run error WITHOUT a
22
+ // FailoverReason (e.g. a delivery-target error) stays 'ok' here:
23
+ // the heartbeat banner owns non-model failures.
24
+ /** FailoverReasons that do not self-heal — the operator must act. */
25
+ export const HARD_REASONS = new Set(['auth_permanent', 'format', 'model_not_found', 'billing']);
26
+ /** FailoverReasons that usually clear on their own (quota windows, outages). */
27
+ export const SOFT_REASONS = new Set([
28
+ 'auth',
29
+ 'rate_limit',
30
+ 'overloaded',
31
+ 'server_error',
32
+ 'timeout',
33
+ 'session_expired',
34
+ 'empty_response',
35
+ 'no_error_details',
36
+ 'unclassified',
37
+ 'unknown',
38
+ ]);
39
+ function clip(s, n = 200) {
40
+ if (typeof s !== 'string')
41
+ return null;
42
+ const t = s.trim();
43
+ if (!t)
44
+ return null;
45
+ return t.length > n ? `${t.slice(0, n)}…` : t;
46
+ }
47
+ export function deriveModelHealth(i) {
48
+ const run = i.lastRun ?? null;
49
+ const cron = i.cron ?? null;
50
+ const model = run?.model ?? null;
51
+ const cronReason = cron?.lastRunStatus === 'error' ? cron.lastErrorReason ?? null : null;
52
+ const runReason = run?.status === 'error' ? run.errorReason ?? null : null;
53
+ const consecutive = cron?.consecutiveErrors ?? 0;
54
+ // Prefer whichever piece of evidence is newer.
55
+ const cronAt = cron?.lastRunAtMs ?? 0;
56
+ const runAt = run?.endedAtMs ?? 0;
57
+ const atMs = Math.max(cronAt, runAt) || i.now;
58
+ const hard = [cronReason, runReason].find((r) => r && HARD_REASONS.has(r)) ?? null;
59
+ if (hard) {
60
+ return {
61
+ level: 'error',
62
+ reason: hard,
63
+ detail: clip(cronReason === hard ? cron?.lastError : run?.error) ?? `model ${hard}`,
64
+ atMs,
65
+ model,
66
+ };
67
+ }
68
+ const anyReason = cronReason ?? runReason;
69
+ if (anyReason && consecutive >= 2) {
70
+ return {
71
+ level: 'error',
72
+ reason: 'consecutive_errors',
73
+ detail: clip(cron?.lastError ?? run?.error) ?? `${consecutive} consecutive model failures (${anyReason})`,
74
+ atMs,
75
+ model,
76
+ };
77
+ }
78
+ if (anyReason && (SOFT_REASONS.has(anyReason) || !HARD_REASONS.has(anyReason))) {
79
+ return {
80
+ level: 'degraded',
81
+ reason: anyReason,
82
+ detail: clip(cronReason ? cron?.lastError : run?.error) ?? `model ${anyReason}`,
83
+ atMs,
84
+ model,
85
+ };
86
+ }
87
+ if (run && ((run.fallbackSteps ?? 0) > 0 || (run.modelChanges ?? 0) > 0)) {
88
+ return {
89
+ level: 'degraded',
90
+ reason: 'model_fallback',
91
+ detail: `${run.fallbackSteps ?? 0} fallback step(s), ${run.modelChanges ?? 0} model change(s) in the last beat`,
92
+ atMs,
93
+ model,
94
+ };
95
+ }
96
+ return { level: 'ok', reason: null, detail: null, atMs, model };
97
+ }
@@ -1,4 +1,4 @@
1
- import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData } from './types.js';
1
+ import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData, AgentRunLifecycleData } from './types.js';
2
2
  import type { TradingMode } from '@reefclaw/shared';
3
3
  import type { ConnectorUpdateOutcome } from './providers/connector-update.js';
4
4
  /** Per-venue credential shapes for the operator set/test RPCs. Binance is an
@@ -126,6 +126,10 @@ export interface ProviderEvents {
126
126
  tradingModeUpdate: (data: TradingModeData) => void;
127
127
  /** Decision trace update (Phase 9c) */
128
128
  decisionTraceUpdate: (data: DecisionTraceData) => void;
129
+ /** An agent turn started/ended/errored, with the gateway's session key so a
130
+ * heartbeat (cron) turn is distinguishable from a chat turn. Local-only
131
+ * (not forwarded to the relay); consumed by the heartbeat flight recorder. */
132
+ agentRunLifecycle: (data: AgentRunLifecycleData) => void;
129
133
  }
130
134
  export type ProviderEventName = keyof ProviderEvents;
131
135
  /**
@@ -1,5 +1,6 @@
1
1
  import type { OpenClawProvider, ProviderEvents, ProviderEventName, ExchangeCredentialsRequest, HlProvisionOutcome, HlAgentWalletStatusOutcome, HlSubmitApprovalOutcome } from '../provider.js';
2
- import type { EmergencyAction, ReconciliationSnapshot, TradingMode } from '../types.js';
2
+ import type { EmergencyAction, ReconciliationSnapshot, TradingMode, AgentRunLifecycleData } from '../types.js';
3
+ import type { CronRunLogLite, HeartbeatRunRecord } from '../heartbeat-transcript.js';
3
4
  import type { GatewayConfig } from '../gateway/gateway-config.js';
4
5
  import { type GetBracketConfigOutcome, type SetBracketRequirementOutcome, type BracketRequirementFlag, type SetTradingModeOutcome, type SetExchangeCredentialsOutcome, type TestExchangeCredentialsOutcome, type ClearExchangeCredentialsOutcome } from './onboarding-commands.js';
5
6
  import { type ConnectorUpdateOutcome } from './connector-update.js';
@@ -56,6 +57,14 @@ export declare class GatewayProvider implements OpenClawProvider {
56
57
  private heartbeatHealth?;
57
58
  private heartbeatHealthReadAtMs;
58
59
  private heartbeatHealthRefreshing;
60
+ /** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
61
+ private heartbeatJobId?;
62
+ /** Last recorded heartbeat run (flight recorder) — dashboard summary. */
63
+ private lastHeartbeat?;
64
+ /** Model-health inputs from the last recorded run (kept apart from the
65
+ * summary so the verdict can be re-derived when cron state changes). */
66
+ private lastHeartbeatRunForHealth?;
67
+ private modelHealth?;
59
68
  private lastTicker;
60
69
  private positions;
61
70
  private balance;
@@ -395,6 +404,21 @@ export declare class GatewayProvider implements OpenClawProvider {
395
404
  * refresh is in flight. Fire-and-forget: the RPC completion re-emits
396
405
  * agent_state, so callers stay synchronous. */
397
406
  private maybeRefreshHeartbeatHealth;
407
+ /** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
408
+ * first), normalised to the recorder's shape. Returns null when the RPC or
409
+ * the job id is unavailable (older gateway / scope) — the recorder then
410
+ * falls back to transcript-only records. Token counts here are RAW counts. */
411
+ fetchCronRuns(limit?: number): Promise<CronRunLogLite[] | null>;
412
+ private loggedCronRunsProbe;
413
+ /** A heartbeat run was recorded — keep a compact summary for the dashboard
414
+ * header, re-derive model health, and re-emit agent_state. */
415
+ noteHeartbeatRun(record: HeartbeatRunRecord): void;
416
+ /** Active trading book for the record's mode tag (matches the webapp's
417
+ * modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
418
+ getTradingBook(): 'paper' | 'live';
419
+ /** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
420
+ onAgentRunLifecycle(listener: (d: AgentRunLifecycleData) => void): () => void;
421
+ private recomputeModelHealth;
398
422
  /** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
399
423
  private mergeAgentIdentity;
400
424
  /**
@@ -6,6 +6,7 @@ import { join } from 'path';
6
6
  import { homedir } from 'os';
7
7
  import { logger, formatError } from '../logger.js';
8
8
  import { isTradingMode } from '../types.js';
9
+ import { deriveModelHealth } from '../model-health.js';
9
10
  import { handshakeLacksWriteScope, relaxGatewayDeviceAuth } from '../config.js';
10
11
  import { toIntelSymbol } from '@reefclaw/shared';
11
12
  import { GatewayHttpClient } from '../gateway/gateway-http-client.js';
@@ -175,6 +176,14 @@ export class GatewayProvider {
175
176
  heartbeatHealth;
176
177
  heartbeatHealthReadAtMs = 0;
177
178
  heartbeatHealthRefreshing = false;
179
+ /** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
180
+ heartbeatJobId;
181
+ /** Last recorded heartbeat run (flight recorder) — dashboard summary. */
182
+ lastHeartbeat;
183
+ /** Model-health inputs from the last recorded run (kept apart from the
184
+ * summary so the verdict can be re-derived when cron state changes). */
185
+ lastHeartbeatRunForHealth;
186
+ modelHealth;
178
187
  lastTicker = null;
179
188
  positions = [];
180
189
  balance = { currency: 'USDT', total: 0, available: 0, locked: 0 };
@@ -318,6 +327,7 @@ export class GatewayProvider {
318
327
  shadowComparison: new Set(),
319
328
  tradingModeUpdate: new Set(),
320
329
  decisionTraceUpdate: new Set(),
330
+ agentRunLifecycle: new Set(),
321
331
  };
322
332
  constructor(config) {
323
333
  this.config = config;
@@ -1368,6 +1378,21 @@ export class GatewayProvider {
1368
1378
  onAgentEvent(payload) {
1369
1379
  if (!this.started || !this.eventParser)
1370
1380
  return;
1381
+ // Turn lifecycle WITH the session key — the heartbeat flight recorder
1382
+ // keys off `agent:main:cron:…` to scan the beat's transcript right after
1383
+ // it ends. Fired before parsing so it never depends on parser state.
1384
+ if (payload.stream === 'lifecycle') {
1385
+ const phase = payload.data?.phase;
1386
+ if (phase === 'start' || phase === 'end' || phase === 'error') {
1387
+ this.fire('agentRunLifecycle', {
1388
+ runId: payload.runId,
1389
+ phase,
1390
+ ...(payload.sessionKey ? { sessionKey: payload.sessionKey } : {}),
1391
+ ...(payload.sessionId ? { sessionId: payload.sessionId } : {}),
1392
+ at: Date.now(),
1393
+ });
1394
+ }
1395
+ }
1371
1396
  const parsed = this.eventParser.parseAgentEvent(payload);
1372
1397
  for (const evt of parsed) {
1373
1398
  // Track order/cancel rates for risk computation
@@ -2118,6 +2143,9 @@ export class GatewayProvider {
2118
2143
  this.heartbeatSeconds = secs;
2119
2144
  this.heartbeatReadAtMs = Date.now();
2120
2145
  }
2146
+ const jobId = hb?.id;
2147
+ if (typeof jobId === 'string' && jobId)
2148
+ this.heartbeatJobId = jobId;
2121
2149
  const state = hb?.state;
2122
2150
  if (!state || typeof state !== 'object')
2123
2151
  return;
@@ -2126,18 +2154,23 @@ export class GatewayProvider {
2126
2154
  consecutiveErrors: typeof s.consecutiveErrors === 'number' && s.consecutiveErrors >= 0 ? s.consecutiveErrors : 0,
2127
2155
  lastRunAtMs: typeof s.lastRunAtMs === 'number' ? s.lastRunAtMs : undefined,
2128
2156
  lastRunStatus: typeof s.lastRunStatus === 'string' ? s.lastRunStatus : undefined,
2157
+ lastErrorReason: typeof s.lastErrorReason === 'string' ? s.lastErrorReason : undefined,
2158
+ lastError: typeof s.lastError === 'string' ? s.lastError.slice(0, 300) : undefined,
2129
2159
  };
2130
2160
  const prev = this.heartbeatHealth;
2131
2161
  const changed = !prev ||
2132
2162
  prev.consecutiveErrors !== next.consecutiveErrors ||
2133
2163
  prev.lastRunAtMs !== next.lastRunAtMs ||
2134
- prev.lastRunStatus !== next.lastRunStatus;
2164
+ prev.lastRunStatus !== next.lastRunStatus ||
2165
+ prev.lastErrorReason !== next.lastErrorReason;
2135
2166
  this.heartbeatHealth = next;
2136
2167
  this.heartbeatHealthReadAtMs = Date.now();
2137
2168
  if (changed) {
2138
2169
  if (next.consecutiveErrors > 0) {
2139
- logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}`);
2170
+ logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}` +
2171
+ (next.lastErrorReason ? ` reason=${next.lastErrorReason}` : ''));
2140
2172
  }
2173
+ this.recomputeModelHealth();
2141
2174
  this.emitAgentState();
2142
2175
  }
2143
2176
  }
@@ -2158,6 +2191,134 @@ export class GatewayProvider {
2158
2191
  return;
2159
2192
  void this.refreshHeartbeatHealth();
2160
2193
  }
2194
+ // ---- Heartbeat flight recorder hooks (skill/src/heartbeat-runs.ts) ----
2195
+ /** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
2196
+ * first), normalised to the recorder's shape. Returns null when the RPC or
2197
+ * the job id is unavailable (older gateway / scope) — the recorder then
2198
+ * falls back to transcript-only records. Token counts here are RAW counts. */
2199
+ async fetchCronRuns(limit = 25) {
2200
+ const ws = this.wsClient;
2201
+ if (!ws)
2202
+ return null;
2203
+ if (!this.heartbeatJobId) {
2204
+ await this.refreshHeartbeatHealth();
2205
+ if (!this.heartbeatJobId)
2206
+ return null;
2207
+ }
2208
+ let resp;
2209
+ try {
2210
+ resp = await ws.sendRpc('cron.runs', { jobId: this.heartbeatJobId, limit, sortDir: 'desc' });
2211
+ }
2212
+ catch (err) {
2213
+ logger.debug(TAG, `cron.runs RPC failed: ${err instanceof Error ? err.message : String(err)}`);
2214
+ return null;
2215
+ }
2216
+ const entries = resp?.entries;
2217
+ if (!Array.isArray(entries)) {
2218
+ if (!this.loggedCronRunsProbe) {
2219
+ this.loggedCronRunsProbe = true;
2220
+ const keys = resp && typeof resp === 'object' ? Object.keys(resp).join(',') : typeof resp;
2221
+ logger.info(TAG, `cron.runs probe: unexpected shape keys=[${keys}] — token counts fall back to transcripts`);
2222
+ }
2223
+ return null;
2224
+ }
2225
+ if (!this.loggedCronRunsProbe) {
2226
+ this.loggedCronRunsProbe = true;
2227
+ const first = entries[0] && typeof entries[0] === 'object' ? Object.keys(entries[0]).join(',') : '(empty)';
2228
+ logger.info(TAG, `cron.runs probe: ${entries.length} entries, keys=[${first}]`);
2229
+ }
2230
+ const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
2231
+ const str = (v) => (typeof v === 'string' && v ? v : null);
2232
+ return entries.map((raw) => {
2233
+ const e = (raw && typeof raw === 'object' ? raw : {});
2234
+ const u = (e.usage && typeof e.usage === 'object' ? e.usage : null);
2235
+ return {
2236
+ sessionId: str(e.sessionId),
2237
+ runId: str(e.runId),
2238
+ status: str(e.status),
2239
+ error: str(e.error),
2240
+ errorReason: str(e.errorReason),
2241
+ runAtMs: num(e.runAtMs) ?? num(e.ts),
2242
+ durationMs: num(e.durationMs),
2243
+ model: str(e.model),
2244
+ provider: str(e.provider),
2245
+ usage: u
2246
+ ? {
2247
+ input: num(u.input_tokens) ?? num(u.input),
2248
+ output: num(u.output_tokens) ?? num(u.output),
2249
+ cacheRead: num(u.cache_read_tokens) ?? num(u.cacheRead),
2250
+ cacheWrite: num(u.cache_write_tokens) ?? num(u.cacheWrite),
2251
+ total: num(u.total_tokens) ?? num(u.total) ?? num(u.totalTokens),
2252
+ }
2253
+ : null,
2254
+ };
2255
+ });
2256
+ }
2257
+ loggedCronRunsProbe = false;
2258
+ /** A heartbeat run was recorded — keep a compact summary for the dashboard
2259
+ * header, re-derive model health, and re-emit agent_state. */
2260
+ noteHeartbeatRun(record) {
2261
+ this.lastHeartbeat = {
2262
+ runId: record.runId,
2263
+ startedAtMs: record.startedAtMs,
2264
+ endedAtMs: record.endedAtMs,
2265
+ durationMs: record.durationMs,
2266
+ status: record.status,
2267
+ toolCallCount: record.toolCallCount,
2268
+ toolErrorCount: record.toolErrorCount,
2269
+ tokensTotal: record.tokens.total,
2270
+ tokensInput: record.tokens.input,
2271
+ tokensOutput: record.tokens.output,
2272
+ model: record.model,
2273
+ };
2274
+ this.lastHeartbeatRunForHealth = {
2275
+ status: record.status,
2276
+ errorReason: record.errorReason,
2277
+ error: record.error,
2278
+ model: record.model,
2279
+ endedAtMs: record.endedAtMs,
2280
+ fallbackSteps: record.modelEvents.filter((e) => e.kind === 'fallback_step').length,
2281
+ modelChanges: record.modelEvents.filter((e) => e.kind === 'model_change').length,
2282
+ };
2283
+ this.recomputeModelHealth();
2284
+ this.emitAgentState();
2285
+ }
2286
+ /** Active trading book for the record's mode tag (matches the webapp's
2287
+ * modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
2288
+ getTradingBook() {
2289
+ return this.tradingMode === 'LIVE' || this.tradingMode === 'MICRO_LIVE' ? 'live' : 'paper';
2290
+ }
2291
+ /** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
2292
+ onAgentRunLifecycle(listener) {
2293
+ this.on('agentRunLifecycle', listener);
2294
+ return () => this.off('agentRunLifecycle', listener);
2295
+ }
2296
+ recomputeModelHealth() {
2297
+ const hb = this.heartbeatHealth;
2298
+ const next = deriveModelHealth({
2299
+ lastRun: this.lastHeartbeatRunForHealth ?? null,
2300
+ cron: hb
2301
+ ? {
2302
+ consecutiveErrors: hb.consecutiveErrors,
2303
+ lastErrorReason: hb.lastErrorReason ?? null,
2304
+ lastError: hb.lastError ?? null,
2305
+ lastRunStatus: hb.lastRunStatus ?? null,
2306
+ lastRunAtMs: hb.lastRunAtMs ?? null,
2307
+ }
2308
+ : null,
2309
+ now: Date.now(),
2310
+ });
2311
+ const prev = this.modelHealth;
2312
+ if (!prev || prev.level !== next.level || prev.reason !== next.reason) {
2313
+ if (next.level !== 'ok') {
2314
+ logger.warn(TAG, `Model health ${next.level}: ${next.reason ?? ''} ${next.detail ?? ''}`.trim());
2315
+ }
2316
+ else if (prev && prev.level !== 'ok') {
2317
+ logger.info(TAG, 'Model health recovered: ok');
2318
+ }
2319
+ }
2320
+ this.modelHealth = next;
2321
+ }
2161
2322
  /** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
2162
2323
  mergeAgentIdentity(found) {
2163
2324
  let changed = false;
@@ -2349,6 +2510,10 @@ export class GatewayProvider {
2349
2510
  },
2350
2511
  }
2351
2512
  : {}),
2513
+ // Flight recorder: absent until the first beat is recorded → payload
2514
+ // stays byte-identical to the pre-feature shape until then.
2515
+ ...(this.lastHeartbeat ? { lastHeartbeat: this.lastHeartbeat } : {}),
2516
+ ...(this.modelHealth ? { modelHealth: this.modelHealth } : {}),
2352
2517
  },
2353
2518
  };
2354
2519
  }
@@ -39,6 +39,7 @@ export class MockProvider {
39
39
  shadowComparison: new Set(),
40
40
  tradingModeUpdate: new Set(),
41
41
  decisionTraceUpdate: new Set(),
42
+ agentRunLifecycle: new Set(),
42
43
  };
43
44
  // ---- Risk limits ----
44
45
  RISK_LIMITS = {
@@ -133,10 +133,51 @@ export interface AgentStateData {
133
133
  /** Last run outcome, e.g. "ok" | "error". */
134
134
  lastRunStatus?: string;
135
135
  };
136
+ /** Compact summary of the last recorded heartbeat run (flight recorder,
137
+ * skill/src/heartbeat-runs.ts). Absent until the first beat is
138
+ * recorded after start — byte-identical to pre-feature until then. */
139
+ lastHeartbeat?: LastHeartbeatSummary;
140
+ /** Is the LLM behind the agent answering? Derived skill-side from the
141
+ * cron job's FailoverReason + the last recorded beat
142
+ * (skill/src/model-health.ts). Absent until first derived. */
143
+ modelHealth?: ModelHealthSummary;
136
144
  };
137
145
  };
138
146
  timestamp: string;
139
147
  }
148
+ /** Dashboard-facing summary of one heartbeat run (the full record lives in
149
+ * the webapp's heartbeat_runs table). Token counts are RAW COUNTS — never
150
+ * translated to a currency here or downstream. */
151
+ export interface LastHeartbeatSummary {
152
+ runId: string;
153
+ startedAtMs: number;
154
+ endedAtMs: number | null;
155
+ durationMs: number | null;
156
+ status: 'ok' | 'error' | 'incomplete' | 'unknown';
157
+ toolCallCount: number;
158
+ toolErrorCount: number;
159
+ tokensTotal: number | null;
160
+ tokensInput: number | null;
161
+ tokensOutput: number | null;
162
+ model: string | null;
163
+ }
164
+ export interface ModelHealthSummary {
165
+ level: 'ok' | 'degraded' | 'error';
166
+ reason: string | null;
167
+ detail: string | null;
168
+ atMs: number;
169
+ model: string | null;
170
+ }
171
+ /** An agent turn's lifecycle as the gateway reports it, WITH the session key
172
+ * (an isolated cron beat runs under `agent:main:cron:<jobId>:run:<sid>`),
173
+ * so consumers can tell a heartbeat turn from a chat turn. */
174
+ export interface AgentRunLifecycleData {
175
+ runId: string;
176
+ phase: 'start' | 'end' | 'error';
177
+ sessionKey?: string;
178
+ sessionId?: string;
179
+ at: number;
180
+ }
140
181
  export interface RiskLimits {
141
182
  position: {
142
183
  maxPositionSize: number;
@@ -1,6 +1,6 @@
1
1
  // Re-export shim for the plugin-side strategy evaluator (facts-out).
2
2
  //
3
- // The generated eval-core copies (registry, strategy-adapter, direction/entry/
3
+ // The generated evaluator-core copies (registry, strategy-adapter, direction/entry/
4
4
  // stop-rules) import their types from `./types.js`. The canonical definitions
5
5
  // live in @reefclaw/shared (type-only → erased at runtime), so this shim makes
6
6
  // those relative imports resolve inside the plugin tree without pulling any
@@ -150,7 +150,7 @@ export interface PositionMetadata {
150
150
  /** The agent's stated profit-realization plan, pinned at entry (indication,
151
151
  * not an enforced mechanic). */
152
152
  realizationRule?: RealizationRule;
153
- /** Re-entry cooldown gate eval, present ONLY when the gate TRIGGERED on this
153
+ /** Re-entry cooldown gate evaluation, present ONLY when the gate TRIGGERED on this
154
154
  * entry (a same-symbol loss within the window) and the order fired anyway
155
155
  * (shadow/observe). Journaled to position_entries.metadata.reentry_cooldown
156
156
  * so the shadow soak can measure the would-block cohort's forward outcomes
@@ -1113,7 +1113,7 @@ export async function createOrderTool(args, deps) {
1113
1113
  const n = num(v);
1114
1114
  return n != null ? Math.max(min, Math.min(max, n)) : undefined;
1115
1115
  };
1116
- // Journal-tag a TRIGGERED cooldown eval (shadow/observe fire-anyway cohort —
1116
+ // Journal-tag a TRIGGERED cooldown evaluation (shadow/observe fire-anyway cohort —
1117
1117
  // the measurement rows the promote-to-enforce decision reads).
1118
1118
  const reentryCooldownTag = reentryCooldownEval?.triggered
1119
1119
  ? {
@@ -22,8 +22,8 @@
22
22
  // Mode semantics mirror the exit gate (docs/CLAUDE/exit-gate.md):
23
23
  // off — not evaluated; byte-identical to the pre-gate path.
24
24
  // shadow — evaluated + logged + journal-tagged; never affects the order.
25
- // observe — as shadow, but a triggered eval logs at WARN (operator-visible).
26
- // enforce — a triggered eval hard-rejects create_order with a recovery hint.
25
+ // observe — as shadow, but a triggered evaluation logs at WARN (operator-visible).
26
+ // enforce — a triggered evaluation hard-rejects create_order with a recovery hint.
27
27
  export function evaluateReentryCooldown(inputs) {
28
28
  const base = {
29
29
  mode: inputs.mode,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@reefclaw/connect",
3
- "version": "0.1.37",
3
+ "version": "0.1.38",
4
4
  "description": "One-command installer that connects your OpenClaw agent to ReefClaw (paper trading, no exchange keys).",
5
5
  "type": "module",
6
6
  "bin": {