@reefclaw/connect 0.1.37 → 0.1.39

package/assets/bridge/index.js

@@ -17,6 +17,7 @@ import { GatewayProvider } from './providers/gateway.js';
 import { resolveConfig } from './config.js';
 import { resolveGatewayConfig, validateGatewayConfig } from './gateway/gateway-config.js';
 import { ShockWakePoller } from './shock-wake.js';
+import { HeartbeatRunRecorder } from './heartbeat-runs.js';
 import { runSetup } from './setup.js';
 import { resolveRelayInstanceId } from './utils/instance-id.js';
 const TAG = 'main';
@@ -244,6 +245,17 @@ async function main() {
         // lane is alive — the gateway ack alone cannot (swallowed-turn class).
         provider.on('chatMessageStart', () => shockWake.noteAssistantActivity());
     }
+    // Heartbeat flight recorder — records what every beat actually DID (tool
+    // calls + results, report, model, raw token counts) from the isolated cron
+    // session's transcript, posts it to the webapp journal, and feeds the
+    // dashboard's last-beat summary + model-health dot. Gateway provider only
+    // (the mock has no OpenClaw sessions dir). Kill-switch: RC_HEARTBEAT_RECORDER=off.
+    const recorderOff = (process.env.RC_HEARTBEAT_RECORDER ?? 'on').trim().toLowerCase() === 'off';
+    const heartbeatRecorder = provider instanceof GatewayProvider && !recorderOff
+        ? new HeartbeatRunRecorder({ source: provider, token })
+        : null;
+    if (recorderOff)
+        logger.info(TAG, 'Heartbeat flight recorder disabled (RC_HEARTBEAT_RECORDER=off)');
     // Handle graceful shutdown
     let shuttingDown = false;
     async function shutdown(signal) {
@@ -259,6 +271,7 @@ async function main() {
             catch { /* best-effort */ }
         }
         shockWake?.stop();
+        heartbeatRecorder?.stop();
         bridge.stop();
         // Allow 1s for final cleanup before force exit
         setTimeout(() => process.exit(0), 1000);
@@ -269,5 +282,6 @@ async function main() {
     logger.info(TAG, `Starting ReefClaw skill (provider=${args.provider}, relay=${relayUrl})`);
     bridge.start();
     shockWake?.start();
+    heartbeatRecorder?.start();
 }
 main();

package/assets/bridge/model-health.d.ts

@@ -0,0 +1,37 @@
+export type ModelHealthLevel = 'ok' | 'degraded' | 'error';
+export interface ModelHealth {
+    level: ModelHealthLevel;
+    /** Machine reason (a FailoverReason, 'model_fallback', 'consecutive_errors'). */
+    reason: string | null;
+    /** Human detail (error text, fallback description), redacted upstream. */
+    detail: string | null;
+    /** Epoch ms of the evidence this verdict rests on. */
+    atMs: number;
+    model: string | null;
+}
+/** FailoverReasons that do not self-heal — the operator must act. */
+export declare const HARD_REASONS: ReadonlySet<string>;
+/** FailoverReasons that usually clear on their own (quota windows, outages). */
+export declare const SOFT_REASONS: ReadonlySet<string>;
+export interface ModelHealthInputs {
+    /** The most recently recorded heartbeat run (flight recorder). */
+    lastRun?: {
+        status: string;
+        errorReason?: string | null;
+        error?: string | null;
+        model?: string | null;
+        endedAtMs?: number | null;
+        fallbackSteps?: number;
+        modelChanges?: number;
+    } | null;
+    /** The heartbeat job's cron state (`cron.list`). */
+    cron?: {
+        consecutiveErrors: number;
+        lastErrorReason?: string | null;
+        lastError?: string | null;
+        lastRunStatus?: string | null;
+        lastRunAtMs?: number | null;
+    } | null;
+    now: number;
+}
+export declare function deriveModelHealth(i: ModelHealthInputs): ModelHealth;

package/assets/bridge/model-health.js

@@ -0,0 +1,97 @@
+// Model health — is the LLM behind the agent actually answering? (pure)
+//
+// Two incident classes motivated this: a Codex quota death (2026-06-07) and a
+// self-inflicted harness/provider mismatch (2026-07-26) both took the agent
+// dark for hours-to-days while the dashboard looked healthy. The gateway
+// classifies every failed model call with a FailoverReason (vendored
+// `embedded-agent-helpers/types.ts`) and persists it on the cron job state
+// (`lastErrorReason`) and the run log (`errorReason`); auth failures fall back
+// SILENTLY to the next model (docs/CLAUDE/agent-runtime.md §4), format
+// failures kill every turn with no fallback (§1). This derives one dot from
+// those signals so the operator sees "model degraded/erroring" instead of
+// inferring it from a stalled last-decision clock.
+//
+// Levels:
+//   error     — a reason that will not self-heal (auth_permanent, format,
+//               model_not_found, billing), or ≥2 consecutive model-classified
+//               failures.
+//   degraded  — a recoverable model reason on the last run (auth, rate_limit,
+//               overloaded, server_error, timeout, …) or a fallback / model
+//               change observed inside the last beat.
+//   ok        — nothing model-related is wrong. A run error WITHOUT a
+//               FailoverReason (e.g. a delivery-target error) stays 'ok' here:
+//               the heartbeat banner owns non-model failures.
+/** FailoverReasons that do not self-heal — the operator must act. */
+export const HARD_REASONS = new Set(['auth_permanent', 'format', 'model_not_found', 'billing']);
+/** FailoverReasons that usually clear on their own (quota windows, outages). */
+export const SOFT_REASONS = new Set([
+    'auth',
+    'rate_limit',
+    'overloaded',
+    'server_error',
+    'timeout',
+    'session_expired',
+    'empty_response',
+    'no_error_details',
+    'unclassified',
+    'unknown',
+]);
+function clip(s, n = 200) {
+    if (typeof s !== 'string')
+        return null;
+    const t = s.trim();
+    if (!t)
+        return null;
+    return t.length > n ? `${t.slice(0, n)}…` : t;
+}
+export function deriveModelHealth(i) {
+    const run = i.lastRun ?? null;
+    const cron = i.cron ?? null;
+    const model = run?.model ?? null;
+    const cronReason = cron?.lastRunStatus === 'error' ? cron.lastErrorReason ?? null : null;
+    const runReason = run?.status === 'error' ? run.errorReason ?? null : null;
+    const consecutive = cron?.consecutiveErrors ?? 0;
+    // Prefer whichever piece of evidence is newer.
+    const cronAt = cron?.lastRunAtMs ?? 0;
+    const runAt = run?.endedAtMs ?? 0;
+    const atMs = Math.max(cronAt, runAt) || i.now;
+    const hard = [cronReason, runReason].find((r) => r && HARD_REASONS.has(r)) ?? null;
+    if (hard) {
+        return {
+            level: 'error',
+            reason: hard,
+            detail: clip(cronReason === hard ? cron?.lastError : run?.error) ?? `model ${hard}`,
+            atMs,
+            model,
+        };
+    }
+    const anyReason = cronReason ?? runReason;
+    if (anyReason && consecutive >= 2) {
+        return {
+            level: 'error',
+            reason: 'consecutive_errors',
+            detail: clip(cron?.lastError ?? run?.error) ?? `${consecutive} consecutive model failures (${anyReason})`,
+            atMs,
+            model,
+        };
+    }
+    if (anyReason && (SOFT_REASONS.has(anyReason) || !HARD_REASONS.has(anyReason))) {
+        return {
+            level: 'degraded',
+            reason: anyReason,
+            detail: clip(cronReason ? cron?.lastError : run?.error) ?? `model ${anyReason}`,
+            atMs,
+            model,
+        };
+    }
+    if (run && ((run.fallbackSteps ?? 0) > 0 || (run.modelChanges ?? 0) > 0)) {
+        return {
+            level: 'degraded',
+            reason: 'model_fallback',
+            detail: `${run.fallbackSteps ?? 0} fallback step(s), ${run.modelChanges ?? 0} model change(s) in the last beat`,
+            atMs,
+            model,
+        };
+    }
+    return { level: 'ok', reason: null, detail: null, atMs, model };
+}

package/assets/bridge/provider.d.ts

@@ -1,4 +1,4 @@
-import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData } from './types.js';
+import type { EmergencyAction, ReconciliationSnapshot, TickerData, CandleData, OrderUpdatePayload, AgentStateData, RiskUpdatePayload, ChatMessage, MarketStructureData, CryptoMetricsData, VolumeAnalysisData, TradeJournalEntry, RegimeData, SignalData, MissionData, AnalyticsData, ShadowComparisonData, TradingModeData, DecisionTraceData, AgentRunLifecycleData } from './types.js';
 import type { TradingMode } from '@reefclaw/shared';
 import type { ConnectorUpdateOutcome } from './providers/connector-update.js';
 /** Per-venue credential shapes for the operator set/test RPCs. Binance is an
@@ -126,6 +126,10 @@ export interface ProviderEvents {
     tradingModeUpdate: (data: TradingModeData) => void;
     /** Decision trace update (Phase 9c) */
     decisionTraceUpdate: (data: DecisionTraceData) => void;
+    /** An agent turn started/ended/errored, with the gateway's session key so a
+     *  heartbeat (cron) turn is distinguishable from a chat turn. Local-only
+     *  (not forwarded to the relay); consumed by the heartbeat flight recorder. */
+    agentRunLifecycle: (data: AgentRunLifecycleData) => void;
 }
 export type ProviderEventName = keyof ProviderEvents;
 /**

package/assets/bridge/providers/gateway.d.ts

@@ -1,5 +1,6 @@
 import type { OpenClawProvider, ProviderEvents, ProviderEventName, ExchangeCredentialsRequest, HlProvisionOutcome, HlAgentWalletStatusOutcome, HlSubmitApprovalOutcome } from '../provider.js';
-import type { EmergencyAction, ReconciliationSnapshot, TradingMode } from '../types.js';
+import type { EmergencyAction, ReconciliationSnapshot, TradingMode, AgentRunLifecycleData } from '../types.js';
+import type { CronRunLogLite, HeartbeatRunRecord } from '../heartbeat-transcript.js';
 import type { GatewayConfig } from '../gateway/gateway-config.js';
 import { type GetBracketConfigOutcome, type SetBracketRequirementOutcome, type BracketRequirementFlag, type SetTradingModeOutcome, type SetExchangeCredentialsOutcome, type TestExchangeCredentialsOutcome, type ClearExchangeCredentialsOutcome } from './onboarding-commands.js';
 import { type ConnectorUpdateOutcome } from './connector-update.js';
@@ -56,6 +57,14 @@ export declare class GatewayProvider implements OpenClawProvider {
     private heartbeatHealth?;
     private heartbeatHealthReadAtMs;
     private heartbeatHealthRefreshing;
+    /** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
+    private heartbeatJobId?;
+    /** Last recorded heartbeat run (flight recorder) — dashboard summary. */
+    private lastHeartbeat?;
+    /** Model-health inputs from the last recorded run (kept apart from the
+     *  summary so the verdict can be re-derived when cron state changes). */
+    private lastHeartbeatRunForHealth?;
+    private modelHealth?;
     private lastTicker;
     private positions;
     private balance;
@@ -395,6 +404,21 @@ export declare class GatewayProvider implements OpenClawProvider {
      *  refresh is in flight. Fire-and-forget: the RPC completion re-emits
      *  agent_state, so callers stay synchronous. */
     private maybeRefreshHeartbeatHealth;
+    /** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
+     *  first), normalised to the recorder's shape. Returns null when the RPC or
+     *  the job id is unavailable (older gateway / scope) — the recorder then
+     *  falls back to transcript-only records. Token counts here are RAW counts. */
+    fetchCronRuns(limit?: number): Promise<CronRunLogLite[] | null>;
+    private loggedCronRunsProbe;
+    /** A heartbeat run was recorded — keep a compact summary for the dashboard
+     *  header, re-derive model health, and re-emit agent_state. */
+    noteHeartbeatRun(record: HeartbeatRunRecord): void;
+    /** Active trading book for the record's mode tag (matches the webapp's
+     *  modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
+    getTradingBook(): 'paper' | 'live';
+    /** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
+    onAgentRunLifecycle(listener: (d: AgentRunLifecycleData) => void): () => void;
+    private recomputeModelHealth;
     /** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
     private mergeAgentIdentity;
     /**

package/assets/bridge/providers/gateway.js

@@ -6,6 +6,7 @@ import { join } from 'path';
 import { homedir } from 'os';
 import { logger, formatError } from '../logger.js';
 import { isTradingMode } from '../types.js';
+import { deriveModelHealth } from '../model-health.js';
 import { handshakeLacksWriteScope, relaxGatewayDeviceAuth } from '../config.js';
 import { toIntelSymbol } from '@reefclaw/shared';
 import { GatewayHttpClient } from '../gateway/gateway-http-client.js';
@@ -175,6 +176,14 @@ export class GatewayProvider {
     heartbeatHealth;
     heartbeatHealthReadAtMs = 0;
     heartbeatHealthRefreshing = false;
+    /** The heartbeat cron job's id (from cron.list) — `cron.runs` needs it. */
+    heartbeatJobId;
+    /** Last recorded heartbeat run (flight recorder) — dashboard summary. */
+    lastHeartbeat;
+    /** Model-health inputs from the last recorded run (kept apart from the
+     *  summary so the verdict can be re-derived when cron state changes). */
+    lastHeartbeatRunForHealth;
+    modelHealth;
     lastTicker = null;
     positions = [];
     balance = { currency: 'USDT', total: 0, available: 0, locked: 0 };
@@ -318,6 +327,7 @@ export class GatewayProvider {
         shadowComparison: new Set(),
         tradingModeUpdate: new Set(),
         decisionTraceUpdate: new Set(),
+        agentRunLifecycle: new Set(),
     };
     constructor(config) {
         this.config = config;
@@ -1368,6 +1378,21 @@ export class GatewayProvider {
     onAgentEvent(payload) {
         if (!this.started || !this.eventParser)
             return;
+        // Turn lifecycle WITH the session key — the heartbeat flight recorder
+        // keys off `agent:main:cron:…` to scan the beat's transcript right after
+        // it ends. Fired before parsing so it never depends on parser state.
+        if (payload.stream === 'lifecycle') {
+            const phase = payload.data?.phase;
+            if (phase === 'start' || phase === 'end' || phase === 'error') {
+                this.fire('agentRunLifecycle', {
+                    runId: payload.runId,
+                    phase,
+                    ...(payload.sessionKey ? { sessionKey: payload.sessionKey } : {}),
+                    ...(payload.sessionId ? { sessionId: payload.sessionId } : {}),
+                    at: Date.now(),
+                });
+            }
+        }
         const parsed = this.eventParser.parseAgentEvent(payload);
         for (const evt of parsed) {
             // Track order/cancel rates for risk computation
@@ -2118,6 +2143,9 @@ export class GatewayProvider {
                 this.heartbeatSeconds = secs;
                 this.heartbeatReadAtMs = Date.now();
             }
+            const jobId = hb?.id;
+            if (typeof jobId === 'string' && jobId)
+                this.heartbeatJobId = jobId;
             const state = hb?.state;
             if (!state || typeof state !== 'object')
                 return;
@@ -2126,18 +2154,23 @@ export class GatewayProvider {
                 consecutiveErrors: typeof s.consecutiveErrors === 'number' && s.consecutiveErrors >= 0 ? s.consecutiveErrors : 0,
                 lastRunAtMs: typeof s.lastRunAtMs === 'number' ? s.lastRunAtMs : undefined,
                 lastRunStatus: typeof s.lastRunStatus === 'string' ? s.lastRunStatus : undefined,
+                lastErrorReason: typeof s.lastErrorReason === 'string' ? s.lastErrorReason : undefined,
+                lastError: typeof s.lastError === 'string' ? s.lastError.slice(0, 300) : undefined,
             };
             const prev = this.heartbeatHealth;
             const changed = !prev ||
                 prev.consecutiveErrors !== next.consecutiveErrors ||
                 prev.lastRunAtMs !== next.lastRunAtMs ||
-                prev.lastRunStatus !== next.lastRunStatus;
+                prev.lastRunStatus !== next.lastRunStatus ||
+                prev.lastErrorReason !== next.lastErrorReason;
             this.heartbeatHealth = next;
             this.heartbeatHealthReadAtMs = Date.now();
             if (changed) {
                 if (next.consecutiveErrors > 0) {
-                    logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}`);
+                    logger.warn(TAG, `Heartbeat health: ${next.consecutiveErrors} consecutive error(s), lastStatus=${next.lastRunStatus ?? 'unknown'}` +
+                        (next.lastErrorReason ? ` reason=${next.lastErrorReason}` : ''));
                 }
+                this.recomputeModelHealth();
                 this.emitAgentState();
             }
         }
@@ -2158,6 +2191,134 @@ export class GatewayProvider {
             return;
         void this.refreshHeartbeatHealth();
     }
+    // ---- Heartbeat flight recorder hooks (skill/src/heartbeat-runs.ts) ----
+    /** The gateway's cron run log for the heartbeat job (`cron.runs`, newest
+     *  first), normalised to the recorder's shape. Returns null when the RPC or
+     *  the job id is unavailable (older gateway / scope) — the recorder then
+     *  falls back to transcript-only records. Token counts here are RAW counts. */
+    async fetchCronRuns(limit = 25) {
+        const ws = this.wsClient;
+        if (!ws)
+            return null;
+        if (!this.heartbeatJobId) {
+            await this.refreshHeartbeatHealth();
+            if (!this.heartbeatJobId)
+                return null;
+        }
+        let resp;
+        try {
+            resp = await ws.sendRpc('cron.runs', { jobId: this.heartbeatJobId, limit, sortDir: 'desc' });
+        }
+        catch (err) {
+            logger.debug(TAG, `cron.runs RPC failed: ${err instanceof Error ? err.message : String(err)}`);
+            return null;
+        }
+        const entries = resp?.entries;
+        if (!Array.isArray(entries)) {
+            if (!this.loggedCronRunsProbe) {
+                this.loggedCronRunsProbe = true;
+                const keys = resp && typeof resp === 'object' ? Object.keys(resp).join(',') : typeof resp;
+                logger.info(TAG, `cron.runs probe: unexpected shape keys=[${keys}] — token counts fall back to transcripts`);
+            }
+            return null;
+        }
+        if (!this.loggedCronRunsProbe) {
+            this.loggedCronRunsProbe = true;
+            const first = entries[0] && typeof entries[0] === 'object' ? Object.keys(entries[0]).join(',') : '(empty)';
+            logger.info(TAG, `cron.runs probe: ${entries.length} entries, keys=[${first}]`);
+        }
+        const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
+        const str = (v) => (typeof v === 'string' && v ? v : null);
+        return entries.map((raw) => {
+            const e = (raw && typeof raw === 'object' ? raw : {});
+            const u = (e.usage && typeof e.usage === 'object' ? e.usage : null);
+            return {
+                sessionId: str(e.sessionId),
+                runId: str(e.runId),
+                status: str(e.status),
+                error: str(e.error),
+                errorReason: str(e.errorReason),
+                runAtMs: num(e.runAtMs) ?? num(e.ts),
+                durationMs: num(e.durationMs),
+                model: str(e.model),
+                provider: str(e.provider),
+                usage: u
+                    ? {
+                        input: num(u.input_tokens) ?? num(u.input),
+                        output: num(u.output_tokens) ?? num(u.output),
+                        cacheRead: num(u.cache_read_tokens) ?? num(u.cacheRead),
+                        cacheWrite: num(u.cache_write_tokens) ?? num(u.cacheWrite),
+                        total: num(u.total_tokens) ?? num(u.total) ?? num(u.totalTokens),
+                    }
+                    : null,
+            };
+        });
+    }
+    loggedCronRunsProbe = false;
+    /** A heartbeat run was recorded — keep a compact summary for the dashboard
+     *  header, re-derive model health, and re-emit agent_state. */
+    noteHeartbeatRun(record) {
+        this.lastHeartbeat = {
+            runId: record.runId,
+            startedAtMs: record.startedAtMs,
+            endedAtMs: record.endedAtMs,
+            durationMs: record.durationMs,
+            status: record.status,
+            toolCallCount: record.toolCallCount,
+            toolErrorCount: record.toolErrorCount,
+            tokensTotal: record.tokens.total,
+            tokensInput: record.tokens.input,
+            tokensOutput: record.tokens.output,
+            model: record.model,
+        };
+        this.lastHeartbeatRunForHealth = {
+            status: record.status,
+            errorReason: record.errorReason,
+            error: record.error,
+            model: record.model,
+            endedAtMs: record.endedAtMs,
+            fallbackSteps: record.modelEvents.filter((e) => e.kind === 'fallback_step').length,
+            modelChanges: record.modelEvents.filter((e) => e.kind === 'model_change').length,
+        };
+        this.recomputeModelHealth();
+        this.emitAgentState();
+    }
+    /** Active trading book for the record's mode tag (matches the webapp's
+     *  modeToBook: LIVE/MICRO_LIVE → live, everything else → paper). */
+    getTradingBook() {
+        return this.tradingMode === 'LIVE' || this.tradingMode === 'MICRO_LIVE' ? 'live' : 'paper';
+    }
+    /** Subscribe to agent turn lifecycle events; returns an unsubscribe. */
+    onAgentRunLifecycle(listener) {
+        this.on('agentRunLifecycle', listener);
+        return () => this.off('agentRunLifecycle', listener);
+    }
+    recomputeModelHealth() {
+        const hb = this.heartbeatHealth;
+        const next = deriveModelHealth({
+            lastRun: this.lastHeartbeatRunForHealth ?? null,
+            cron: hb
+                ? {
+                    consecutiveErrors: hb.consecutiveErrors,
+                    lastErrorReason: hb.lastErrorReason ?? null,
+                    lastError: hb.lastError ?? null,
+                    lastRunStatus: hb.lastRunStatus ?? null,
+                    lastRunAtMs: hb.lastRunAtMs ?? null,
+                }
+                : null,
+            now: Date.now(),
+        });
+        const prev = this.modelHealth;
+        if (!prev || prev.level !== next.level || prev.reason !== next.reason) {
+            if (next.level !== 'ok') {
+                logger.warn(TAG, `Model health ${next.level}: ${next.reason ?? ''} ${next.detail ?? ''}`.trim());
+            }
+            else if (prev && prev.level !== 'ok') {
+                logger.info(TAG, 'Model health recovered: ok');
+            }
+        }
+        this.modelHealth = next;
+    }
     /** Merge newly-resolved identity fields into the cache. Returns true if anything changed. */
     mergeAgentIdentity(found) {
         let changed = false;
@@ -2349,6 +2510,10 @@ export class GatewayProvider {
                         },
                     }
                     : {}),
+                // Flight recorder: absent until the first beat is recorded → payload
+                // stays byte-identical to the pre-feature shape until then.
+                ...(this.lastHeartbeat ? { lastHeartbeat: this.lastHeartbeat } : {}),
+                ...(this.modelHealth ? { modelHealth: this.modelHealth } : {}),
             },
         };
     }

package/assets/bridge/providers/mock.js

@@ -39,6 +39,7 @@ export class MockProvider {
         shadowComparison: new Set(),
         tradingModeUpdate: new Set(),
         decisionTraceUpdate: new Set(),
+        agentRunLifecycle: new Set(),
     };
     // ---- Risk limits ----
     RISK_LIMITS = {

package/assets/bridge/types.d.ts

@@ -133,10 +133,51 @@ export interface AgentStateData {
                 /** Last run outcome, e.g. "ok" | "error". */
                 lastRunStatus?: string;
             };
+            /** Compact summary of the last recorded heartbeat run (flight recorder,
+             *  skill/src/heartbeat-runs.ts). Absent until the first beat is
+             *  recorded after start — byte-identical to pre-feature until then. */
+            lastHeartbeat?: LastHeartbeatSummary;
+            /** Is the LLM behind the agent answering? Derived skill-side from the
+             *  cron job's FailoverReason + the last recorded beat
+             *  (skill/src/model-health.ts). Absent until first derived. */
+            modelHealth?: ModelHealthSummary;
         };
     };
     timestamp: string;
 }
+/** Dashboard-facing summary of one heartbeat run (the full record lives in
+ *  the webapp's heartbeat_runs table). Token counts are RAW COUNTS — never
+ *  translated to a currency here or downstream. */
+export interface LastHeartbeatSummary {
+    runId: string;
+    startedAtMs: number;
+    endedAtMs: number | null;
+    durationMs: number | null;
+    status: 'ok' | 'error' | 'incomplete' | 'unknown';
+    toolCallCount: number;
+    toolErrorCount: number;
+    tokensTotal: number | null;
+    tokensInput: number | null;
+    tokensOutput: number | null;
+    model: string | null;
+}
+export interface ModelHealthSummary {
+    level: 'ok' | 'degraded' | 'error';
+    reason: string | null;
+    detail: string | null;
+    atMs: number;
+    model: string | null;
+}
+/** An agent turn's lifecycle as the gateway reports it, WITH the session key
+ *  (an isolated cron beat runs under `agent:main:cron:<jobId>:run:<sid>`),
+ *  so consumers can tell a heartbeat turn from a chat turn. */
+export interface AgentRunLifecycleData {
+    runId: string;
+    phase: 'start' | 'end' | 'error';
+    sessionKey?: string;
+    sessionId?: string;
+    at: number;
+}
 export interface RiskLimits {
     position: {
         maxPositionSize: number;

package/assets/plugin/index.js

@@ -1001,6 +1001,13 @@ const paperTradingPlugin = {
                 logger.warn(TAG, `State reload failed: ${formatError(err)}`);
             }
         };
+        // Out-of-tool readers (periodic DB-vs-exchange sweep, stop-watcher) read
+        // the PaperAdapter directly and never went through reloadState — in the
+        // process that did not execute the entry they saw a stale book and posted
+        // false `reconciler_observed_flat` closes seconds after every entry
+        // (2026-09-14). Hook on the simulator so every adapter built from it
+        // (boot + every reconnect path) refreshes before a read.
+        simulator.setStateRefresher(reloadState);
         // Read config: ReefClaw plugin config from ~/.reefclaw/plugin-config.json
         // (OpenClaw's schema validation rejects custom keys in plugin entries)
         let connectionToken = '';

package/assets/plugin/ingest/reconcile-db-vs-exchange.d.ts

@@ -13,6 +13,14 @@ export interface DbVsExchangeContext {
      *  venue's snapshot can never contain. Absent → unscoped (legacy). */
     resolveExchange?: () => 'binance' | 'hyperliquid';
 }
+/** Rows opened more recently than this are never orphan-closed. An entry
+ *  that filled seconds ago and is "absent from the exchange" is propagation
+ *  lag, not an orphan: the paper book's debounced state.json save (1s) has
+ *  not landed in the other process yet, or a live fill's REST snapshot raced
+ *  the journal POST. A real orphan is still harvested by the next sweep
+ *  (5 min) — the grace costs nothing and removes the false-close class that
+ *  hit every entry on a paying tenant's box on 2026-09-14. */
+export declare const DEFAULT_MIN_OPEN_AGE_MS = 120000;
 /**
  * Close webapp `positions` rows that are status='open' but absent from the
  * (trusted) exchange snapshot. Returns the number of synthetic closes posted.
@@ -23,6 +31,9 @@ export interface DbVsExchangeContext {
 export interface DbReconcileOptions {
     /** Provenance tag written into closeAssessment.source (default boot sweep). */
     source?: string;
+    /** Young-row grace (ms); rows with `nowMs - openedAt < minOpenAgeMs` are
+     *  skipped with a log line. Default DEFAULT_MIN_OPEN_AGE_MS; 0 disables. */
+    minOpenAgeMs?: number;
     /** Attribution hook (issue #203): given an orphaned symbol, return a short
      *  human-readable description of what the execution engine last knew about
      *  it (e.g. the simulator's last trade). Logged with the synthetic close so
@@ -32,6 +43,8 @@ export interface DbReconcileOptions {
 export declare function reconcileDbOpenVsExchange(ctx: DbVsExchangeContext, exchangeSymbols: Iterable<string>, nowMs?: number, opts?: DbReconcileOptions): Promise<number>;
 export declare const DEFAULT_DB_RECONCILE_INTERVAL_MS = 300000;
 export interface PeriodicDbReconcileDeps extends DbVsExchangeContext {
+    /** Young-row grace forwarded to every sweep (default DEFAULT_MIN_OPEN_AGE_MS). */
+    minOpenAgeMs?: number;
     /** Resolve the ACTIVE adapter each tick (follows runtime reconnects). */
     resolveAdapter: () => {
         getPositionsOrNull(symbol?: string): Promise<Array<{

package/assets/plugin/ingest/reconcile-db-vs-exchange.js

@@ -43,6 +43,14 @@ const TAG = 'reconcile-db-vs-exchange';
 function canonical(symbol) {
     return symbol.split(':')[0];
 }
+/** Rows opened more recently than this are never orphan-closed. An entry
+ *  that filled seconds ago and is "absent from the exchange" is propagation
+ *  lag, not an orphan: the paper book's debounced state.json save (1s) has
+ *  not landed in the other process yet, or a live fill's REST snapshot raced
+ *  the journal POST. A real orphan is still harvested by the next sweep
+ *  (5 min) — the grace costs nothing and removes the false-close class that
+ *  hit every entry on a paying tenant's box on 2026-09-14. */
+export const DEFAULT_MIN_OPEN_AGE_MS = 120_000;
 export async function reconcileDbOpenVsExchange(ctx, exchangeSymbols, nowMs = Date.now(), opts = {}) {
     if (!ctx.decisionsClient || !ctx.userId)
         return 0;
@@ -58,7 +66,16 @@ export async function reconcileDbOpenVsExchange(ctx, exchangeSymbols, nowMs = Da
     const exchangeSet = new Set();
     for (const s of exchangeSymbols)
         exchangeSet.add(canonical(s));
-    const orphans = resp.positions.filter((p) => !exchangeSet.has(canonical(p.symbol)));
+    const minOpenAgeMs = opts.minOpenAgeMs ?? DEFAULT_MIN_OPEN_AGE_MS;
+    const absent = resp.positions.filter((p) => !exchangeSet.has(canonical(p.symbol)));
+    const orphans = absent.filter((p) => {
+        const ageMs = nowMs - p.openedAt;
+        if (Number.isFinite(ageMs) && ageMs < minOpenAgeMs) {
+            logger.info(TAG, `${p.symbol}: opened ${Math.round(ageMs / 1000)}s ago and absent from the snapshot — within the ${Math.round(minOpenAgeMs / 1000)}s grace, not treated as an orphan (propagation lag; next sweep re-checks)`);
+            return false;
+        }
+        return true;
+    });
     if (orphans.length === 0) {
         logger.info(TAG, `DB reconcile: all ${resp.positions.length} open row(s) present on exchange`);
         return 0;
@@ -149,7 +166,11 @@ export function startPeriodicDbReconcile(deps, intervalMs = resolveDbReconcileIn
                 logger.warn(TAG, 'periodic sweep skipped — positions fetch untrusted (null)');
                 return 0;
             }
-            return await reconcileDbOpenVsExchange(deps, positions.map((p) => p.symbol), Date.now(), { source: 'db_exchange_sweep_periodic', describeLastExit: deps.describeLastExit });
+            return await reconcileDbOpenVsExchange(deps, positions.map((p) => p.symbol), Date.now(), {
+                source: 'db_exchange_sweep_periodic',
+                describeLastExit: deps.describeLastExit,
+                minOpenAgeMs: deps.minOpenAgeMs,
+            });
         }
         catch (err) {
             logger.warn(TAG, `periodic sweep failed: ${err instanceof Error ? err.message : String(err)}`);

package/assets/plugin/openclaw.plugin.json

@@ -1,7 +1,7 @@
 {
   "id": "reefclaw-paper-trading",
   "name": "ReefClaw Trading",
-  "version": "0.1.25",
+  "version": "0.1.27",
   "description": "Supervised trading plugin for the ReefClaw dashboard. It runs on YOUR machine and starts in PAPER mode with no API keys. It cannot trade real funds until you supply exchange credentials and step PAPER→MICRO_LIVE→LIVE yourself from the dashboard — the agent cannot make that change (the tool is refused without operator provenance). Exchange keys stay local, are used only to sign requests to the exchange, and are never transmitted to ReefClaw (asserted by a test in this package). Trading telemetry — positions, fills, decision journal — is sent to ReefClaw to render the dashboard. Every live position carries exchange-native protective stops. Remote updates to the agent's trading instructions are applied only after an Ed25519 signature is verified against a public key pinned in this build.",
   "author": "ReefClaw",
   "activation": {