@reefclaw/openclaw-plugin 0.1.25 → 0.1.26

package/bridge/gateway/gateway-ws-client.d.ts

@@ -45,6 +45,8 @@ export type GatewayWsEventPayload = {
         stream: string;
         data: unknown;
         seq?: number;
+        sessionKey?: string;
+        sessionId?: string;
     };
     presence: {
         payload: unknown;

package/bridge/gateway/gateway-ws-client.js

@@ -270,11 +270,17 @@ export class GatewayWsClient {
                 this.resetStaleTimer();
                 if (frame.payload && typeof frame.payload === 'object') {
                     const p = frame.payload;
+                    // sessionKey/sessionId ride along (gateway AgentEventPayload): a
+                    // cron beat's key is `agent:main:cron:<jobId>:run:<sessionId>`,
+                    // which is how the heartbeat flight recorder tells a beat's turn
+                    // from an operator chat turn. Optional — older gateways omit them.
                     this.emit('agent', {
                         runId: p.runId,
                         stream: p.stream,
                         data: p.data,
                         seq: frame.seq,
+                        ...(typeof p.sessionKey === 'string' ? { sessionKey: p.sessionKey } : {}),
+                        ...(typeof p.sessionId === 'string' ? { sessionId: p.sessionId } : {}),
                     });
                 }
                 break;

package/bridge/heartbeat-runs-state.d.ts

@@ -0,0 +1,16 @@
+export declare const STATE_VERSION = 1;
+/** Session ids remembered — ~2 days of 15-min beats. */
+export declare const RECORDED_MAX = 300;
+export interface HeartbeatRunsState {
+    version: number;
+    /** Session ids already recorded (or deliberately skipped), oldest first. */
+    recorded: string[];
+    /** mtime watermark (epoch ms) of the last completed scan; 0 = never. */
+    lastScanMtimeMs: number;
+}
+export declare function defaultStateFile(): string;
+export declare function emptyState(): HeartbeatRunsState;
+export declare function loadState(file?: string): HeartbeatRunsState;
+/** Best-effort write; returns false when the filesystem refused. */
+export declare function saveState(state: HeartbeatRunsState, file?: string): boolean;
+export declare function markRecorded(state: HeartbeatRunsState, sessionId: string): void;

package/bridge/heartbeat-runs-state.js

@@ -0,0 +1,58 @@
+// Heartbeat flight recorder — persisted scan state (fs only, network-free).
+//
+// The recorder must survive a bridge restart without re-posting every beat it
+// already reported, so it keeps a bounded list of recorded session ids plus
+// the mtime watermark of its last scan at ~/.reefclaw/heartbeat-runs-state.json.
+// Fail-open everywhere: an unreadable/garbled file is the same as a fresh
+// install (the webapp upsert is idempotent on (user, run id), so a re-post is
+// harmless — just wasted bytes).
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { homedir } from 'node:os';
+export const STATE_VERSION = 1;
+/** Session ids remembered — ~2 days of 15-min beats. */
+export const RECORDED_MAX = 300;
+export function defaultStateFile() {
+    return join(homedir(), '.reefclaw', 'heartbeat-runs-state.json');
+}
+export function emptyState() {
+    return { version: STATE_VERSION, recorded: [], lastScanMtimeMs: 0 };
+}
+export function loadState(file = defaultStateFile()) {
+    try {
+        if (!existsSync(file))
+            return emptyState();
+        const raw = JSON.parse(readFileSync(file, 'utf8'));
+        if (!raw || typeof raw !== 'object' || raw.version !== STATE_VERSION)
+            return emptyState();
+        const recorded = Array.isArray(raw.recorded) ? raw.recorded.filter((s) => typeof s === 'string') : [];
+        const last = typeof raw.lastScanMtimeMs === 'number' && Number.isFinite(raw.lastScanMtimeMs) ? raw.lastScanMtimeMs : 0;
+        return { version: STATE_VERSION, recorded: recorded.slice(-RECORDED_MAX), lastScanMtimeMs: last };
+    }
+    catch {
+        return emptyState();
+    }
+}
+/** Best-effort write; returns false when the filesystem refused. */
+export function saveState(state, file = defaultStateFile()) {
+    try {
+        mkdirSync(dirname(file), { recursive: true });
+        const trimmed = {
+            version: STATE_VERSION,
+            recorded: state.recorded.slice(-RECORDED_MAX),
+            lastScanMtimeMs: state.lastScanMtimeMs,
+        };
+        writeFileSync(file, JSON.stringify(trimmed), 'utf8');
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+export function markRecorded(state, sessionId) {
+    if (state.recorded.includes(sessionId))
+        return;
+    state.recorded.push(sessionId);
+    if (state.recorded.length > RECORDED_MAX)
+        state.recorded.splice(0, state.recorded.length - RECORDED_MAX);
+}

package/bridge/heartbeat-runs.d.ts

@@ -0,0 +1,99 @@
+import { type CronRunLogLite, type HeartbeatRunRecord } from './heartbeat-transcript.js';
+import { type HeartbeatRunsState } from './heartbeat-runs-state.js';
+import type { AgentRunLifecycleData } from './types.js';
+/** Regular scan cadence. */
+export declare const SCAN_INTERVAL_MS = 60000;
+/** Delay after a cron-session turn ends before scanning (lets the gateway
+ *  flush the transcript tail + write the run-log row). */
+export declare const POST_TURN_DELAY_MS = 4000;
+/** First scan after start — after the WS connects and the box settles. */
+export declare const FIRST_SCAN_DELAY_MS = 15000;
+/** A file changed more recently than this is possibly still being written. */
+export declare const SETTLE_MS = 3000;
+/** A mid-chain transcript older than this is recorded as 'incomplete'. */
+export declare const INCOMPLETE_AFTER_MS: number;
+/** Watermark slack — files can land with an mtime older than "now". */
+export declare const SCAN_SLACK_MS: number;
+/** Fresh install: look back this far for beats to backfill. */
+export declare const BOOTSTRAP_LOOKBACK_MS: number;
+/** Records processed per scan (bounds CPU + POST burst on a backfill). */
+export declare const MAX_PER_SCAN = 5;
+/** Give up on a run after this many failed POST cycles. */
+export declare const MAX_POST_ATTEMPTS = 5;
+/** Cool down `cron.runs` for this long after it errors (older gateway). */
+export declare const CRON_RUNS_COOLDOWN_MS: number;
+export type { AgentRunLifecycleData };
+/** What the recorder needs from the provider — kept narrow so it is trivially
+ *  mockable and the provider stays the only thing that talks to the gateway. */
+export interface HeartbeatRunSource {
+    /** `cron.runs` for the heartbeat job, newest first; null when unavailable. */
+    fetchCronRuns(limit: number): Promise<CronRunLogLite[] | null>;
+    /** Dashboard hook: the provider stores a compact summary + model health. */
+    noteHeartbeatRun(record: HeartbeatRunRecord): void;
+    /** Active trading book, for the record's mode tag. */
+    getTradingBook(): 'paper' | 'live';
+    /** Subscribe to agent turn lifecycle (with the gateway's session key). */
+    onAgentRunLifecycle(listener: (d: AgentRunLifecycleData) => void): () => void;
+}
+export interface HeartbeatRunRecorderOptions {
+    source: HeartbeatRunSource;
+    /** The rc_ connection token (Bearer for the webapp ingest). */
+    token: string;
+    webappUrl?: string;
+    sessionsDir?: string;
+    stateFile?: string;
+    fetchImpl?: typeof fetch;
+    now?: () => number;
+    scanIntervalMs?: number;
+    /** Disable the periodic timer (tests drive `scan()` directly). */
+    manual?: boolean;
+}
+export interface PostOutcome {
+    ok: boolean;
+    /** 4xx → terminal (never retry); network/5xx → retry. */
+    terminal: boolean;
+    status: number | null;
+    error?: string;
+}
+export declare class HeartbeatRunRecorder {
+    private readonly opts;
+    private readonly sessionsDir;
+    private readonly stateFile;
+    private timer;
+    private kickTimer;
+    private unsubscribe;
+    private scanning;
+    private stopped;
+    private readonly attempts;
+    private cronRunsUnavailableUntil;
+    private probeLogged;
+    /** Counters for a one-line health summary in logs. */
+    readonly stats: {
+        scans: number;
+        recorded: number;
+        posted: number;
+        postFailed: number;
+        skippedNonCron: number;
+        deferred: number;
+    };
+    constructor(opts: HeartbeatRunRecorderOptions);
+    private now;
+    start(): void;
+    stop(): void;
+    /** A cron-session turn ended → scan shortly (debounced). */
+    onLifecycle(d: AgentRunLifecycleData): void;
+    private kick;
+    /** One scan cycle. Safe to call concurrently (re-entrancy guarded). */
+    scan(): Promise<{
+        processed: number;
+        deferred: number;
+    }>;
+    /** Is this session a heartbeat? Head marker first (cheap), trajectory key
+     *  second (covers a prompt without the marker), else not a heartbeat. */
+    private classify;
+    private fetchCronRunsSafe;
+    /** POST one record to the webapp. Never throws. */
+    post(record: HeartbeatRunRecord): Promise<PostOutcome>;
+}
+/** Test seam: expose the state loader so tests can assert persistence. */
+export declare function readRecorderState(stateFile: string): HeartbeatRunsState;

package/bridge/heartbeat-runs.js

@@ -0,0 +1,300 @@
+// Heartbeat flight recorder — RECORDER (scheduling + cron run-log enrichment
+// + webapp POST). File reads live in heartbeat-transcript.ts / -state.ts.
+//
+// What it does, every 60s and ~4s after any cron-session turn ends:
+//   1. list transcripts in the OpenClaw sessions dir newer than the last
+//      scan watermark (minus slack) that are not yet recorded;
+//   2. keep the ones whose first user message carries the `[cron:…]`
+//      heartbeat marker (or whose trajectory key is a cron key); mark every
+//      other session (the main chat session, subagents) as skipped once so
+//      it is never re-read;
+//   3. skip a beat still being written (last assistant message ended on a
+//      tool call and the file changed <25 min ago) — it comes back next scan;
+//   4. enrich from the gateway's `cron.runs` log (status, FailoverReason,
+//      duration, model, TOKEN COUNTS) when the RPC is available;
+//   5. hand the record to the provider (dashboard header: last beat + model
+//      health) and POST it to the webapp (idempotent on (user, run id)).
+//
+// Failure posture: fail-open and bounded. A webapp outage retries the same
+// run up to 5 cycles then gives up on it (the next beat is a fresh record);
+// a 4xx is terminal (never re-posted); the recorder never blocks trading,
+// chat, or emergency paths — it only reads files and posts JSON.
+//
+// Deliberately NOT recorded: shock wakes and operator chat turns (they run in
+// the main session, whose transcript is one long file). Heartbeats only.
+import { logger } from './logger.js';
+import { buildHeartbeatRunRecord, isCronSessionKey, listSessionCandidates, readCronMarker, readSession, resolveSessionsDir, shrinkRecord, } from './heartbeat-transcript.js';
+import { defaultStateFile, loadState, markRecorded, saveState } from './heartbeat-runs-state.js';
+const TAG = 'heartbeat-runs';
+/** Regular scan cadence. */
+export const SCAN_INTERVAL_MS = 60_000;
+/** Delay after a cron-session turn ends before scanning (lets the gateway
+ *  flush the transcript tail + write the run-log row). */
+export const POST_TURN_DELAY_MS = 4_000;
+/** First scan after start — after the WS connects and the box settles. */
+export const FIRST_SCAN_DELAY_MS = 15_000;
+/** A file changed more recently than this is possibly still being written. */
+export const SETTLE_MS = 3_000;
+/** A mid-chain transcript older than this is recorded as 'incomplete'. */
+export const INCOMPLETE_AFTER_MS = 25 * 60_000;
+/** Watermark slack — files can land with an mtime older than "now". */
+export const SCAN_SLACK_MS = 15 * 60_000;
+/** Fresh install: look back this far for beats to backfill. */
+export const BOOTSTRAP_LOOKBACK_MS = 6 * 60 * 60_000;
+/** Records processed per scan (bounds CPU + POST burst on a backfill). */
+export const MAX_PER_SCAN = 5;
+/** Give up on a run after this many failed POST cycles. */
+export const MAX_POST_ATTEMPTS = 5;
+/** Cool down `cron.runs` for this long after it errors (older gateway). */
+export const CRON_RUNS_COOLDOWN_MS = 60 * 60_000;
+export class HeartbeatRunRecorder {
+    opts;
+    sessionsDir;
+    stateFile;
+    timer = null;
+    kickTimer = null;
+    unsubscribe = null;
+    scanning = false;
+    stopped = false;
+    attempts = new Map();
+    cronRunsUnavailableUntil = 0;
+    probeLogged = false;
+    /** Counters for a one-line health summary in logs. */
+    stats = { scans: 0, recorded: 0, posted: 0, postFailed: 0, skippedNonCron: 0, deferred: 0 };
+    constructor(opts) {
+        this.opts = opts;
+        this.sessionsDir = opts.sessionsDir ?? resolveSessionsDir();
+        this.stateFile = opts.stateFile ?? defaultStateFile();
+    }
+    now() {
+        return this.opts.now?.() ?? Date.now();
+    }
+    start() {
+        if (this.stopped)
+            return;
+        this.unsubscribe = this.opts.source.onAgentRunLifecycle((d) => this.onLifecycle(d));
+        if (this.opts.manual)
+            return;
+        const interval = this.opts.scanIntervalMs ?? SCAN_INTERVAL_MS;
+        this.kick(FIRST_SCAN_DELAY_MS);
+        this.timer = setInterval(() => void this.scan(), interval);
+        this.timer.unref?.();
+        logger.info(TAG, `Heartbeat flight recorder started (dir=${this.sessionsDir}, every ${Math.round(interval / 1000)}s)`);
+    }
+    stop() {
+        this.stopped = true;
+        if (this.timer)
+            clearInterval(this.timer);
+        if (this.kickTimer)
+            clearTimeout(this.kickTimer);
+        this.timer = null;
+        this.kickTimer = null;
+        this.unsubscribe?.();
+        this.unsubscribe = null;
+    }
+    /** A cron-session turn ended → scan shortly (debounced). */
+    onLifecycle(d) {
+        if (d.phase === 'start')
+            return;
+        if (!isCronSessionKey(d.sessionKey))
+            return;
+        this.kick(POST_TURN_DELAY_MS);
+    }
+    kick(delayMs) {
+        if (this.stopped || this.opts.manual)
+            return;
+        if (this.kickTimer)
+            clearTimeout(this.kickTimer);
+        this.kickTimer = setTimeout(() => {
+            this.kickTimer = null;
+            void this.scan();
+        }, delayMs);
+        this.kickTimer.unref?.();
+    }
+    /** One scan cycle. Safe to call concurrently (re-entrancy guarded). */
+    async scan() {
+        if (this.scanning || this.stopped)
+            return { processed: 0, deferred: 0 };
+        this.scanning = true;
+        this.stats.scans++;
+        const now = this.now();
+        let processed = 0;
+        let deferred = 0;
+        try {
+            const state = loadState(this.stateFile);
+            const since = state.lastScanMtimeMs > 0 ? state.lastScanMtimeMs - SCAN_SLACK_MS : now - BOOTSTRAP_LOOKBACK_MS;
+            const exclude = new Set(state.recorded);
+            const candidates = listSessionCandidates(this.sessionsDir, { minMtimeMs: since, exclude })
+                .filter((c) => now - c.mtimeMs >= SETTLE_MS)
+                .sort((a, b) => a.mtimeMs - b.mtimeMs);
+            let cronRuns; // undefined = not fetched yet
+            let dirty = false;
+            for (const c of candidates) {
+                if (processed >= MAX_PER_SCAN) {
+                    deferred++;
+                    continue;
+                }
+                const verdict = this.classify(c);
+                if (verdict === 'not-cron') {
+                    markRecorded(state, c.sessionId);
+                    this.stats.skippedNonCron++;
+                    dirty = true;
+                    continue;
+                }
+                if (verdict === 'unreadable') {
+                    deferred++;
+                    continue;
+                }
+                const parsed = readSession(c);
+                if (!parsed) {
+                    deferred++;
+                    continue;
+                }
+                const { transcript, trajectory } = parsed;
+                const finished = transcript.complete || trajectory?.ended != null;
+                if (!finished && now - c.mtimeMs < INCOMPLETE_AFTER_MS) {
+                    deferred++;
+                    this.stats.deferred++;
+                    continue;
+                }
+                if (cronRuns === undefined)
+                    cronRuns = await this.fetchCronRunsSafe(now);
+                const runLog = cronRuns?.find((e) => e.sessionId === c.sessionId || e.runId === c.sessionId) ?? null;
+                let record = buildHeartbeatRunRecord({
+                    sessionId: c.sessionId,
+                    transcript,
+                    trajectory,
+                    runLog,
+                    mode: this.opts.source.getTradingBook(),
+                    fallbackStartedAtMs: c.mtimeMs,
+                });
+                record = shrinkRecord(record);
+                try {
+                    this.opts.source.noteHeartbeatRun(record);
+                }
+                catch (err) {
+                    logger.debug(TAG, `noteHeartbeatRun threw: ${err instanceof Error ? err.message : String(err)}`);
+                }
+                const outcome = await this.post(record);
+                processed++;
+                if (outcome.ok) {
+                    this.stats.posted++;
+                    this.stats.recorded++;
+                    markRecorded(state, c.sessionId);
+                    this.attempts.delete(c.sessionId);
+                    dirty = true;
+                    logger.info(TAG, `Heartbeat run recorded: ${c.sessionId.slice(0, 8)} status=${record.status} tools=${record.toolCallCount}` +
+                        ` (${record.toolErrorCount} err) tokens=${record.tokens.total ?? 'n/a'}/${record.tokens.source}` +
+                        ` model=${record.model ?? '?'} dur=${record.durationMs != null ? Math.round(record.durationMs / 1000) : '?'}s`);
+                }
+                else {
+                    this.stats.postFailed++;
+                    const n = (this.attempts.get(c.sessionId) ?? 0) + 1;
+                    this.attempts.set(c.sessionId, n);
+                    if (outcome.terminal || n >= MAX_POST_ATTEMPTS) {
+                        markRecorded(state, c.sessionId);
+                        this.attempts.delete(c.sessionId);
+                        dirty = true;
+                        logger.warn(TAG, `Heartbeat run ${c.sessionId.slice(0, 8)} NOT persisted (${outcome.terminal ? 'rejected' : 'gave up'}: ` +
+                            `${outcome.status ?? 'network'} ${outcome.error ?? ''}) — shown on the dashboard, not in the journal`);
+                    }
+                    else {
+                        deferred++;
+                        logger.debug(TAG, `POST failed for ${c.sessionId.slice(0, 8)} (attempt ${n}): ${outcome.error ?? outcome.status}`);
+                    }
+                }
+            }
+            // Advance the watermark only when nothing was deferred, so a beat still
+            // being written stays inside the window until it finishes.
+            if (deferred === 0) {
+                state.lastScanMtimeMs = now;
+                dirty = true;
+            }
+            if (dirty)
+                saveState(state, this.stateFile);
+        }
+        catch (err) {
+            logger.warn(TAG, `scan failed: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        finally {
+            this.scanning = false;
+        }
+        return { processed, deferred };
+    }
+    /** Is this session a heartbeat? Head marker first (cheap), trajectory key
+     *  second (covers a prompt without the marker), else not a heartbeat. */
+    classify(c) {
+        const marker = readCronMarker(c.file);
+        if (marker === undefined) {
+            // Inconclusive head — fall back to the trajectory key when present.
+            if (c.trajectoryFile) {
+                const parsed = readSession(c);
+                if (!parsed)
+                    return 'unreadable';
+                if (parsed.transcript.cron)
+                    return 'cron';
+                if (isCronSessionKey(parsed.trajectory?.sessionKey))
+                    return 'cron';
+                return 'not-cron';
+            }
+            return c.sizeBytes === 0 ? 'unreadable' : 'not-cron';
+        }
+        return marker ? 'cron' : 'not-cron';
+    }
+    async fetchCronRunsSafe(now) {
+        if (now < this.cronRunsUnavailableUntil)
+            return null;
+        try {
+            const runs = await this.opts.source.fetchCronRuns(25);
+            if (runs === null) {
+                this.cronRunsUnavailableUntil = now + CRON_RUNS_COOLDOWN_MS;
+                if (!this.probeLogged) {
+                    this.probeLogged = true;
+                    logger.info(TAG, 'cron.runs unavailable on this gateway — token counts fall back to the trajectory/transcript');
+                }
+                return null;
+            }
+            return runs;
+        }
+        catch (err) {
+            this.cronRunsUnavailableUntil = now + CRON_RUNS_COOLDOWN_MS;
+            logger.debug(TAG, `cron.runs failed: ${err instanceof Error ? err.message : String(err)}`);
+            return null;
+        }
+    }
+    /** POST one record to the webapp. Never throws. */
+    async post(record) {
+        // www is load-bearing: reefclaw.com 307-redirects and Node fetch strips the
+        // Authorization header on cross-origin redirect (verified 2026-03-18).
+        const base = (this.opts.webappUrl ?? process.env.REEFCLAW_API_URL ?? 'https://www.reefclaw.com').replace(/\/$/, '');
+        const fetchImpl = this.opts.fetchImpl ?? fetch;
+        try {
+            const res = await fetchImpl(`${base}/api/internal/heartbeat-runs`, {
+                method: 'POST',
+                headers: {
+                    'content-type': 'application/json',
+                    authorization: `Bearer ${this.opts.token}`,
+                },
+                body: JSON.stringify(record),
+                signal: AbortSignal.timeout(15_000),
+            });
+            if (res.ok)
+                return { ok: true, terminal: false, status: res.status };
+            let text = '';
+            try {
+                text = (await res.text()).slice(0, 200);
+            }
+            catch {
+                /* ignore */
+            }
+            return { ok: false, terminal: res.status >= 400 && res.status < 500, status: res.status, error: text };
+        }
+        catch (err) {
+            return { ok: false, terminal: false, status: null, error: err instanceof Error ? err.message : String(err) };
+        }
+    }
+}
+/** Test seam: expose the state loader so tests can assert persistence. */
+export function readRecorderState(stateFile) {
+    return loadState(stateFile);
+}