@totalreclaw/totalreclaw 3.3.12-rc.2 → 3.3.12-rc.21

package/dist/trajectory-poller.js

@@ -21,8 +21,59 @@
  *   existing extraction pipeline. Per-file byte-offset is tracked in
  *   ~/.totalreclaw/extract-state.json so we never re-process lines.
  *
- *   When the upstream OpenClaw bug is fixed, the agent_end hook starts
- *   firing again — both paths can coexist with offset-based dedup.
+ * RC1 capture strategy (Phase 4, 2026-06-22):
+ *   RC1 capture = POLLER-PRIMARY, FLUSH-SHADOWED. This poller is the
+ *   capture workhorse. OpenClaw's host-facing `flushPlanResolver`
+ *   (wired in native-memory.ts via buildFlushPlan) returns TR's
+ *   extraction plan so the memory slot is "complete" and the host CAN
+ *   drive flushes — but TR's encrypt→on-chain capture does NOT depend
+ *   on the host invoking the flush. The flush-driven capture path
+ *   (host flush → read scratch file → encrypt → on-chain) is NOT
+ *   wired on TR's side today; that is H2/RC2-gated. The poller
+ *   guarantees capture works regardless of host flush cadence: even
+ *   if the host never flushes, the poller still captures on its own
+ *   60 s schedule. This is the graceful-degradation stance —
+ *   recall-native + capture-poller meets both primary bars even if
+ *   H2 (host flush cadence) fails at the RC1 QA gate.
+ *
+ *   RC2 retires this poller IF H2 confirms the host invokes
+ *   flushPlanResolver at useful cadence. If H2 fails, the poller
+ *   stays as the long-term capture mechanism.
+ *
+ * Idempotency / offset-dedup:
+ *   The poller is idempotent across poller-restart and gateway-reload.
+ *   `extract-state.json` is the single source of truth for "what's
+ *   been consumed from each trajectory file": loadState() reads it at
+ *   the start of every poll iteration, parseNewMessages() only reads
+ *   bytes past the recorded offset, and the file is rewritten (via
+ *   saveState) only when at least one file's offset advanced. A
+ *   poller that crashes mid-extraction loses at most the in-flight
+ *   pass (state was not yet saved); the next poller re-reads from the
+ *   last persisted offset and re-runs the extraction — duplicate
+ *   facts are caught downstream by the dedup pass, and re-extraction
+ *   of the same slice is bounded to one retry (state then advances).
+ *
+ *   `parseNewMessages` caps newOffset at the last full newline so a
+ *   partially-flushed trailing line is re-read on the next poll
+ *   rather than dropped or double-counted.
+ *
+ * Cross-path double-write (NOT a risk today; RC2 work):
+ *   Today the poller is the only capture path that actually fires.
+ *   The agent_end / before_compaction / before_reset hook handlers
+ *   in index.ts still REGISTER storeExtractedFacts callbacks, but on
+ *   OpenClaw 2026.5.x the host never fires those events for
+ *   non-bundled plugins (the bug that motivated this poller), so in
+ *   practice the poller is the sole capture path. If a future
+ *   OpenClaw release un-blocks the hooks (or when the RC2
+ *   flush-driven capture path is wired), BOTH the poller AND the
+ *   parallel path would capture from overlapping state with NO
+ *   shared dedup — the hooks do NOT consult `extract-state.json`.
+ *   That is acceptable for RC1 (only the poller fires) but becomes
+ *   a real double-write risk at RC2. See the TODO(RC2/H2) marker on
+ *   STATE_FILE below for where the shared last-captured-offset
+ *   resolver would live. The earlier claim that "both paths can
+ *   coexist with offset-based dedup" was aspirational; no such
+ *   shared offset exists today.
  *
  * Module boundary (scanner constraint):
  *   This file does disk I/O (fs.read* on trajectory files + state file)
@@ -39,7 +90,24 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 const DEFAULT_POLL_INTERVAL_MS = 60_000;
+// Per-file offset tracker (poller's own source of truth for what's been
+// consumed from each trajectory file). Today this is consulted ONLY by the
+// poller — the agent_end / before_compaction / before_reset hooks in
+// index.ts do NOT read or write it, so if any of those hooks ever fire
+// alongside the poller they would double-capture from overlapping state.
+//
+// TODO(RC2/H2): when the flush-driven capture path is wired (host invokes
+// flushPlanResolver → TR reads the scratch file → encrypt → on-chain) OR
+// the upstream agent_end block is lifted, ALL capture paths MUST consult
+// a shared last-captured-offset resolver before encrypting. The simplest
+// shape is to expose loadState()/saveState() (or a `consumed(file)`
+// helper) to the hook handlers and the flush callback so they skip any
+// byte range the poller already consumed, and conversely the poller must
+// skip any range the hooks/flush path recorded. Until then, this file is
+// poller-private and there is no cross-path double-write risk (only the
+// poller fires on OpenClaw 2026.5.x).
 const STATE_FILE = path.join(os.homedir(), '.totalreclaw', 'extract-state.json');
 /**
  * Skip trajectory files older than this. A user who installs
@@ -50,16 +118,95 @@ const STATE_FILE = path.join(os.homedir(), '.totalreclaw', 'extract-state.json')
  * re-scanned, and skip the extraction path entirely.
  */
 const STALE_TRAJECTORY_AGE_MS = 7 * 24 * 60 * 60 * 1000; // 7 days
+/**
+ * Module-global handle to the currently-running poller (rc.20, #402).
+ *
+ * OpenClaw's SIGUSR1 restarts are IN-PROCESS — the module cache survives, so a
+ * same-version re-register hits THIS module instance again. Without a guard,
+ * each re-register started a fresh setInterval on top of the previous one and
+ * live pollers accumulated (a fresh container boot was observed with 2 from
+ * boot double-register). `startTrajectoryPoller` stops the previous poller
+ * from this module instance before starting a new one.
+ */
+let activePoller = null;
+/**
+ * Path to THIS module's own file, captured once at load. Each poll tick
+ * verifies it still exists AND is the same file (same inode + mtime); if the
+ * plugin dir was removed, or the file at this path was swapped for a different
+ * one (an old version uninstalled then a new version reinstalled at the SAME
+ * path within seconds — the gateway restart is an in-process signal, so the
+ * stale module instance survives), the poller self-terminates so a zombie
+ * module instance from a stale version can't keep submitting UserOps (rc.20,
+ * #402). An existence-only check missed the same-path reinstall case (review
+ * LOW-2, observed live: OpenClaw recreates dist at the same path in ~45s, so
+ * `existsSync` stayed true and the old-version poller ran on). Tests override
+ * this path via `opts.sentinelPath`.
+ */
+const MODULE_SENTINEL = fileURLToPath(import.meta.url);
 /**
  * Start the trajectory poller. Runs an initial poll after 5 s, then
  * every `pollIntervalMs` (default 60 s). Returns a handle the caller
  * can use to stop polling and run one-shot polls in tests.
+ *
+ * Lifecycle guards (rc.20, #402): a previously-started poller from this module
+ * instance is stopped first (singleton), and each tick self-terminates if this
+ * module's file is gone.
  */
 export function startTrajectoryPoller(deps, opts = {}) {
     const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
     const stateFile = opts.stateFile ?? STATE_FILE;
+    const sentinelPath = opts.sentinelPath ?? MODULE_SENTINEL;
+    // Capture the sentinel's file identity (inode + mtime) once at start so each
+    // tick can tell "same file still there" from "different file swapped in at
+    // the same path" (reinstall/upgrade). If the initial stat throws, fall back
+    // to existence-only semantics rather than crashing startup.
+    let sentinelIdentity = null;
+    try {
+        const st = fs.statSync(sentinelPath);
+        sentinelIdentity = { ino: st.ino, mtimeMs: st.mtimeMs };
+    }
+    catch {
+        sentinelIdentity = null;
+    }
+    // Singleton guard: stop any poller this module instance started earlier so a
+    // same-version re-register (in-process SIGUSR1 restart) does not stack a
+    // second live setInterval on top of the first.
+    if (activePoller) {
+        activePoller.stop();
+        deps.logger.info('extractd: previous poller stopped (re-register)');
+    }
+    let timer;
+    let initialTimeout;
+    const stop = () => {
+        if (timer)
+            clearInterval(timer);
+        if (initialTimeout)
+            clearTimeout(initialTimeout);
+        if (activePoller === handle)
+            activePoller = null;
+    };
     const pollAndExtract = async () => {
         try {
+            // Cross-version self-termination: if this module's own file is gone, or a
+            // DIFFERENT file was swapped in at the same path (uninstall→reinstall of a
+            // newer version), stop — a zombie poller from a stale module instance must
+            // not keep capturing.
+            let sentinelStat = null;
+            try {
+                sentinelStat = fs.statSync(sentinelPath);
+            }
+            catch {
+                // File gone (plugin dir removed).
+                deps.logger.warn('extractd: poller self-terminated (plugin dir removed)');
+                stop();
+                return;
+            }
+            if (sentinelIdentity !== null &&
+                (sentinelStat.ino !== sentinelIdentity.ino || sentinelStat.mtimeMs !== sentinelIdentity.mtimeMs)) {
+                deps.logger.warn('extractd: poller self-terminated (plugin file replaced — reinstall/upgrade detected)');
+                stop();
+                return;
+            }
             await deps.ensureInitialized();
             if (deps.isPairingPending())
                 return;
@@ -149,24 +296,23 @@ export function startTrajectoryPoller(deps, opts = {}) {
             deps.logger.error(`extractd: poll iteration failed: ${msg}`);
         }
     };
-    const timer = setInterval(() => {
+    timer = setInterval(() => {
         void pollAndExtract();
     }, pollIntervalMs);
     if (typeof timer.unref === 'function')
         timer.unref();
-    const initialTimeout = setTimeout(() => {
+    initialTimeout = setTimeout(() => {
         void pollAndExtract();
     }, 5_000);
     if (typeof initialTimeout.unref === 'function')
         initialTimeout.unref();
     deps.logger.info(`extractd: trajectory poller started (interval=${Math.round(pollIntervalMs / 1000)}s)`);
-    return {
-        stop: () => {
-            clearInterval(timer);
-            clearTimeout(initialTimeout);
-        },
+    const handle = {
+        stop,
         pollOnce: pollAndExtract,
     };
+    activePoller = handle;
+    return handle;
 }
 // ---------------------------------------------------------------------------
 // Filesystem scan + trajectory parser