failproofai 1.0.0-beta.2 → 1.0.0-beta.3

package/.next/standalone/crates/fpai-collect/src/health.rs

@@ -0,0 +1,369 @@
+//! Per-source liveness, written to `~/.failproofai/collector-health.json`.
+//!
+//! # Why per-source, and not one flag
+//!
+//! With a dozen sources, a single "collector ok" bit is worse than useless: a
+//! source whose root vanished, whose cursor desynced, or whose vendor changed
+//! its format goes quiet, and quiet is exactly what a healthy idle source looks
+//! like. The failure that motivates this is not hypothetical — a collector that
+//! captures nothing reads identically to one capturing everything, and the only
+//! way anyone notices is that a dashboard is emptier than expected weeks later.
+//!
+//! So every source reports three things it cannot fake: whether its root exists,
+//! when it last produced an event, and what went wrong most recently. A reader
+//! comparing "root present, zero events, no errors" against "root absent" can
+//! tell a broken source from an unused one.
+//!
+//! # Deliberately not a health CHECK
+//!
+//! This records facts and does not decide whether they are good. "No events for
+//! six hours" is an outage on a busy machine and completely normal on a laptop
+//! that was shut. Whoever consumes this has the context to judge; the daemon
+//! does not, and a threshold guessed here would produce alerts nobody trusts.
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+use serde::{Deserialize, Serialize};
+
+/// Longest error message kept. An error is a diagnostic, not a log — a
+/// multi-megabyte parse failure would make this file unreadable and could push
+/// a credential-bearing string into a file the daemon writes.
+const MAX_ERROR_LEN: usize = 500;
+
+/// What one source reports about itself.
+#[derive(Debug, Default, Clone, Deserialize, Serialize)]
+pub struct SourceHealth {
+    /// Whether the directory or database this source reads exists at all.
+    /// Distinguishes "this agent is not installed" from "this agent is broken".
+    pub root_present: bool,
+    /// Unix seconds of the last event this source produced. Zero means none
+    /// since the daemon started.
+    pub last_event_ts: u64,
+    /// Events produced since the daemon started.
+    pub events: u64,
+    /// How far through its input the source is, in whatever unit it uses
+    /// (bytes for a file tailer, rows for a database poller).
+    pub cursor: u64,
+    /// Most recent error, truncated. `None` once a poll succeeds again, so a
+    /// transient failure does not look permanent.
+    pub last_error: Option<String>,
+    /// Total failed polls. Kept even after `last_error` clears, so a source
+    /// that fails intermittently is distinguishable from one that never has.
+    pub errors: u64,
+}
+
+/// The whole record, as written.
+#[derive(Debug, Default, Deserialize, Serialize)]
+pub struct HealthFile {
+    /// When this snapshot was written, so a reader can tell a stale file from a
+    /// current one — a daemon that died leaves its last record behind.
+    pub ts: u64,
+    pub sources: BTreeMap<String, SourceHealth>,
+}
+
+/// Shared, cheap-to-update health state.
+///
+/// A `Mutex` rather than per-source atomics because the map is written whole:
+/// contention is a handful of updates a second across a dozen sources, and the
+/// alternative is a lock-free structure nobody needs.
+#[derive(Debug, Default)]
+pub struct Health {
+    sources: Mutex<BTreeMap<String, SourceHealth>>,
+    writes: AtomicU64,
+}
+
+impl Health {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Record that a source polled successfully.
+    ///
+    /// Clears `last_error` on purpose: a source that recovered should not keep
+    /// showing a fault from an hour ago, and the `errors` counter preserves the
+    /// fact that it happened.
+    pub fn record_poll(&self, source: &str, root_present: bool, events: u64, cursor: u64) {
+        let Ok(mut map) = self.sources.lock() else {
+            return; // poisoned: health is never worth propagating a panic
+        };
+        let entry = map.entry(source.to_string()).or_default();
+        entry.root_present = root_present;
+        entry.cursor = cursor;
+        entry.last_error = None;
+        if events > 0 {
+            entry.events += events;
+            entry.last_event_ts = now_secs();
+        }
+    }
+
+    /// Record that a source's poll failed.
+    pub fn record_error(&self, source: &str, error: &str) {
+        let Ok(mut map) = self.sources.lock() else {
+            return;
+        };
+        let entry = map.entry(source.to_string()).or_default();
+        entry.errors += 1;
+        entry.last_error = Some(truncate(error, MAX_ERROR_LEN));
+    }
+
+    /// Snapshot for writing.
+    pub fn snapshot(&self) -> HealthFile {
+        let sources = self.sources.lock().map(|m| m.clone()).unwrap_or_default();
+        HealthFile {
+            ts: now_secs(),
+            sources,
+        }
+    }
+
+    /// Write the record atomically (tmp → rename).
+    ///
+    /// No fsync: this is a heartbeat, not data. Losing the last few seconds of
+    /// it on a power cut costs nothing, and paying an fsync every interval for
+    /// a file nobody reconstructs state from would be the wrong trade.
+    pub fn write(&self, path: &Path) -> std::io::Result<()> {
+        if let Some(dir) = path.parent() {
+            std::fs::create_dir_all(dir)?;
+        }
+        let body = serde_json::to_string_pretty(&self.snapshot()).map_err(std::io::Error::other)?;
+        let tmp = path.with_extension("json.tmp");
+        std::fs::write(&tmp, body)?;
+        std::fs::rename(&tmp, path)?;
+        self.writes.fetch_add(1, Ordering::Relaxed);
+        Ok(())
+    }
+
+    pub fn writes(&self) -> u64 {
+        self.writes.load(Ordering::Relaxed)
+    }
+}
+
+/// The process-wide health registry.
+///
+/// A global rather than a field on every `Spec`, for the same reason `tracing`
+/// is: health is process-scoped telemetry that every engine touches, and
+/// threading it through each source's configuration would add a parameter to
+/// nine call sites and every test that builds one, to carry a value none of
+/// them vary. Unset until the daemon installs one, and every method on the
+/// returned handle is a no-op in that state — so a source reports health
+/// unconditionally and a test that never installs one pays nothing.
+static GLOBAL: std::sync::OnceLock<std::sync::Arc<Health>> = std::sync::OnceLock::new();
+
+/// Install the process health registry. The first call wins; later ones are
+/// ignored, so a stray second install cannot silently orphan the record the
+/// writer task is publishing.
+pub fn install(health: std::sync::Arc<Health>) {
+    let _ = GLOBAL.set(health);
+}
+
+/// Report a successful poll to the process registry, if one is installed.
+pub fn report_poll(source: &str, root_present: bool, events: u64, cursor: u64) {
+    if let Some(h) = GLOBAL.get() {
+        h.record_poll(source, root_present, events, cursor);
+    }
+}
+
+/// Report a failed poll to the process registry, if one is installed.
+pub fn report_error(source: &str, error: &str) {
+    if let Some(h) = GLOBAL.get() {
+        h.record_error(source, error);
+    }
+}
+
+/// Where the record lives.
+pub fn health_path(home: &Path) -> PathBuf {
+    home.join("collector-health.json")
+}
+
+/// Remove the record on a clean shutdown.
+///
+/// Absence means "no daemon running", which is unambiguous. Leaving a stale
+/// file behind makes a stopped daemon look like a running one whose sources all
+/// went quiet — the exact confusion this module exists to prevent.
+pub fn remove(path: &Path) {
+    let _ = std::fs::remove_file(path);
+}
+
+/// Rewrite the record on an interval until shutdown, then delete it.
+///
+/// A task rather than a write-on-every-change: sources update health many times
+/// a second, and rewriting the file each time would turn a diagnostic into a
+/// disk-IO cost on the hot path.
+pub async fn writer_task(
+    health: std::sync::Arc<Health>,
+    path: PathBuf,
+    interval: std::time::Duration,
+    sd: crate::supervisor::Shutdown,
+) -> Result<(), crate::supervisor::TaskError> {
+    loop {
+        if let Err(err) = health.write(&path) {
+            // Logged, never fatal: failing to write a heartbeat must not stop
+            // the collection it is describing.
+            tracing::warn!(path = %path.display(), %err, "could not write the collector health record");
+        }
+        if !sd.sleep(interval).await {
+            // Clean shutdown: absence of the file means "no daemon running",
+            // which is unambiguous in a way a stale record is not.
+            remove(&path);
+            return Ok(());
+        }
+    }
+}
+
+/// How often the record is rewritten.
+pub const WRITE_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+
+fn now_secs() -> u64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_secs())
+        .unwrap_or(0)
+}
+
+fn truncate(s: &str, cap: usize) -> String {
+    if s.len() <= cap {
+        return s.to_string();
+    }
+    let mut end = cap;
+    while end > 0 && !s.is_char_boundary(end) {
+        end -= 1;
+    }
+    format!("{}…", &s[..end])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::time::{SystemTime, UNIX_EPOCH};
+
+    fn tmpdir(name: &str) -> PathBuf {
+        let d = std::env::temp_dir().join(format!(
+            "fpai-health-{}-{}-{}",
+            name,
+            std::process::id(),
+            SystemTime::now()
+                .duration_since(UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        std::fs::create_dir_all(&d).unwrap();
+        d
+    }
+
+    #[test]
+    fn a_broken_source_is_distinguishable_from_an_unused_one() {
+        // The whole point. Both produce zero events; only one is a problem.
+        let h = Health::new();
+        h.record_poll("claude", true, 0, 0); // installed, quiet
+        h.record_poll("goose", false, 0, 0); // not installed at all
+
+        let snap = h.snapshot();
+        assert!(snap.sources["claude"].root_present);
+        assert!(!snap.sources["goose"].root_present);
+        assert_eq!(snap.sources["claude"].events, 0);
+    }
+
+    #[test]
+    fn a_recovered_source_stops_showing_a_stale_error_but_keeps_the_count() {
+        // A source that failed an hour ago and works now must not read as
+        // broken — while still being distinguishable from one that never failed.
+        let h = Health::new();
+        h.record_error("codex", "permission denied");
+        assert_eq!(h.snapshot().sources["codex"].errors, 1);
+        assert!(h.snapshot().sources["codex"].last_error.is_some());
+
+        h.record_poll("codex", true, 3, 900);
+        let s = &h.snapshot().sources["codex"];
+        assert!(
+            s.last_error.is_none(),
+            "a recovered source must clear its error"
+        );
+        assert_eq!(s.errors, 1, "but the fact it happened must survive");
+    }
+
+    #[test]
+    fn events_accumulate_and_stamp_a_time_only_when_there_were_some() {
+        // A poll that found nothing must not advance last_event_ts, or an idle
+        // source would look permanently busy.
+        let h = Health::new();
+        h.record_poll("claude", true, 0, 10);
+        assert_eq!(h.snapshot().sources["claude"].last_event_ts, 0);
+
+        h.record_poll("claude", true, 5, 20);
+        let s = &h.snapshot().sources["claude"];
+        assert_eq!(s.events, 5);
+        assert!(s.last_event_ts > 0);
+        assert_eq!(s.cursor, 20);
+
+        h.record_poll("claude", true, 2, 30);
+        assert_eq!(
+            h.snapshot().sources["claude"].events,
+            7,
+            "events accumulate"
+        );
+    }
+
+    #[test]
+    fn an_enormous_error_cannot_bloat_the_record() {
+        // An error is a diagnostic, not a log. A multi-megabyte parse failure
+        // would make the file unreadable.
+        let h = Health::new();
+        h.record_error("pi", &"x".repeat(100_000));
+        let msg = h.snapshot().sources["pi"].last_error.clone().unwrap();
+        assert!(msg.len() <= MAX_ERROR_LEN + 4, "got {} bytes", msg.len());
+    }
+
+    #[test]
+    fn a_multibyte_error_is_truncated_on_a_char_boundary() {
+        // Slicing mid-codepoint would panic inside the health writer, which is
+        // the last place a panic is acceptable.
+        let h = Health::new();
+        h.record_error("pi", &"é".repeat(1000));
+        assert!(h.snapshot().sources["pi"].last_error.is_some());
+    }
+
+    #[test]
+    fn the_record_round_trips_through_disk() {
+        let dir = tmpdir("write");
+        let path = health_path(&dir);
+        let h = Health::new();
+        h.record_poll("claude", true, 4, 100);
+        h.record_error("codex", "boom");
+        h.write(&path).unwrap();
+
+        let parsed: HealthFile =
+            serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
+        assert!(parsed.ts > 0, "a reader must be able to spot a stale file");
+        assert_eq!(parsed.sources["claude"].events, 4);
+        assert_eq!(parsed.sources["codex"].last_error.as_deref(), Some("boom"));
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn writing_leaves_no_tmp_file_behind() {
+        let dir = tmpdir("tmp");
+        let path = health_path(&dir);
+        Health::new().write(&path).unwrap();
+        let stray = std::fs::read_dir(&dir)
+            .unwrap()
+            .filter_map(|e| e.ok())
+            .any(|e| e.file_name().to_string_lossy().ends_with(".tmp"));
+        assert!(!stray, "the atomic write left its tmp behind");
+        std::fs::remove_dir_all(&dir).ok();
+    }
+
+    #[test]
+    fn a_clean_shutdown_removes_the_record() {
+        // Absence means "no daemon". A stale file makes a stopped daemon look
+        // like a running one whose sources all went quiet.
+        let dir = tmpdir("remove");
+        let path = health_path(&dir);
+        Health::new().write(&path).unwrap();
+        assert!(path.exists());
+        remove(&path);
+        assert!(!path.exists());
+        std::fs::remove_dir_all(&dir).ok();
+    }
+}

package/.next/standalone/crates/fpai-collect/src/lib.rs

@@ -0,0 +1,34 @@
+//! Session-log and hook-activity collection for `failproofaid`.
+//!
+//! This crate is deliberately separate from `failproofaid` rather than a
+//! module inside it. The daemon crate gates every tool call on the machine
+//! (its CLI fails closed on an unreachable daemon), so it is kept small and
+//! its tests kept fast; collection is a much larger body of code with a much
+//! lower blast radius, and it should be buildable and testable without
+//! standing up a socket server at all.
+//!
+//! The only thing the daemon needs from here is
+//! [`supervisor::spawn_supervised`], which hands back a handle it joins during
+//! shutdown. Everything else is internal.
+
+pub mod config;
+pub mod cursor;
+pub mod delivery;
+pub mod filetail;
+pub mod health;
+pub mod redact;
+pub mod sources;
+pub mod spool;
+pub mod sqlitepoll;
+pub mod supervisor;
+pub mod uploader;
+
+pub use config::{CollectorConfig, DEFAULT_INGEST_URL, HooksVerbosity, Ingest, Redact, Settings};
+pub use delivery::Delivery;
+pub use health::{Health, HealthFile, SourceHealth, health_path};
+pub use spool::SpoolWriter;
+pub use supervisor::{
+    CollectorHandle, DEFAULT_FLUSH_BUDGET, Shutdown, SupervisorMetrics, TaskError, TaskSpec,
+    spawn_supervised,
+};
+pub use uploader::{UploadError, UploadMetrics, Uploader};