@matt82198/aesop 0.1.0 → 0.3.1

package/state_store/coordination.py

@@ -0,0 +1,209 @@
+"""state_store.coordination — lease-by-append claim for exclusive resource access.
+
+Implements DB-native CLAIM (lease-via-append) on top of the existing event log:
+to claim resource R, append a claim_requested event to the 'claims' stream;
+the winner is the lowest-version append for that resource key; readers fold
+the stream to see who holds it.
+
+Fail-CLOSED by construction: if you cannot append or read, you do NOT hold
+the claim and must not dispatch. TTL + claim_released events handle crashed
+holders (analogous to lock.mjs PID-liveness staleness, but expressed as events).
+
+No risky changes to store.py or api.py — uses only existing append/read primitives.
+
+Stdlib only: time.
+"""
+from __future__ import annotations
+
+import time
+
+
+def fold_claims(events: list) -> dict[str, str]:
+    """Fold a claims stream into the current state of who holds each resource.
+
+    Processes claim_requested and claim_released events to determine the winner
+    for each resource. The winner is the lowest-version un-released claim for
+    that resource (first serialized append wins; ties impossible because versions
+    are unique). A claim is valid only if it has NOT been released by a
+    subsequent claim_released event. If an instance releases and then re-claims,
+    the re-claim is the new active claim for that instance.
+
+    Args:
+        events: list of event dicts from the claims stream
+                (as returned by EventStore.read('claims'))
+
+    Returns:
+        dict mapping resource_id -> holding_instance_id for all currently
+        held resources. Empty dict if no claims exist or all have been released.
+    """
+    # Track all claims by resource and instance (as list, to preserve order)
+    claims_by_resource = {}  # resource -> {instance_id: [versions...]}
+    # Track all releases by resource and instance (as sorted list)
+    releases_by_resource = {}  # resource -> {instance_id: [release_versions]}
+
+    # First pass: collect all claims and releases
+    for ev in events:
+        etype = ev.get("type")
+        payload = ev.get("payload") or {}
+        version = ev.get("version", 0)
+
+        if etype == "claim_requested":
+            resource = payload.get("resource")
+            instance_id = payload.get("instance_id")
+            if resource is not None and instance_id is not None:
+                if resource not in claims_by_resource:
+                    claims_by_resource[resource] = {}
+                if instance_id not in claims_by_resource[resource]:
+                    claims_by_resource[resource][instance_id] = []
+                claims_by_resource[resource][instance_id].append(version)
+
+        elif etype == "claim_released":
+            resource = payload.get("resource")
+            instance_id = payload.get("instance_id")
+            if resource is not None and instance_id is not None:
+                if resource not in releases_by_resource:
+                    releases_by_resource[resource] = {}
+                if instance_id not in releases_by_resource[resource]:
+                    releases_by_resource[resource][instance_id] = []
+                releases_by_resource[resource][instance_id].append(version)
+
+    # Second pass: determine current holders
+    holders = {}
+    for resource, claims_dict in claims_by_resource.items():
+        # For each instance claiming this resource, find the latest un-released claim
+        active_claims = {}
+        for instance_id, claim_versions in claims_dict.items():
+            releases = sorted(releases_by_resource.get(resource, {}).get(instance_id, []))
+
+            # Process claims in order, tracking "current active" claim within streaks
+            # (separated by releases). The current active claim is the latest claim
+            # that comes after the most recent release.
+            current_active = None
+            for claim_v in sorted(claim_versions):
+                # Check if there's a release between the current active and this claim
+                if current_active is not None:
+                    # Check if current_active was released
+                    if any(r > current_active for r in releases):
+                        # Yes, released; start a new streak with this claim
+                        current_active = claim_v
+                    # else: already in this streak, keep current_active
+                else:
+                    # First claim for this instance
+                    current_active = claim_v
+
+            # After processing all claims, check if current_active is released
+            if current_active is not None:
+                if not any(r > current_active for r in releases):
+                    # Not released; it's active
+                    active_claims[instance_id] = current_active
+
+        # Find the minimum version among active claims (the winner)
+        if active_claims:
+            winner_instance = min(active_claims.keys(), key=lambda x: active_claims[x])
+            holders[resource] = winner_instance
+
+    return holders
+
+
+def try_claim(store, resource: str, instance_id: str, ttl: float = 300.0) -> bool:
+    """Attempt to claim exclusive access to a resource.
+
+    Appends a claim_requested event to the 'claims' stream, then re-reads
+    and folds to check if this instance won the claim. Fail-CLOSED: if ANY
+    exception occurs (append fails, read fails, or exception during fold),
+    return False (claim not held, do not proceed).
+
+    If this instance does NOT win, it retracts its claim by appending a
+    claim_released event (scoped to this instance + resource) before returning
+    False. This prevents stale-claim resurrection: a losing claim left un-retracted
+    in the stream could later become the winner if the true holder releases.
+
+    Args:
+        store: StateAPI or EventStore instance (must have append() and get() methods)
+        resource: the resource identifier to claim (e.g., "wave_123", "lane_0", ...)
+        instance_id: the instance identifier requesting the claim
+        ttl: time-to-live in seconds (default 300s = 5min); embedded in the payload
+             for later TTL-based expiry checks (not enforced here, but available
+             for reconciliation)
+
+    Returns:
+        bool: True if this instance holds the claim after the append,
+              False otherwise (including on any error).
+    """
+    try:
+        # Append claim request to the claims stream
+        store.append(
+            "claims",
+            "claim_requested",
+            {"resource": resource, "instance_id": instance_id, "ttl": ttl},
+            actor=instance_id,
+        )
+
+        # Re-read claims stream and fold to see current state
+        events = store.get("claims")
+        claims = fold_claims(events)
+
+        # Check if we won
+        if claims.get(resource) == instance_id:
+            return True
+
+        # We did NOT win: retract our claim to prevent stale-claim resurrection.
+        # Fail-closed: if retract fails, still return False (never a false grant).
+        try:
+            store.append(
+                "claims",
+                "claim_released",
+                {"resource": resource, "instance_id": instance_id},
+                actor=instance_id,
+            )
+        except Exception:
+            # Retract failed, but we still don't hold the claim; return False.
+            pass
+
+        return False
+    except Exception:
+        # Fail-closed: any exception means we don't hold the claim
+        return False
+
+
+def release(store, resource: str, instance_id: str) -> None:
+    """Release a claimed resource.
+
+    Appends a claim_released event to the 'claims' stream, marking that
+    this instance no longer holds the resource. Idempotent: releasing
+    a resource that was not held is a no-op (the fold will see the
+    release but no matching claim, so it has no effect).
+
+    Args:
+        store: StateAPI or EventStore instance (must have append() method)
+        resource: the resource identifier being released
+        instance_id: the instance identifier releasing the claim
+    """
+    store.append(
+        "claims",
+        "claim_released",
+        {"resource": resource, "instance_id": instance_id},
+        actor=instance_id,
+    )
+
+
+def current_holder(store, resource: str) -> str | None:
+    """Return the instance_id currently holding a resource, or None if unclaimed.
+
+    Reads and folds the claims stream to find the winner for the resource.
+    Returns None if the resource is not claimed or has been released.
+
+    Args:
+        store: StateAPI or EventStore instance (must have get() method)
+        resource: the resource identifier to query
+
+    Returns:
+        The instance_id of the current holder, or None if unclaimed.
+    """
+    try:
+        events = store.get("claims")
+        claims = fold_claims(events)
+        return claims.get(resource)
+    except Exception:
+        # Fail-closed: on any error, we cannot determine the holder
+        return None

package/state_store/identity.py

@@ -0,0 +1,51 @@
+"""state_store.identity — stable per-instance orchestrator identity.
+
+Derives a stable instance_id for each orchestrator process: hostname:pid:nonce.
+The nonce is a short random string (for entropy) so two orchestrators on the
+same box with different pids still get different identities.
+
+Identity is deterministic within a process lifetime (cached) but varies
+across process restarts (the nonce is not stable across restarts by design,
+so instance_id changes on restart). Recommended: auto-derived for zero-config
+solo mode, with optional aesop.config.json override for stable identities
+across restarts.
+
+Stdlib only: socket, os, random, string.
+"""
+from __future__ import annotations
+
+import os
+import random
+import socket
+import string
+
+
+# Cached instance_id, computed once per process
+_INSTANCE_ID_CACHE: str | None = None
+
+
+def get_instance_id() -> str:
+    """Return a stable per-instance orchestrator id.
+
+    Computed once per process and cached. The id is the form:
+        hostname:pid:nonce
+
+    Where nonce is a random 6-character alphanumeric string, used to
+    differentiate multiple orchestrators on the same box.
+
+    Returns:
+        A stable string identifier for this orchestrator instance.
+    """
+    global _INSTANCE_ID_CACHE
+    if _INSTANCE_ID_CACHE is None:
+        _INSTANCE_ID_CACHE = _derive_instance_id()
+    return _INSTANCE_ID_CACHE
+
+
+def _derive_instance_id() -> str:
+    """Derive a new instance_id from hostname, pid, and random nonce."""
+    hostname = socket.gethostname()
+    pid = os.getpid()
+    # Random 6-char alphanumeric nonce for entropy across processes
+    nonce = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
+    return f"{hostname}:{pid}:{nonce}"

package/state_store/projections.py

@@ -124,3 +124,66 @@ def save_snapshot(store, stream: str, event_version: int, projection: dict) -> N
         projection: the materialized state dict (e.g. full tracker projection)
     """
     store.save_snapshot(stream, event_version, projection)
+
+
+AGENT_LIFECYCLE_VERSION = 1
+
+
+def project_agent_lifecycle(events: list) -> dict:
+    """Project current agent lifecycle state from agent_* events.
+
+    Folds agent lifecycle events into a map of agent_id -> {state, transitions, last_activity}.
+
+    Event types:
+    - agent_dispatched: payload {"agent_id", "timestamp"}; marks dispatch start
+    - agent_working: payload {"agent_id", "timestamp"}; marks work in progress
+    - agent_done: payload {"agent_id", "timestamp"}; marks completion
+    - agent_stalled: payload {"agent_id", "timestamp"}; marks stall/error
+
+    Returns a dict: {"version": 1, "agents": [{"id", "state", "transitions", "last_activity"}, ...]}
+    where transitions is a list of {state, at (ISO 8601 timestamp)} in chronological order.
+    """
+    agents = {}
+
+    for ev in events:
+        etype = ev.get("type")
+        payload = ev.get("payload") or {}
+
+        if etype in ("agent_dispatched", "agent_working", "agent_done", "agent_stalled"):
+            agent_id = payload.get("agent_id")
+            timestamp = payload.get("timestamp")
+
+            if agent_id is None:
+                continue
+
+            # Initialize agent if first time seeing it
+            if agent_id not in agents:
+                agents[agent_id] = {
+                    "id": agent_id,
+                    "state": None,
+                    "transitions": [],
+                    "last_activity": None,
+                }
+
+            # Map event type to state
+            state_map = {
+                "agent_dispatched": "dispatch",
+                "agent_working": "working",
+                "agent_done": "done",
+                "agent_stalled": "stalled",
+            }
+            new_state = state_map.get(etype)
+
+            # Only append transition if state actually changed
+            if new_state and new_state != agents[agent_id]["state"]:
+                agents[agent_id]["state"] = new_state
+                if timestamp:
+                    agents[agent_id]["transitions"].append({
+                        "state": new_state,
+                        "at": timestamp,
+                    })
+                    agents[agent_id]["last_activity"] = timestamp
+
+    # Return as versioned list (ordered by agent_id for determinism)
+    agent_list = sorted(agents.values(), key=lambda a: a["id"])
+    return {"version": AGENT_LIFECYCLE_VERSION, "agents": agent_list}

package/state_store/read_api.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""
+state_store.read_api — Typed read-only facade for state surfaces.
+
+Consolidates read patterns for: tracker snapshot, orchestrator-status,
+heartbeat freshness, and ledger rows. This facade allows the underlying
+state representation to change (git → SQLite → Postgres) without altering
+caller code.
+
+Callers use:
+  api = ReadAPI(state_dir)
+  tracker = api.read_tracker_snapshot()
+  status = api.read_orchestrator_status()
+  is_fresh = api.is_orchestrator_status_fresh(threshold_s=300)
+  is_fresh = api.check_heartbeat_fresh(".watchdog-heartbeat", 200)
+  rows = api.read_ledger_rows()
+"""
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+# Ensure tools module is importable (sys.path fix for bootstrapping)
+repo_root = Path(__file__).parent.parent
+if str(repo_root) not in sys.path:
+    sys.path.insert(0, str(repo_root))
+
+from tools.common import check_heartbeat_staleness
+from tools.fleet_ledger import parse_ledger_rows as parse_ledger_rows_impl
+
+
+class ReadAPI:
+    """Read-only facade over state surfaces (tracker, orchestrator-status, heartbeats, ledger).
+
+    Designed to be swappable: backends can change (git → SQLite → Postgres) without
+    altering call sites. Current implementation reads from filesystem; future may
+    read from SQLite projections or API.
+    """
+
+    def __init__(self, state_dir):
+        """Initialize the read API with a state directory.
+
+        Args:
+            state_dir: Path to the state directory (e.g., "state" or "/absolute/path/state")
+        """
+        self.state_dir = Path(state_dir)
+        self.state_dir.mkdir(parents=True, exist_ok=True)
+
+    def read_tracker_snapshot(self):
+        """Read the current tracker snapshot.
+
+        Returns the tracker.json file, or an empty dict if missing/unreadable.
+        In the future, this may read from the SQLite projection instead.
+
+        Returns:
+            dict: Tracker snapshot (items, metadata, etc.) or {} if unavailable.
+        """
+        tracker_file = self.state_dir / "tracker.json"
+        if not tracker_file.exists():
+            return {}
+
+        try:
+            content = tracker_file.read_text(encoding="utf-8")
+            return json.loads(content)
+        except Exception:
+            # Malformed JSON or read error: return empty dict (fail-open)
+            return {}
+
+    def read_orchestrator_status(self):
+        """Read orchestrator-status.json if present.
+
+        Returns:
+            dict with "updated_at" (required) and optional "phase", "activity" fields,
+            or None if file missing/unreadable.
+        """
+        status_file = self.state_dir / "orchestrator-status.json"
+        if not status_file.exists():
+            return None
+
+        try:
+            content = status_file.read_text(encoding="utf-8")
+            data = json.loads(content)
+            # Only require updated_at field for freshness checks;
+            # phase is optional (may not be set when status.json created before phase exists)
+            # Don't validate here—let callers decide what fields are required for their use case
+            return data
+        except Exception:
+            return None
+
+    def is_orchestrator_status_fresh(self, threshold_s=300):
+        """Check if orchestrator-status.json exists and is fresh.
+
+        Reads updated_at timestamp and compares to current time.
+        Returns False if file missing, unparseable, or stale.
+
+        Args:
+            threshold_s: Staleness threshold in seconds (default: 300s)
+
+        Returns:
+            bool: True if file is fresh, False if stale/missing/malformed.
+        """
+        status = self.read_orchestrator_status()
+        if status is None:
+            return False
+
+        try:
+            updated_at_str = status.get("updated_at")
+            if not updated_at_str:
+                return False
+
+            # Parse ISO format timestamp (handle both "Z" and "+00:00")
+            normalized = updated_at_str.replace("Z", "+00:00")
+            updated_at = datetime.fromisoformat(normalized)
+
+            now = datetime.now(timezone.utc)
+            age = (now - updated_at).total_seconds()
+
+            # Treat future-dated timestamps as stale (fail-closed)
+            if age < -120:  # More than 2min in future = clock skew, treat as stale
+                return False
+
+            # Clamp small negative ages to 0
+            age = max(0, age)
+
+            return age < threshold_s
+        except Exception:
+            return False
+
+    def check_heartbeat_fresh(self, heartbeat_filename, threshold_s=200):
+        """Check if a heartbeat file is fresh.
+
+        Delegates to tools.common.check_heartbeat_staleness (single source of truth).
+
+        Args:
+            heartbeat_filename: Filename in state_dir (e.g., ".watchdog-heartbeat")
+            threshold_s: Staleness threshold in seconds (default: 200s)
+
+        Returns:
+            bool: True if file exists and is fresh, False otherwise.
+        """
+        hb_file = self.state_dir / heartbeat_filename
+        # Use the shared implementation from common.py; invert staleness to freshness
+        is_stale, _, _ = check_heartbeat_staleness(hb_file, threshold_s)
+        return not is_stale
+
+    def read_ledger_rows(self):
+        """Read OUTCOMES-LEDGER.md and parse rows.
+
+        Delegates to tools.fleet_ledger.parse_ledger_rows() (single source of truth).
+        Returns empty list if file missing or unparseable.
+
+        Returns:
+            list: List of row dicts parsed from the ledger table, or [].
+        """
+        # Use the shared implementation from fleet_ledger.py
+        return parse_ledger_rows_impl()