@matt82198/aesop 0.1.0-rc.1 → 0.2.0

package/state_store/coordination.py

@@ -0,0 +1,209 @@
+"""state_store.coordination — lease-by-append claim for exclusive resource access.
+
+Implements DB-native CLAIM (lease-via-append) on top of the existing event log:
+to claim resource R, append a claim_requested event to the 'claims' stream;
+the winner is the lowest-version append for that resource key; readers fold
+the stream to see who holds it.
+
+Fail-CLOSED by construction: if you cannot append or read, you do NOT hold
+the claim and must not dispatch. TTL + claim_released events handle crashed
+holders (analogous to lock.mjs PID-liveness staleness, but expressed as events).
+
+No risky changes to store.py or api.py — uses only existing append/read primitives.
+
+Stdlib only: time.
+"""
+from __future__ import annotations
+
+import time
+
+
+def fold_claims(events: list) -> dict[str, str]:
+    """Fold a claims stream into the current state of who holds each resource.
+
+    Processes claim_requested and claim_released events to determine the winner
+    for each resource. The winner is the lowest-version un-released claim for
+    that resource (first serialized append wins; ties impossible because versions
+    are unique). A claim is valid only if it has NOT been released by a
+    subsequent claim_released event. If an instance releases and then re-claims,
+    the re-claim is the new active claim for that instance.
+
+    Args:
+        events: list of event dicts from the claims stream
+                (as returned by EventStore.read('claims'))
+
+    Returns:
+        dict mapping resource_id -> holding_instance_id for all currently
+        held resources. Empty dict if no claims exist or all have been released.
+    """
+    # Track all claims by resource and instance (as list, to preserve order)
+    claims_by_resource = {}  # resource -> {instance_id: [versions...]}
+    # Track all releases by resource and instance (as sorted list)
+    releases_by_resource = {}  # resource -> {instance_id: [release_versions]}
+
+    # First pass: collect all claims and releases
+    for ev in events:
+        etype = ev.get("type")
+        payload = ev.get("payload") or {}
+        version = ev.get("version", 0)
+
+        if etype == "claim_requested":
+            resource = payload.get("resource")
+            instance_id = payload.get("instance_id")
+            if resource is not None and instance_id is not None:
+                if resource not in claims_by_resource:
+                    claims_by_resource[resource] = {}
+                if instance_id not in claims_by_resource[resource]:
+                    claims_by_resource[resource][instance_id] = []
+                claims_by_resource[resource][instance_id].append(version)
+
+        elif etype == "claim_released":
+            resource = payload.get("resource")
+            instance_id = payload.get("instance_id")
+            if resource is not None and instance_id is not None:
+                if resource not in releases_by_resource:
+                    releases_by_resource[resource] = {}
+                if instance_id not in releases_by_resource[resource]:
+                    releases_by_resource[resource][instance_id] = []
+                releases_by_resource[resource][instance_id].append(version)
+
+    # Second pass: determine current holders
+    holders = {}
+    for resource, claims_dict in claims_by_resource.items():
+        # For each instance claiming this resource, find the latest un-released claim
+        active_claims = {}
+        for instance_id, claim_versions in claims_dict.items():
+            releases = sorted(releases_by_resource.get(resource, {}).get(instance_id, []))
+
+            # Process claims in order, tracking "current active" claim within streaks
+            # (separated by releases). The current active claim is the latest claim
+            # that comes after the most recent release.
+            current_active = None
+            for claim_v in sorted(claim_versions):
+                # Check if there's a release between the current active and this claim
+                if current_active is not None:
+                    # Check if current_active was released
+                    if any(r > current_active for r in releases):
+                        # Yes, released; start a new streak with this claim
+                        current_active = claim_v
+                    # else: already in this streak, keep current_active
+                else:
+                    # First claim for this instance
+                    current_active = claim_v
+
+            # After processing all claims, check if current_active is released
+            if current_active is not None:
+                if not any(r > current_active for r in releases):
+                    # Not released; it's active
+                    active_claims[instance_id] = current_active
+
+        # Find the minimum version among active claims (the winner)
+        if active_claims:
+            winner_instance = min(active_claims.keys(), key=lambda x: active_claims[x])
+            holders[resource] = winner_instance
+
+    return holders
+
+
+def try_claim(store, resource: str, instance_id: str, ttl: float = 300.0) -> bool:
+    """Attempt to claim exclusive access to a resource.
+
+    Appends a claim_requested event to the 'claims' stream, then re-reads
+    and folds to check if this instance won the claim. Fail-CLOSED: if ANY
+    exception occurs (append fails, read fails, or exception during fold),
+    return False (claim not held, do not proceed).
+
+    If this instance does NOT win, it retracts its claim by appending a
+    claim_released event (scoped to this instance + resource) before returning
+    False. This prevents stale-claim resurrection: a losing claim left un-retracted
+    in the stream could later become the winner if the true holder releases.
+
+    Args:
+        store: StateAPI or EventStore instance (must have append() and get() methods)
+        resource: the resource identifier to claim (e.g., "wave_123", "lane_0", ...)
+        instance_id: the instance identifier requesting the claim
+        ttl: time-to-live in seconds (default 300s = 5min); embedded in the payload
+             for later TTL-based expiry checks (not enforced here, but available
+             for reconciliation)
+
+    Returns:
+        bool: True if this instance holds the claim after the append,
+              False otherwise (including on any error).
+    """
+    try:
+        # Append claim request to the claims stream
+        store.append(
+            "claims",
+            "claim_requested",
+            {"resource": resource, "instance_id": instance_id, "ttl": ttl},
+            actor=instance_id,
+        )
+
+        # Re-read claims stream and fold to see current state
+        events = store.get("claims")
+        claims = fold_claims(events)
+
+        # Check if we won
+        if claims.get(resource) == instance_id:
+            return True
+
+        # We did NOT win: retract our claim to prevent stale-claim resurrection.
+        # Fail-closed: if retract fails, still return False (never a false grant).
+        try:
+            store.append(
+                "claims",
+                "claim_released",
+                {"resource": resource, "instance_id": instance_id},
+                actor=instance_id,
+            )
+        except Exception:
+            # Retract failed, but we still don't hold the claim; return False.
+            pass
+
+        return False
+    except Exception:
+        # Fail-closed: any exception means we don't hold the claim
+        return False
+
+
+def release(store, resource: str, instance_id: str) -> None:
+    """Release a claimed resource.
+
+    Appends a claim_released event to the 'claims' stream, marking that
+    this instance no longer holds the resource. Idempotent: releasing
+    a resource that was not held is a no-op (the fold will see the
+    release but no matching claim, so it has no effect).
+
+    Args:
+        store: StateAPI or EventStore instance (must have append() method)
+        resource: the resource identifier being released
+        instance_id: the instance identifier releasing the claim
+    """
+    store.append(
+        "claims",
+        "claim_released",
+        {"resource": resource, "instance_id": instance_id},
+        actor=instance_id,
+    )
+
+
+def current_holder(store, resource: str) -> str | None:
+    """Return the instance_id currently holding a resource, or None if unclaimed.
+
+    Reads and folds the claims stream to find the winner for the resource.
+    Returns None if the resource is not claimed or has been released.
+
+    Args:
+        store: StateAPI or EventStore instance (must have get() method)
+        resource: the resource identifier to query
+
+    Returns:
+        The instance_id of the current holder, or None if unclaimed.
+    """
+    try:
+        events = store.get("claims")
+        claims = fold_claims(events)
+        return claims.get(resource)
+    except Exception:
+        # Fail-closed: on any error, we cannot determine the holder
+        return None

package/state_store/identity.py

@@ -0,0 +1,51 @@
+"""state_store.identity — stable per-instance orchestrator identity.
+
+Derives a stable instance_id for each orchestrator process: hostname:pid:nonce.
+The nonce is a short random string (for entropy) so two orchestrators on the
+same box with different pids still get different identities.
+
+Identity is deterministic within a process lifetime (cached) but varies
+across process restarts (the nonce is not stable across restarts by design,
+so instance_id changes on restart). Recommended: auto-derived for zero-config
+solo mode, with optional aesop.config.json override for stable identities
+across restarts.
+
+Stdlib only: socket, os, random, string.
+"""
+from __future__ import annotations
+
+import os
+import random
+import socket
+import string
+
+
+# Cached instance_id, computed once per process
+_INSTANCE_ID_CACHE: str | None = None
+
+
+def get_instance_id() -> str:
+    """Return a stable per-instance orchestrator id.
+
+    Computed once per process and cached. The id is the form:
+        hostname:pid:nonce
+
+    Where nonce is a random 6-character alphanumeric string, used to
+    differentiate multiple orchestrators on the same box.
+
+    Returns:
+        A stable string identifier for this orchestrator instance.
+    """
+    global _INSTANCE_ID_CACHE
+    if _INSTANCE_ID_CACHE is None:
+        _INSTANCE_ID_CACHE = _derive_instance_id()
+    return _INSTANCE_ID_CACHE
+
+
+def _derive_instance_id() -> str:
+    """Derive a new instance_id from hostname, pid, and random nonce."""
+    hostname = socket.gethostname()
+    pid = os.getpid()
+    # Random 6-char alphanumeric nonce for entropy across processes
+    nonce = "".join(random.choices(string.ascii_lowercase + string.digits, k=6))
+    return f"{hostname}:{pid}:{nonce}"

package/state_store/store.py

@@ -16,6 +16,60 @@ import sqlite3
 import sys
 import time
 
+# Retry configuration for database lock contention (especially under parallel CI shards)
+_MAX_DB_LOCK_RETRIES = 3
+_DB_LOCK_RETRY_BASE_DELAY = 0.05  # 50ms base, exponential backoff
+
+
+class ConcurrencyConflict(Exception):
+    """Raised when an optimistic concurrency control check fails on append.
+
+    Signifies that the event stream's current version does not match the
+    expected version provided to append(), indicating that another writer
+    has extended the stream since the caller last read it. No event is
+    written when this exception is raised (fail-closed).
+
+    Attributes:
+        expected_version: The version the caller expected
+        actual_version: The version found in the database
+    """
+
+    def __init__(self, expected_version: int, actual_version: int):
+        self.expected_version = expected_version
+        self.actual_version = actual_version
+        super().__init__(
+            f"Concurrency conflict: expected version {expected_version}, "
+            f"but stream is at version {actual_version}"
+        )
+
+
+def _retry_on_db_lock(func, max_retries=_MAX_DB_LOCK_RETRIES, base_delay=_DB_LOCK_RETRY_BASE_DELAY):
+    """Retry a callable up to max_retries times if it hits 'database is locked' error.
+
+    Used for defense-in-depth when multiple EventStore instances or CI shards
+    briefly contend on WAL locks. Implements exponential backoff.
+
+    Args:
+        func: A callable that may raise sqlite3.OperationalError('database is locked').
+        max_retries: Number of attempts (default 3).
+        base_delay: Base delay in seconds for exponential backoff (default 0.05s).
+
+    Returns:
+        The return value of func().
+
+    Raises:
+        sqlite3.OperationalError: If all retries are exhausted or non-lock error occurs.
+    """
+    for attempt in range(max_retries):
+        try:
+            return func()
+        except sqlite3.OperationalError as e:
+            if "database is locked" not in str(e):
+                raise
+            if attempt == max_retries - 1:
+                raise
+            time.sleep(base_delay * (2 ** attempt))  # exponential: 0.05s, 0.1s, 0.2s
+
 
 class EventStore:
     """Append-only event log stored at ``db_path``.
@@ -25,73 +79,128 @@ class EventStore:
     ``PRAGMA busy_timeout`` and assigns the per-stream version inside a
     ``BEGIN IMMEDIATE`` transaction, so the read-max-version-then-insert is
     atomic and two writers can never collide or duplicate a version.
+
+    Implements defense-in-depth retry logic for 'database is locked' errors that
+    can occur under heavy parallel contention (e.g., CI shards).
     """
 
     def __init__(self, db_path: str):
         self.db_path = db_path
-        conn = sqlite3.connect(self.db_path)
-        try:
-            conn.execute("PRAGMA journal_mode=WAL")
-            conn.execute("PRAGMA busy_timeout=5000")
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS events (
-                    id      INTEGER PRIMARY KEY AUTOINCREMENT,
-                    ts      REAL    NOT NULL,
-                    actor   TEXT    NOT NULL,
-                    stream  TEXT    NOT NULL,
-                    type    TEXT    NOT NULL,
-                    payload TEXT    NOT NULL,
-                    version INTEGER NOT NULL
+
+        def _init_db():
+            conn = sqlite3.connect(self.db_path)
+            try:
+                conn.execute("PRAGMA journal_mode=WAL")
+                conn.execute("PRAGMA busy_timeout=5000")
+                conn.execute(
+                    """
+                    CREATE TABLE IF NOT EXISTS events (
+                        id      INTEGER PRIMARY KEY AUTOINCREMENT,
+                        ts      REAL    NOT NULL,
+                        actor   TEXT    NOT NULL,
+                        stream  TEXT    NOT NULL,
+                        type    TEXT    NOT NULL,
+                        payload TEXT    NOT NULL,
+                        version INTEGER NOT NULL
+                    )
+                    """
                 )
-                """
-            )
-            conn.execute(
-                "CREATE INDEX IF NOT EXISTS idx_events_stream_version "
-                "ON events(stream, version)"
-            )
-            conn.execute(
-                """
-                CREATE TABLE IF NOT EXISTS snapshots (
-                    id           INTEGER PRIMARY KEY AUTOINCREMENT,
-                    ts           REAL    NOT NULL,
-                    stream       TEXT    NOT NULL,
-                    event_version INTEGER NOT NULL,
-                    projection   TEXT    NOT NULL,
-                    checksum     TEXT    NOT NULL
+                conn.execute(
+                    "CREATE INDEX IF NOT EXISTS idx_events_stream_version "
+                    "ON events(stream, version)"
                 )
-                """
-            )
-            conn.execute(
-                "CREATE INDEX IF NOT EXISTS idx_snapshots_stream_version "
-                "ON snapshots(stream, event_version)"
-            )
-            conn.commit()
-        finally:
-            conn.close()
+                conn.execute(
+                    """
+                    CREATE TABLE IF NOT EXISTS snapshots (
+                        id           INTEGER PRIMARY KEY AUTOINCREMENT,
+                        ts           REAL    NOT NULL,
+                        stream       TEXT    NOT NULL,
+                        event_version INTEGER NOT NULL,
+                        projection   TEXT    NOT NULL,
+                        checksum     TEXT    NOT NULL
+                    )
+                    """
+                )
+                conn.execute(
+                    "CREATE INDEX IF NOT EXISTS idx_snapshots_stream_version "
+                    "ON snapshots(stream, event_version)"
+                )
+                conn.commit()
+            finally:
+                conn.close()
 
-    def append(self, stream: str, event_type: str, payload: dict, actor: str = "system") -> int:
-        """Append one event to ``stream``; return its new per-stream version (1-based)."""
-        conn = sqlite3.connect(self.db_path)
-        try:
-            conn.execute("PRAGMA busy_timeout=5000")
-            # BEGIN IMMEDIATE takes the write lock up front so the
-            # read-max-then-insert below is atomic under contention.
-            conn.execute("BEGIN IMMEDIATE")
-            row = conn.execute(
-                "SELECT COALESCE(MAX(version), 0) FROM events WHERE stream = ?",
-                (stream,),
-            ).fetchone()
-            version = row[0] + 1
-            conn.execute(
-                "INSERT INTO events (ts, actor, stream, type, payload, version) "
-                "VALUES (?, ?, ?, ?, ?, ?)",
-                (time.time(), actor, stream, event_type, json.dumps(payload), version),
-            )
-            conn.commit()
-            return version
-        finally:
-            conn.close()
+        _retry_on_db_lock(_init_db)
+
+    def append(
+        self,
+        stream: str,
+        event_type: str,
+        payload: dict,
+        actor: str = "system",
+        expected_version: int | None = None,
+    ) -> int:
+        """Append one event to ``stream``; return its new per-stream version (1-based).
+
+        Supports optimistic concurrency control via the optional ``expected_version``
+        parameter. If provided, the append succeeds ONLY if the stream's current
+        maximum version equals ``expected_version``. If the versions do not match,
+        raises ConcurrencyConflict WITHOUT writing any event (fail-closed, atomic).
+
+        The version check and append are both performed under BEGIN IMMEDIATE so
+        they are atomic (no TOCTOU window).
+
+        Args:
+            stream: The stream name (e.g. "tracker", "claims")
+            event_type: The event type (e.g. "claim_requested", "claim_released")
+            payload: The event payload dict (will be JSON-serialized)
+            actor: The actor performing the append (default "system")
+            expected_version: Optional OCC check: if provided, append only if the
+                             stream's current max version equals this value.
+                             If None (default), OCC is disabled (backward-compatible).
+
+        Returns:
+            The new per-stream version assigned to this event (1-based).
+
+        Raises:
+            ConcurrencyConflict: If expected_version is provided and the stream's
+                               current max version does not match. The exception
+                               carries expected_version and actual_version for retry.
+                               No event is written when this exception is raised.
+        """
+
+        def _do_append():
+            conn = sqlite3.connect(self.db_path)
+            try:
+                conn.execute("PRAGMA busy_timeout=5000")
+                # BEGIN IMMEDIATE takes the write lock up front so the
+                # read-max-then-insert below is atomic under contention.
+                conn.execute("BEGIN IMMEDIATE")
+                row = conn.execute(
+                    "SELECT COALESCE(MAX(version), 0) FROM events WHERE stream = ?",
+                    (stream,),
+                ).fetchone()
+                current_version = row[0]
+
+                # Perform OCC check (if expected_version provided)
+                if expected_version is not None:
+                    if current_version != expected_version:
+                        # Mismatch: abort transaction and raise
+                        conn.rollback()
+                        raise ConcurrencyConflict(expected_version, current_version)
+
+                # OCC check passed or not enabled: proceed with append
+                version = current_version + 1
+                conn.execute(
+                    "INSERT INTO events (ts, actor, stream, type, payload, version) "
+                    "VALUES (?, ?, ?, ?, ?, ?)",
+                    (time.time(), actor, stream, event_type, json.dumps(payload), version),
+                )
+                conn.commit()
+                return version
+            finally:
+                conn.close()
+
+        return _retry_on_db_lock(_do_append)
 
     def read(self, stream: str) -> list:
         """Return all events for ``stream`` ascending by version (empty if none)."""
@@ -138,19 +247,22 @@ class EventStore:
             event_version: the per-stream event version this snapshot was computed through
             projection: the materialized state dict (e.g. the full tracker projection)
         """
-        conn = sqlite3.connect(self.db_path)
-        try:
-            conn.execute("PRAGMA busy_timeout=5000")
-            projection_json = json.dumps(projection, separators=(",", ":"), sort_keys=True)
-            checksum = hashlib.sha256(projection_json.encode()).hexdigest()
-            conn.execute(
-                "INSERT INTO snapshots (ts, stream, event_version, projection, checksum) "
-                "VALUES (?, ?, ?, ?, ?)",
-                (time.time(), stream, event_version, projection_json, checksum),
-            )
-            conn.commit()
-        finally:
-            conn.close()
+        def _do_save_snapshot():
+            conn = sqlite3.connect(self.db_path)
+            try:
+                conn.execute("PRAGMA busy_timeout=5000")
+                projection_json = json.dumps(projection, separators=(",", ":"), sort_keys=True)
+                checksum = hashlib.sha256(projection_json.encode()).hexdigest()
+                conn.execute(
+                    "INSERT INTO snapshots (ts, stream, event_version, projection, checksum) "
+                    "VALUES (?, ?, ?, ?, ?)",
+                    (time.time(), stream, event_version, projection_json, checksum),
+                )
+                conn.commit()
+            finally:
+                conn.close()
+
+        _retry_on_db_lock(_do_save_snapshot)
 
     def read_snapshot(self, stream: str) -> tuple | None:
         """Read the most recent snapshot for a stream.

package/templates/wave-presets/data.json

@@ -0,0 +1,65 @@
+{
+  "name": "Data Pipeline & Analytics",
+  "description": "Bootstrap a data project wave with ETL pipeline, data warehouse, and analytics components",
+  "categories": ["data-engineering", "analytics", "infrastructure"],
+  "items": [
+    {
+      "slug": "etl-pipeline",
+      "ownsFiles": [
+        "src/pipeline/extract.py",
+        "src/pipeline/transform.py",
+        "src/pipeline/load.py",
+        "src/pipeline/main.py",
+        "src/pipeline/config.yaml",
+        "tests/pipeline/",
+        "dags/",
+        "requirements.txt"
+      ],
+      "prompt": "Build the {project_name} ETL pipeline. Implement data extraction from CSV/APIs, transformation logic for cleaning and aggregation, and loading into a data warehouse. Use Apache Airflow or Python scheduler for orchestration. Include error handling, logging, and retry logic. Create sample DAGs that run daily. Tests must validate transform outputs.",
+      "testCmd": "python -m pytest tests/pipeline/ -v --cov=src/pipeline",
+      "workDir": "{base_dir}"
+    },
+    {
+      "slug": "data-warehouse",
+      "ownsFiles": [
+        "warehouse/schema/",
+        "warehouse/migrations/",
+        "warehouse/seeds/",
+        "warehouse/macros/",
+        "dbt_project.yml",
+        "warehouse/models/",
+        "tests/warehouse/"
+      ],
+      "prompt": "Set up the {project_name} data warehouse using dbt or SQL-based approach. Design fact and dimension tables for analytics. Create data models, transformations, and data quality tests. Build marts for reporting (daily sales, user cohorts, product performance). Use {base_dir}/warehouse as root. Ensure idempotent, incremental loading strategies.",
+      "testCmd": "dbt test --select +state:modified --state ./target",
+      "workDir": "{base_dir}"
+    },
+    {
+      "slug": "analytics-dashboard",
+      "ownsFiles": [
+        "src/dashboard/app.py",
+        "src/dashboard/pages/",
+        "src/dashboard/components/",
+        "src/dashboard/utils/",
+        "tests/dashboard/",
+        "requirements-dash.txt"
+      ],
+      "prompt": "Build the {project_name} analytics dashboard using Streamlit or Plotly Dash. Create interactive visualizations for: daily revenue trends, user acquisition cohorts, product performance leaderboard. Connect to the data warehouse for live queries. Add filters and date pickers. Deploy instructions for local and cloud. Responsive and accessible design.",
+      "testCmd": "python -m pytest tests/dashboard/ -v && streamlit run src/dashboard/app.py --logger.level=debug 2>&1 | head -50",
+      "workDir": "{base_dir}"
+    },
+    {
+      "slug": "data-quality",
+      "ownsFiles": [
+        "src/quality/checks.py",
+        "src/quality/validations.py",
+        "src/quality/reports.py",
+        "tests/quality/",
+        "config/quality.yaml"
+      ],
+      "prompt": "Implement data quality framework for {project_name}. Create validation rules for data freshness, null percentages, value ranges. Build automated checks that run after ETL. Generate quality reports (pass/fail/anomalies). Integrate with monitoring (alerting on failures). Include documentation on data lineage and SLAs.",
+      "testCmd": "python -m pytest tests/quality/ -v",
+      "workDir": "{base_dir}"
+    }
+  ]
+}