devcouncil 0.1.1 → 0.3.0

package/src/devcouncil/storage/db.py

@@ -1,6 +1,9 @@
+from collections import defaultdict
 from contextlib import contextmanager
+from datetime import datetime, timezone
 
 from sqlmodel import SQLModel, create_engine, Session
+from sqlalchemy import inspect, text
 from sqlalchemy.exc import OperationalError
 from pathlib import Path
 from typing import Optional
@@ -8,7 +11,41 @@ from typing import Optional
 from devcouncil.storage.models import SchemaVersionModel
 
 
-SCHEMA_VERSION = 1
+# v3: per-task indexes on the task/audit tables (task_id, lease status).
+# v4: machine-routable gap columns (file, line, suggested_command,
+#     acceptance_criterion_id) so the repair contract survives a reload.
+# v5: partial unique index enforcing one ACTIVE lease per task (single-writer).
+# v6: gap column expected_verification_method (repair loop tells an executor-remediable
+#     "incomplete" from a manual/llm one across a reload).
+SCHEMA_VERSION = 6
+
+
+# Module-level caches keyed by *resolved* paths so distinct project roots stay
+# independent (important for test isolation, where every test uses a fresh
+# tmp_path). `_db_instances` returns the same Database (and its single engine)
+# for a given project root instead of rebuilding the engine + re-running the
+# schema check on every get_db() call. `_dedup_done` records db paths whose
+# active-lease dedup migration has already run this process, so the one-time
+# table scan does not repeat on subsequent opens of the same database.
+_db_instances: dict[Path, "Database"] = {}
+_dedup_done: set[Path] = set()
+
+
+def reset_db_cache() -> None:
+    """Drop all cached Database instances and per-path guards.
+
+    Disposes pooled engine connections so the underlying SQLite files can be
+    safely removed/recreated. Intended for tests (or long-lived processes) that
+    rebuild a project's .devcouncil database under a path already opened this
+    process; without this a cached engine could point at a stale/deleted file.
+    """
+    for db in _db_instances.values():
+        try:
+            db.engine.dispose()
+        except Exception:
+            pass
+    _db_instances.clear()
+    _dedup_done.clear()
 
 
 class Database:
@@ -28,11 +65,17 @@ class Database:
                 session.add(SchemaVersionModel(id="singleton", version=SCHEMA_VERSION))
                 session.commit()
                 return
-            if current.version != SCHEMA_VERSION:
+            if current.version > SCHEMA_VERSION:
                 raise RuntimeError(
                     f"Unsupported DevCouncil schema version {current.version}; "
                     f"expected {SCHEMA_VERSION}."
                 )
+            if current.version < SCHEMA_VERSION:
+                self._create_tables()
+                current.version = SCHEMA_VERSION
+                session.add(current)
+                session.commit()
+                return
 
     def _create_tables(self):
         try:
@@ -40,6 +83,84 @@ class Database:
         except OperationalError as exc:
             if "already exists" not in str(exc):
                 raise
+        self._create_missing_columns()
+        self._dedup_active_leases()
+        self._create_missing_indexes()
+
+    def _dedup_active_leases(self):
+        # The partial unique index (ux_task_leases_active) can only be created if no task
+        # already has two ACTIVE leases. Older databases predate the constraint, so
+        # collapse any duplicates first — keep the newest active lease per task, mark the
+        # rest stale — making index creation succeed and restoring single-writer state.
+        #
+        # This only needs to run once per database per process: after the first pass the
+        # partial unique index (created immediately after, in _create_missing_indexes)
+        # prevents any new duplicate active leases, so re-scanning on every open is wasted
+        # work. Guard on the resolved path so distinct databases remain independent.
+        key = self.db_path.resolve()
+        if key in _dedup_done:
+            return
+        inspector = inspect(self.engine)
+        if "task_leases" not in inspector.get_table_names():
+            # Table not materialized yet; don't record as done so a later open retries.
+            return
+        with self.engine.begin() as conn:
+            rows = conn.execute(
+                text("SELECT id, task_id, created_at FROM task_leases WHERE status = 'active'")
+            ).fetchall()
+            by_task = defaultdict(list)
+            for row in rows:
+                by_task[row.task_id].append(((row.created_at or ""), row.id))
+            stale_ids = []
+            for items in by_task.values():
+                if len(items) <= 1:
+                    continue
+                items.sort()  # ascending by (created_at, id); keep the last = newest
+                stale_ids.extend(item[1] for item in items[:-1])
+            if stale_ids:
+                now = datetime.now(timezone.utc).isoformat()
+                for lease_id in stale_ids:
+                    conn.execute(
+                        text("UPDATE task_leases SET status = 'stale', released_at = :ts WHERE id = :id"),
+                        {"ts": now, "id": lease_id},
+                    )
+        _dedup_done.add(key)
+
+    def _create_missing_columns(self):
+        # create_all never alters an existing table, so columns added to a model
+        # later (e.g. the v4 gap routing columns) are missing on databases created by
+        # an older version — and a SELECT of the model would then fail. Add any
+        # missing column as a nullable ADD COLUMN (the only shape SQLite can add in
+        # place without a table rewrite); non-nullable additions are intentionally
+        # skipped because existing rows could not satisfy them.
+        inspector = inspect(self.engine)
+        existing_tables = set(inspector.get_table_names())
+        for table in SQLModel.metadata.sorted_tables:
+            if table.name not in existing_tables:
+                continue
+            existing_cols = {col["name"] for col in inspector.get_columns(table.name)}
+            for column in table.columns:
+                if column.name in existing_cols or not column.nullable:
+                    continue
+                ddl_type = column.type.compile(self.engine.dialect)
+                stmt = text(f'ALTER TABLE "{table.name}" ADD COLUMN "{column.name}" {ddl_type}')
+                try:
+                    with self.engine.begin() as conn:
+                        conn.execute(stmt)
+                except OperationalError as exc:
+                    if "duplicate column name" not in str(exc).lower():
+                        raise
+
+    def _create_missing_indexes(self):
+        # create_all skips tables that already exist, so indexes added to the
+        # model definitions later never materialize on existing databases.
+        for table in SQLModel.metadata.sorted_tables:
+            for index in table.indexes:
+                try:
+                    index.create(self.engine, checkfirst=True)
+                except OperationalError as exc:
+                    if "already exists" not in str(exc):
+                        raise
 
     @contextmanager
     def get_session(self):
@@ -58,9 +179,25 @@ class Database:
 def get_db(project_root: Path = Path(".")) -> Optional[Database]:
     dev_dir = project_root / ".devcouncil"
     if not dev_dir.exists():
+        # Checked before the cache so a not-yet-initialized (or removed) project
+        # never yields a stale cached Database.
         return None
-    
+
+    key = project_root.resolve()
     db_path = dev_dir / "state.sqlite"
+    cached = _db_instances.get(key)
+    # Only reuse the cached instance while its underlying file still exists. If the
+    # .devcouncil dir was wiped and recreated under the same path within one process,
+    # the cached engine points at a deleted file — dispose its pool and rebuild fresh.
+    if cached is not None:
+        if db_path.exists():
+            return cached
+        try:
+            cached.engine.dispose()
+        except Exception:
+            pass
+
     db = Database(db_path)
     db.ensure_schema_version()
+    _db_instances[key] = db
     return db

package/src/devcouncil/storage/models.py

@@ -1,4 +1,5 @@
 from sqlmodel import SQLModel, Field
+from sqlalchemy import Index, text
 from typing import Optional
 
 
@@ -61,6 +62,17 @@ class GapModel(SQLModel, table=True):
     evidence_json: str = Field(default="[]")
     recommended_fix: str
     blocking: bool
+    # Machine-routable repair hints. Persisted (v4) so the typed next-actions
+    # contract survives a reload — a reconnecting/crashed agent can read outstanding
+    # gaps without the heuristic reconstruction the loader used to fall back to.
+    file: Optional[str] = None
+    line: Optional[int] = None
+    suggested_command: Optional[str] = None
+    acceptance_criterion_id: Optional[str] = None
+    # Verification method the criterion expects (unit_test/manual/llm_review/...). Persisted
+    # so the repair loop can tell an executor-remediable "incomplete" (an automatable check
+    # that did not run) from one a human must close (manual/llm_review) after a reload.
+    expected_verification_method: Optional[str] = None
 
 class CritiqueFindingModel(SQLModel, table=True):
     __tablename__ = "critique_findings"
@@ -81,3 +93,116 @@ class ProjectStateModel(SQLModel, table=True):
     id: str = Field(primary_key=True, default="singleton")
     current_phase: str
     history_json: str = Field(default="[]")
+
+
+class TaskLeaseModel(SQLModel, table=True):
+    __tablename__ = "task_leases"
+    # Enforce the single-writer guarantee at the DB level: at most one ACTIVE lease per
+    # task. A partial unique index turns a concurrent double-checkout race into an
+    # IntegrityError (surfaced as lease_conflict) instead of two live leases.
+    __table_args__ = (
+        Index(
+            "ux_task_leases_active",
+            "task_id",
+            unique=True,
+            sqlite_where=text("status = 'active'"),
+        ),
+    )
+    id: str = Field(primary_key=True)
+    task_id: str = Field(index=True)
+    owner: str
+    agent: Optional[str] = None
+    client_id: Optional[str] = None
+    run_id: Optional[str] = None
+    branch: Optional[str] = None
+    lease_token: str
+    status: str = Field(default="active", index=True)
+    created_at: str
+    expires_at: Optional[str] = None
+    released_at: Optional[str] = None
+
+
+class ShellSessionModel(SQLModel, table=True):
+    __tablename__ = "shell_sessions"
+    id: str = Field(primary_key=True)
+    task_id: str = Field(index=True)
+    lease_id: Optional[str] = None
+    shell: str
+    cwd: str
+    status: str
+    started_at: str
+    ended_at: Optional[str] = None
+
+
+class ShellCommandEventModel(SQLModel, table=True):
+    __tablename__ = "shell_command_events"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    task_id: str = Field(index=True)
+    session_id: Optional[str] = None
+    command: str
+    status: str
+    exit_code: Optional[int] = None
+    reason: str = ""
+    stdout_path: str = ""
+    stderr_path: str = ""
+    created_at: str
+
+
+class FileChangeEventModel(SQLModel, table=True):
+    __tablename__ = "file_change_events"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    task_id: Optional[str] = Field(default=None, index=True)
+    lease_id: Optional[str] = None
+    session_id: Optional[str] = None
+    path: str
+    operation: str
+    allowed: bool
+    reason: str = ""
+    created_at: str
+
+
+class SemanticDiffModel(SQLModel, table=True):
+    __tablename__ = "semantic_diffs"
+    id: Optional[int] = Field(default=None, primary_key=True)
+    task_id: str = Field(index=True)
+    before_snapshot_path: str
+    after_snapshot_path: str
+    classifications_json: str = "[]"
+    summary: str = ""
+    created_at: str
+
+
+class AgentHandoffModel(SQLModel, table=True):
+    __tablename__ = "agent_handoffs"
+    id: str = Field(primary_key=True)
+    task_id: str = Field(index=True)
+    from_agent: str
+    to_agent: str
+    run_id: str
+    manifest_path: str
+    status: str
+    created_at: str
+
+
+class CorrectionManifestModel(SQLModel, table=True):
+    __tablename__ = "correction_manifests"
+    id: str = Field(primary_key=True)
+    task_id: str = Field(index=True)
+    run_id: Optional[str] = None
+    manifest_path: str
+    retry_budget: int = 3
+    attempt: int = 0
+    status: str
+    created_at: str
+
+
+class VerificationRunModel(SQLModel, table=True):
+    __tablename__ = "verification_runs"
+    id: str = Field(primary_key=True)
+    task_id: str = Field(index=True)
+    sandbox: str
+    environment_json: str = "{}"
+    commands_json: str = "[]"
+    status: str
+    started_at: str
+    finished_at: Optional[str] = None