arkaos 4.21.0 → 4.23.0

package/core/cognition/evolve.py

@@ -0,0 +1,299 @@
+"""Evolve — turn accumulated cross-project signals into instinct proposals.
+
+Propose-only, LLM-free. The engine ingests the raw signals sessions
+already accumulate (``~/.arkaos/gotchas.json`` — error patterns with
+occurrence counts and the projects they appeared in) into the
+``InsightStore`` as deterministic instincts: confidence derives from the
+occurrence count inside the [0.3, 0.9] band, evidence_count mirrors the
+raw count, and the record id is a stable digest of (source, title,
+project) so re-ingestion is idempotent. It then renders a markdown
+proposal at ``~/.arkaos/evolve-proposals/<date>.md`` with the
+cross-project promotion candidates (``InsightStore.promotable`` — this
+module is that API's first production caller) and the strongest
+instincts.
+
+Boundaries: Dreaming owns vault clustering (LLM, multi-backend) — evolve
+never re-clusters markdown; it consumes structured stores only. Nothing
+is promoted, modified, or deleted: the proposal file is the only output
+(the reorganizer contract). Client identifiers are redacted and
+project names are never rendered — only distinct-project counts.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import tempfile
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+
+from core.cognition.insights.store import InsightStore
+from core.cognition.memory.schemas import (
+    INSTINCT_CONFIDENCE_MAX,
+    INSTINCT_CONFIDENCE_MIN,
+    ActionableInsight,
+)
+from core.cognition.reorganizer import md_escape, redact_clients
+
+_DEFAULT_OUTPUT_DIR = Path.home() / ".arkaos" / "evolve-proposals"
+_DEFAULT_GOTCHAS_PATH = Path.home() / ".arkaos" / "gotchas.json"
+_DEFAULT_DB_PATH = Path.home() / ".arkaos" / "insights.db"
+_PROPOSAL_FILENAME_FMT = "%Y-%m-%d.md"
+_CONFIDENCE_STEP = 0.1
+_TOP_INSTINCTS = 10
+
+
+@dataclass(frozen=True)
+class PromotionCandidate:
+    """Cross-project instinct eligible for promotion (propose-only)."""
+
+    title: str
+    project_count: int
+    mean_confidence: float
+    evidence_count: int
+
+
+@dataclass(frozen=True)
+class EvolveReport:
+    """Evolve run outcome. Never contains client or project identifiers."""
+
+    date: str
+    ingested: int
+    pending_instincts: int
+    candidates: list[PromotionCandidate] = field(default_factory=list)
+    proposal_path: str | None = None
+
+
+def derive_confidence(count: int) -> float:
+    """Occurrence count -> confidence, deterministic, inside the band."""
+    raw = INSTINCT_CONFIDENCE_MIN + _CONFIDENCE_STEP * (max(1, count) - 1)
+    return min(INSTINCT_CONFIDENCE_MAX, raw)
+
+
+def _stable_id(source: str, title: str, project: str) -> str:
+    digest = hashlib.sha1(f"{source}:{title}:{project}".encode())
+    return digest.hexdigest()[:32]
+
+
+def _load_gotchas(gotchas_path: Path) -> list[dict]:
+    try:
+        data = json.loads(gotchas_path.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return []
+    entries = data.get("gotchas") if isinstance(data, dict) else data
+    return [e for e in entries or [] if isinstance(e, dict)]
+
+
+def _instinct_from_gotcha(gotcha: dict, project: str) -> ActionableInsight | None:
+    title = str(gotcha.get("pattern") or "").strip()
+    if not title or not project.strip():
+        return None
+    count = int(gotcha.get("count") or 1)
+    return ActionableInsight(
+        id=_stable_id("gotcha", title, project),
+        project=project,
+        trigger="evolve-ingest",
+        category="technical",
+        severity="improve" if count >= 5 else "consider",
+        title=title,
+        description=str(gotcha.get("full_pattern") or title),
+        recommendation=str(gotcha.get("suggestion") or "").strip()
+        or "Recurring signal — consider a rule, skill, or fix.",
+        context=f"tool={gotcha.get('tool', '?')} count={count}",
+        confidence=derive_confidence(count),
+        evidence_count=max(1, count),
+    )
+
+
+def ingest_gotchas(store: InsightStore, gotchas_path: Path) -> int:
+    """Mirror gotcha signals into the store as project-scoped instincts.
+
+    Deterministic derivation from the source of truth: re-ingesting
+    recomputes the same ids and values (INSERT OR REPLACE), so repeat
+    runs are idempotent.
+    """
+    ingested = 0
+    for gotcha in _load_gotchas(gotchas_path):
+        projects = gotcha.get("projects") or []
+        for project in {str(p) for p in projects if str(p).strip()}:
+            insight = _instinct_from_gotcha(gotcha, project)
+            if insight is None:
+                continue
+            store.save(insight)
+            ingested += 1
+    return ingested
+
+
+def _candidates(
+    store: InsightStore, min_projects: int, min_confidence: float,
+) -> list[PromotionCandidate]:
+    titles = store.promotable(
+        min_projects=min_projects, min_confidence=min_confidence
+    )
+    rows = [i for i in store.get_all_pending() if i.scope == "project"]
+    out: list[PromotionCandidate] = []
+    for title in titles:
+        group = [i for i in rows if i.title == title]
+        if not group:
+            continue
+        out.append(
+            PromotionCandidate(
+                title=title,
+                project_count=len({i.project for i in group}),
+                mean_confidence=round(
+                    sum(i.confidence for i in group) / len(group), 2
+                ),
+                evidence_count=sum(i.evidence_count for i in group),
+            )
+        )
+    return out
+
+
+def build_proposal(
+    *,
+    db_path: Path | None = None,
+    gotchas_path: Path | None = None,
+    output_dir: Path | None = None,
+    dry_run: bool = False,
+    min_projects: int = 2,
+    min_confidence: float = 0.8,
+) -> EvolveReport:
+    """Ingest signals, compute promotion candidates, write the proposal."""
+    store = InsightStore(db_path or _DEFAULT_DB_PATH)
+    ingested = ingest_gotchas(store, gotchas_path or _DEFAULT_GOTCHAS_PATH)
+    pending = store.get_all_pending()
+    candidates = _candidates(store, min_projects, min_confidence)
+    date = datetime.now(UTC).strftime("%Y-%m-%d")
+    markdown = _render(date, ingested, pending, candidates)
+    if dry_run:
+        print(markdown)
+        return EvolveReport(date, ingested, len(pending), candidates)
+    path = _write_report(markdown, output_dir)
+    return EvolveReport(date, ingested, len(pending), candidates, str(path))
+
+
+def _render_candidate(candidate: PromotionCandidate) -> str:
+    title = md_escape(redact_clients(candidate.title))
+    return (
+        f"| {title} | {candidate.project_count} | "
+        f"{candidate.mean_confidence:.2f} | {candidate.evidence_count} |"
+    )
+
+
+def _render_instinct(insight: ActionableInsight) -> str:
+    title = md_escape(redact_clients(insight.title))
+    rec = md_escape(redact_clients(insight.recommendation))
+    return (
+        f"- **{title}** — confidence {insight.confidence:.2f}, "
+        f"evidence {insight.evidence_count}. {rec}"
+    )
+
+
+def _render_summary(
+    date: str, ingested: int, pending_count: int, candidate_count: int,
+) -> list[str]:
+    return [
+        f"# Evolve proposal — {date}",
+        "",
+        "> Propose-only: nothing was promoted or modified. Review and",
+        "> act via the squads; promotion stays a deliberate operator call.",
+        "",
+        "## Summary",
+        "",
+        f"- Instincts ingested this run: **{ingested}**",
+        f"- Pending instincts in store: **{pending_count}**",
+        f"- Promotion candidates: **{candidate_count}**",
+        "",
+    ]
+
+
+def _render_candidates_block(candidates: list[PromotionCandidate]) -> list[str]:
+    lines = ["## Promotion candidates (cross-project)", ""]
+    if candidates:
+        lines += [
+            "| Instinct | Projects | Mean confidence | Evidence |",
+            "|---|---|---|---|",
+            *[_render_candidate(c) for c in candidates],
+        ]
+    else:
+        lines.append("None yet — needs the same signal in 2+ projects.")
+    return lines
+
+
+def _strongest_by_title(pending: list[ActionableInsight]) -> list[ActionableInsight]:
+    """Top instincts, one row per title (a cross-project pattern has one
+    project-scoped row per project — the operator needs it once)."""
+    ranked = sorted(
+        pending, key=lambda i: (i.confidence, i.evidence_count), reverse=True
+    )
+    seen: set[str] = set()
+    out: list[ActionableInsight] = []
+    for insight in ranked:
+        if insight.title in seen:
+            continue
+        seen.add(insight.title)
+        out.append(insight)
+        if len(out) == _TOP_INSTINCTS:
+            break
+    return out
+
+
+def _render_instincts_block(pending: list[ActionableInsight]) -> list[str]:
+    strongest = _strongest_by_title(pending)
+    lines = ["", "## Strongest instincts", ""]
+    if strongest:
+        lines += [_render_instinct(i) for i in strongest]
+    else:
+        lines.append("Store is empty — no signals accumulated yet.")
+    return lines
+
+
+def _render(
+    date: str,
+    ingested: int,
+    pending: list[ActionableInsight],
+    candidates: list[PromotionCandidate],
+) -> str:
+    lines = (
+        _render_summary(date, ingested, len(pending), len(candidates))
+        + _render_candidates_block(candidates)
+        + _render_instincts_block(pending)
+    )
+    return "\n".join(lines) + "\n"
+
+
+def _write_report(markdown: str, output_dir: Path | None) -> Path:
+    """Atomic markdown write to an allowlisted output directory."""
+    out = _validate_output_dir(output_dir)
+    out.mkdir(parents=True, exist_ok=True)
+    report_path = out / datetime.now(UTC).strftime(
+        _PROPOSAL_FILENAME_FMT
+    )
+    tmp_path = report_path.with_suffix(f".tmp-{os.getpid()}.md")
+    tmp_path.write_text(markdown, encoding="utf-8")
+    os.replace(tmp_path, report_path)
+    return report_path
+
+
+def _validate_output_dir(output_dir: Path | None) -> Path:
+    """Allowlist writes to ~/.arkaos or the system tempdir (defence in
+    depth — same contract as the reorganizer)."""
+    if output_dir is None:
+        return _DEFAULT_OUTPUT_DIR
+    resolved = Path(output_dir).expanduser().resolve()
+    allowed_roots = (
+        (Path.home() / ".arkaos").resolve(),
+        Path(tempfile.gettempdir()).resolve(),
+    )
+    for root in allowed_roots:
+        try:
+            resolved.relative_to(root)
+            return resolved
+        except ValueError:
+            continue
+    raise ValueError(
+        "output_dir must be under one of "
+        f"{[str(r) for r in allowed_roots]}; got {resolved}"
+    )

package/core/cognition/evolve_cli.py

@@ -0,0 +1,55 @@
+"""CLI for the evolve engine: ``python -m core.cognition.evolve_cli``.
+
+Mirrors ``reorganizer_cli``: argparse in, one ``build_proposal`` call,
+human-readable summary out. ``--dry-run`` prints the proposal to stdout
+without writing anything.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from core.cognition.evolve import build_proposal
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="python -m core.cognition.evolve_cli",
+        description=(
+            "Ingest accumulated signals into instincts and write a "
+            "propose-only evolution proposal."
+        ),
+    )
+    parser.add_argument(
+        "--min-projects", type=int, default=2,
+        help="distinct projects required for a promotion candidate",
+    )
+    parser.add_argument(
+        "--min-confidence", type=float, default=0.8,
+        help="mean confidence required for a promotion candidate",
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true",
+        help="print the proposal to stdout; write nothing",
+    )
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = _build_parser().parse_args(argv)
+    report = build_proposal(
+        dry_run=args.dry_run,
+        min_projects=args.min_projects,
+        min_confidence=args.min_confidence,
+    )
+    if not args.dry_run:
+        print(
+            f"evolve: ingested={report.ingested} "
+            f"pending={report.pending_instincts} "
+            f"candidates={len(report.candidates)} -> {report.proposal_path}"
+        )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/core/cognition/insights/store.py

@@ -4,7 +4,9 @@ Persists ActionableInsight instances with support for status-based retrieval,
 project filtering, presentation lifecycle, and dismissal analytics.
 """
 
+import contextlib
 import sqlite3
+from collections.abc import Iterator
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -24,11 +26,20 @@ class InsightStore:
         self._db_path = db_path
         self._init_db()
 
-    def _conn(self) -> sqlite3.Connection:
+    @contextlib.contextmanager
+    def _conn(self) -> Iterator[sqlite3.Connection]:
+        """Per-operation connection: transaction semantics AND a real
+        close. ``with sqlite3.connect(...)`` alone only commits/rolls
+        back — it never closes, leaking one connection per operation
+        (ResourceWarning under coverage, amplified by ingest loops)."""
         conn = sqlite3.connect(self._db_path)
-        conn.row_factory = sqlite3.Row
-        conn.execute("PRAGMA journal_mode=WAL")
-        return conn
+        try:
+            conn.row_factory = sqlite3.Row
+            conn.execute("PRAGMA journal_mode=WAL")
+            with conn:
+                yield conn
+        finally:
+            conn.close()
 
     # Instinct columns added after the original 12-column schema. An
     # operator's existing DB has the old shape, so these are ALTER-added

package/core/forge/persistence.py

@@ -4,8 +4,9 @@ import os
 import re as _re
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Optional
+
 import yaml
+
 from core.forge.schema import ForgePlan, ForgeStatus
 
 
@@ -23,7 +24,9 @@ def save_plan(plan: ForgePlan) -> Path:
     plans.mkdir(parents=True, exist_ok=True)
     target = plans / f"{plan.id}.yaml"
     data = plan.model_dump(mode="json")
-    fd = NamedTemporaryFile(mode="w", dir=str(plans), suffix=".tmp", delete=False, encoding="utf-8")
+    fd = NamedTemporaryFile(  # noqa: SIM115 — handle outlives the block: os.replace consumes fd.name
+        mode="w", dir=str(plans), suffix=".tmp", delete=False, encoding="utf-8"
+    )
     try:
         yaml.dump(data, fd, default_flow_style=False, allow_unicode=True)
         fd.close()
@@ -35,7 +38,7 @@ def save_plan(plan: ForgePlan) -> Path:
     return target
 
 
-def load_plan(plan_id: str) -> Optional[ForgePlan]:
+def load_plan(plan_id: str) -> ForgePlan | None:
     """Load a forge plan by ID. Returns None if not found."""
     path = _plans_dir() / f"{plan_id}.yaml"
     if not path.exists():
@@ -54,12 +57,20 @@ def list_plans() -> list[dict]:
         if path.name == "active.yaml":
             continue
         data = yaml.safe_load(path.read_text(encoding="utf-8"))
+        if not isinstance(data, dict):
+            continue
+        # Pre-v4 plans stored complexity/critic as scalars — a summary
+        # must tolerate every shape that ever hit disk.
+        complexity = data.get("complexity")
+        critic = data.get("critic")
         results.append({
             "id": data.get("id", path.stem),
             "name": data.get("name", ""),
             "status": data.get("status", "draft"),
-            "tier": data.get("complexity", {}).get("tier", "shallow"),
-            "confidence": data.get("critic", {}).get("confidence", 0.0),
+            "tier": complexity.get("tier", "shallow")
+            if isinstance(complexity, dict) else "shallow",
+            "confidence": critic.get("confidence", 0.0)
+            if isinstance(critic, dict) else 0.0,
             "created_at": data.get("created_at", ""),
         })
     return results
@@ -72,7 +83,7 @@ def set_active_plan(plan_id: str) -> None:
     link.write_text(plan_id, encoding="utf-8")
 
 
-def get_active_plan() -> Optional[ForgePlan]:
+def get_active_plan() -> ForgePlan | None:
     """Get the currently active forge plan."""
     link = _active_link()
     if not link.exists():
@@ -94,7 +105,10 @@ def clear_active_plan() -> None:
 
 def _obsidian_forge_dir() -> Path:
     """Obsidian vault path for Forge documents."""
-    return Path.home() / "Documents" / "Personal" / "Projects" / "WizardingCode Internal" / "ArkaOS" / "Forge"
+    return (
+        Path.home() / "Documents" / "Personal" / "Projects"
+        / "WizardingCode Internal" / "ArkaOS" / "Forge"
+    )
 
 
 def export_to_obsidian(plan: ForgePlan) -> Path:
@@ -150,7 +164,8 @@ def _render_obsidian_context(plan: ForgePlan) -> list[str]:
     ctx = plan.context
     return [
         "## Context",
-        f"Repo: {ctx.repo} | Branch: {ctx.branch} | Commit: {ctx.commit_at_forge} | ArkaOS: {ctx.arkaos_version}",
+        f"Repo: {ctx.repo} | Branch: {ctx.branch} | "
+        f"Commit: {ctx.commit_at_forge} | ArkaOS: {ctx.arkaos_version}",
         "",
         "## Prompt",
         f"> {ctx.prompt}",
@@ -195,7 +210,10 @@ def _render_obsidian_critic(plan: ForgePlan) -> list[str]:
     if critic.risks:
         lines.append("### Risks")
         for risk in critic.risks:
-            lines.append(f"- **{risk.risk}** ({risk.severity.value}) — Mitigation: {risk.mitigation}")
+            lines.append(
+                f"- **{risk.risk}** ({risk.severity.value}) — "
+                f"Mitigation: {risk.mitigation}"
+            )
         lines.append("")
     return lines
 

package/core/forge/schema.py

@@ -6,23 +6,21 @@ structured ForgePlan that downstream agents consume.
 """
 
 from enum import Enum
-from typing import List, Optional
 
 from pydantic import BaseModel, Field, field_validator
 
-
 # ---------------------------------------------------------------------------
 # Enums
 # ---------------------------------------------------------------------------
 
-class ForgeTier(str, Enum):
+class ForgeTier(str, Enum):  # noqa: UP042 — StrEnum flips str()/format(); migration is its own change
     """Execution tier determined by complexity score."""
     SHALLOW = "shallow"
     STANDARD = "standard"
     DEEP = "deep"
 
 
-class ForgeStatus(str, Enum):
+class ForgeStatus(str, Enum):  # noqa: UP042 — StrEnum flips str()/format(); migration is its own change
     """Lifecycle status of a ForgePlan."""
     DRAFT = "draft"
     REVIEWING = "reviewing"
@@ -34,21 +32,21 @@ class ForgeStatus(str, Enum):
     ARCHIVED = "archived"
 
 
-class ExplorerLens(str, Enum):
+class ExplorerLens(str, Enum):  # noqa: UP042 — StrEnum flips str()/format(); migration is its own change
     """The analytical perspective used when exploring a plan."""
     PRAGMATIC = "pragmatic"          # Focus on fastest viable path
     ARCHITECTURAL = "architectural"  # Focus on long-term design health
     CONTRARIAN = "contrarian"        # Challenge assumptions, surface risks
 
 
-class RiskSeverity(str, Enum):
+class RiskSeverity(str, Enum):  # noqa: UP042 — StrEnum flips str()/format(); migration is its own change
     """Severity level for identified risks."""
     LOW = "low"
     MEDIUM = "medium"
     HIGH = "high"
 
 
-class ExecutionPathType(str, Enum):
+class ExecutionPathType(str, Enum):  # noqa: UP042 — StrEnum flips str()/format(); migration is its own change
     """Type of execution artefact that fulfils a plan step."""
     SKILL = "skill"
     WORKFLOW = "workflow"
@@ -60,12 +58,22 @@ class ExecutionPathType(str, Enum):
 # ---------------------------------------------------------------------------
 
 class ComplexityDimensions(BaseModel):
-    """Five-axis complexity breakdown, each scored 0–100."""
+    """Five-axis complexity breakdown, each scored 0-100."""
 
-    scope: int = Field(default=0, description="Breadth of change across the codebase or system.")
-    dependencies: int = Field(default=0, description="Number and criticality of upstream/downstream dependencies.")
-    ambiguity: int = Field(default=0, description="How unclear or under-specified the requirements are.")
-    risk: int = Field(default=0, description="Potential for breakage, data loss, or security impact.")
+    scope: int = Field(
+        default=0, description="Breadth of change across the codebase or system."
+    )
+    dependencies: int = Field(
+        default=0,
+        description="Number and criticality of upstream/downstream dependencies.",
+    )
+    ambiguity: int = Field(
+        default=0, description="How unclear or under-specified the requirements are."
+    )
+    risk: int = Field(
+        default=0,
+        description="Potential for breakage, data loss, or security impact.",
+    )
     novelty: int = Field(default=0, description="How unlike existing patterns this work is.")
 
     @field_validator("scope", "dependencies", "ambiguity", "risk", "novelty", mode="before")
@@ -78,14 +86,19 @@ class ComplexityDimensions(BaseModel):
 class ComplexityScore(BaseModel):
     """Aggregated complexity result produced by the Complexity Scorer."""
 
-    score: int = Field(default=0, description="Composite 0–100 score derived from all dimensions.")
-    tier: ForgeTier = Field(default=ForgeTier.SHALLOW, description="Execution tier selected based on the composite score.")
-    dimensions: ComplexityDimensions = Field(default_factory=ComplexityDimensions, description="Per-dimension breakdown.")
-    similar_plans: List[str] = Field(
+    score: int = Field(default=0, description="Composite 0-100 score derived from all dimensions.")
+    tier: ForgeTier = Field(
+        default=ForgeTier.SHALLOW,
+        description="Execution tier selected based on the composite score.",
+    )
+    dimensions: ComplexityDimensions = Field(
+        default_factory=ComplexityDimensions, description="Per-dimension breakdown."
+    )
+    similar_plans: list[str] = Field(
         default_factory=list,
         description="IDs of previously completed plans with similar profiles.",
     )
-    reused_patterns: List[str] = Field(
+    reused_patterns: list[str] = Field(
         default_factory=list,
         description="Named patterns from the ArkaOS pattern library reused in this plan.",
     )
@@ -213,13 +226,23 @@ class ForgePlan(BaseModel):
             "contains only constitution-enforced phases and confidence is 0.0."
         ),
     )
-    dispatch_errors: List[str] = Field(
+    dispatch_errors: list[str] = Field(
         default_factory=list,
         description="Dispatch failures collected during forge (lens: error class: message).",
     )
 
     status: ForgeStatus = ForgeStatus.DRAFT
-    approved_at: Optional[str] = None
-    approved_by: Optional[str] = None
-    executed_at: Optional[str] = None
-    completion_notes: Optional[str] = None
+    approved_at: str | None = None
+    approved_by: str | None = None
+    rejected_at: str | None = None
+    rejected_by: str | None = None
+    executed_at: str | None = None
+    completion_notes: str | None = None
+    review_note: str | None = Field(
+        default=None,
+        description=(
+            "Operator note attached at decision time (plan-canvas review) "
+            "— the 'ponto-e-anota' trail; distinct from completion_notes, "
+            "which belongs to execution."
+        ),
+    )