aw-ecc 1.4.21

package/skills/continuous-learning/evaluate-session.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Continuous Learning - Session Evaluator
+# Runs on Stop hook to extract reusable patterns from Claude Code sessions
+#
+# Why Stop hook instead of UserPromptSubmit:
+# - Stop runs once at session end (lightweight)
+# - UserPromptSubmit runs every message (heavy, adds latency)
+#
+# Hook config (in ~/.claude/settings.json):
+# {
+#   "hooks": {
+#     "Stop": [{
+#       "matcher": "*",
+#       "hooks": [{
+#         "type": "command",
+#         "command": "~/.claude/skills/continuous-learning/evaluate-session.sh"
+#       }]
+#     }]
+#   }
+# }
+#
+# Patterns to detect: error_resolution, debugging_techniques, workarounds, project_specific
+# Patterns to ignore: simple_typos, one_time_fixes, external_api_issues
+# Extracted skills saved to: ~/.claude/skills/learned/
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CONFIG_FILE="$SCRIPT_DIR/config.json"
+LEARNED_SKILLS_PATH="${HOME}/.claude/skills/learned"
+MIN_SESSION_LENGTH=10
+
+# Load config if exists
+if [ -f "$CONFIG_FILE" ]; then
+  if ! command -v jq &>/dev/null; then
+    echo "[ContinuousLearning] jq is required to parse config.json but not installed, using defaults" >&2
+  else
+    MIN_SESSION_LENGTH=$(jq -r '.min_session_length // 10' "$CONFIG_FILE")
+    LEARNED_SKILLS_PATH=$(jq -r '.learned_skills_path // "~/.claude/skills/learned/"' "$CONFIG_FILE" | sed "s|~|$HOME|")
+  fi
+fi
+
+# Ensure learned skills directory exists
+mkdir -p "$LEARNED_SKILLS_PATH"
+
+# Get transcript path from stdin JSON (Claude Code hook input)
+# Falls back to env var for backwards compatibility
+stdin_data=$(cat)
+transcript_path=$(echo "$stdin_data" | grep -o '"transcript_path":"[^"]*"' | head -1 | cut -d'"' -f4)
+if [ -z "$transcript_path" ]; then
+  transcript_path="${CLAUDE_TRANSCRIPT_PATH:-}"
+fi
+
+if [ -z "$transcript_path" ] || [ ! -f "$transcript_path" ]; then
+  exit 0
+fi
+
+# Count messages in session
+message_count=$(grep -c '"type":"user"' "$transcript_path" 2>/dev/null || echo "0")
+
+# Skip short sessions
+if [ "$message_count" -lt "$MIN_SESSION_LENGTH" ]; then
+  echo "[ContinuousLearning] Session too short ($message_count messages), skipping" >&2
+  exit 0
+fi
+
+# Signal to Claude that session should be evaluated for extractable patterns
+echo "[ContinuousLearning] Session has $message_count messages - evaluate for extractable patterns" >&2
+echo "[ContinuousLearning] Save learned skills to: $LEARNED_SKILLS_PATH" >&2

package/skills/continuous-learning-v2/SKILL.md

@@ -0,0 +1,365 @@
+---
+name: continuous-learning-v2
+description: Instinct-based learning system that observes sessions via hooks, creates atomic instincts with confidence scoring, and evolves them into skills/commands/agents. v2.1 adds project-scoped instincts to prevent cross-project contamination.
+origin: ECC
+version: 2.1.0
+---
+
+# Continuous Learning v2.1 - Instinct
+-Based Architecture
+
+An advanced learning system that turns your Claude Code sessions into reusable knowledge through atomic "instincts" - small learned behaviors with confidence scoring.
+
+**v2.1** adds **project-scoped instincts** — React patterns stay in your React project, Python conventions stay in your Python project, and universal patterns (like "always validate input") are shared globally.
+
+## When to Activate
+
+- Setting up automatic learning from Claude Code sessions
+- Configuring instinct-based behavior extraction via hooks
+- Tuning confidence thresholds for learned behaviors
+- Reviewing, exporting, or importing instinct libraries
+- Evolving instincts into full skills, commands, or agents
+- Managing project-scoped vs global instincts
+- Promoting instincts from project to global scope
+
+## What's New in v2.1
+
+| Feature | v2.0 | v2.1 |
+|---------|------|------|
+| Storage | Global (~/.claude/homunculus/) | Project-scoped (projects/<hash>/) |
+| Scope | All instincts apply everywhere | Project-scoped + global |
+| Detection | None | git remote URL / repo path |
+| Promotion | N/A | Project → global when seen in 2+ projects |
+| Commands | 4 (status/evolve/export/import) | 6 (+promote/projects) |
+| Cross-project | Contamination risk | Isolated by default |
+
+## What's New in v2 (vs v1)
+
+| Feature | v1 | v2 |
+|---------|----|----|
+| Observation | Stop hook (session end) | PreToolUse/PostToolUse (100% reliable) |
+| Analysis | Main context | Background agent (Haiku) |
+| Granularity | Full skills | Atomic "instincts" |
+| Confidence | None | 0.3-0.9 weighted |
+| Evolution | Direct to skill | Instincts -> cluster -> skill/command/agent |
+| Sharing | None | Export/import instincts |
+
+## The Instinct Model
+
+An instinct is a small learned behavior:
+
+```yaml
+---
+id: prefer-functional-style
+trigger: "when writing new functions"
+confidence: 0.7
+domain: "code-style"
+source: "session-observation"
+scope: project
+project_id: "a1b2c3d4e5f6"
+project_name: "my-react-app"
+---
+
+# Prefer Functional Style
+
+## Action
+Use functional patterns over classes when appropriate.
+
+## Evidence
+- Observed 5 instances of functional pattern preference
+- User corrected class-based approach to functional on 2025-01-15
+```
+
+**Properties:**
+- **Atomic** -- one trigger, one action
+- **Confidence-weighted** -- 0.3 = tentative, 0.9 = near certain
+- **Domain-tagged** -- code-style, testing, git, debugging, workflow, etc.
+- **Evidence-backed** -- tracks what observations created it
+- **Scope-aware** -- `project` (default) or `global`
+
+## How It Works
+
+```
+Session Activity (in a git repo)
+      |
+      | Hooks capture prompts + tool use (100% reliable)
+      | + detect project context (git remote / repo path)
+      v
++---------------------------------------------+
+|  projects/<project-hash>/observations.jsonl  |
+|   (prompts, tool calls, outcomes, project)   |
++---------------------------------------------+
+      |
+      | Observer agent reads (background, Haiku)
+      v
++---------------------------------------------+
+|          PATTERN DETECTION                   |
+|   * User corrections -> instinct             |
+|   * Error resolutions -> instinct            |
+|   * Repeated workflows -> instinct           |
+|   * Scope decision: project or global?       |
++---------------------------------------------+
+      |
+      | Creates/updates
+      v
++---------------------------------------------+
+|  projects/<project-hash>/instincts/personal/ |
+|   * prefer-functional.yaml (0.7) [project]   |
+|   * use-react-hooks.yaml (0.9) [project]     |
++---------------------------------------------+
+|  instincts/personal/  (GLOBAL)               |
+|   * always-validate-input.yaml (0.85) [global]|
+|   * grep-before-edit.yaml (0.6) [global]     |
++---------------------------------------------+
+      |
+      | /evolve clusters + /promote
+      v
++---------------------------------------------+
+|  projects/<hash>/evolved/ (project-scoped)   |
+|  evolved/ (global)                           |
+|   * commands/new-feature.md                  |
+|   * skills/testing-workflow.md               |
+|   * agents/refactor-specialist.md            |
++---------------------------------------------+
+```
+
+## Project Detection
+
+The system automatically detects your current project:
+
+1. **`CLAUDE_PROJECT_DIR` env var** (highest priority)
+2. **`git remote get-url origin`** -- hashed to create a portable project ID (same repo on different machines gets the same ID)
+3. **`git rev-parse --show-toplevel`** -- fallback using repo path (machine-specific)
+4. **Global fallback** -- if no project is detected, instincts go to global scope
+
+Each project gets a 12-character hash ID (e.g., `a1b2c3d4e5f6`). A registry file at `~/.claude/homunculus/projects.json` maps IDs to human-readable names.
+
+## Quick Start
+
+### 1. Enable Observation Hooks
+
+Add to your `~/.claude/settings.json`.
+
+**If installed as a plugin** (recommended):
+
+```json
+{
+  "hooks": {
+    "PreToolUse": [{
+      "matcher": "*",
+      "hooks": [{
+        "type": "command",
+        "command": "${CLAUDE_PLUGIN_ROOT}/skills/continuous-learning-v2/hooks/observe.sh"
+      }]
+    }],
+    "PostToolUse": [{
+      "matcher": "*",
+      "hooks": [{
+        "type": "command",
+        "command": "${CLAUDE_PLUGIN_ROOT}/skills/continuous-learning-v2/hooks/observe.sh"
+      }]
+    }]
+  }
+}
+```
+
+**If installed manually** to `~/.claude/skills`:
+
+```json
+{
+  "hooks": {
+    "PreToolUse": [{
+      "matcher": "*",
+      "hooks": [{
+        "type": "command",
+        "command": "~/.claude/skills/continuous-learning-v2/hooks/observe.sh"
+      }]
+    }],
+    "PostToolUse": [{
+      "matcher": "*",
+      "hooks": [{
+        "type": "command",
+        "command": "~/.claude/skills/continuous-learning-v2/hooks/observe.sh"
+      }]
+    }]
+  }
+}
+```
+
+### 2. Initialize Directory Structure
+
+The system creates directories automatically on first use, but you can also create them manually:
+
+```bash
+# Global directories
+mkdir -p ~/.claude/homunculus/{instincts/{personal,inherited},evolved/{agents,skills,commands},projects}
+
+# Project directories are auto-created when the hook first runs in a git repo
+```
+
+### 3. Use the Instinct Commands
+
+```bash
+/instinct-status     # Show learned instincts (project + global)
+/evolve              # Cluster related instincts into skills/commands
+/instinct-export     # Export instincts to file
+/instinct-import     # Import instincts from others
+/promote             # Promote project instincts to global scope
+/projects            # List all known projects and their instinct counts
+```
+
+## Commands
+
+| Command | Description |
+|---------|-------------|
+| `/instinct-status` | Show all instincts (project-scoped + global) with confidence |
+| `/evolve` | Cluster related instincts into skills/commands, suggest promotions |
+| `/instinct-export` | Export instincts (filterable by scope/domain) |
+| `/instinct-import <file>` | Import instincts with scope control |
+| `/promote [id]` | Promote project instincts to global scope |
+| `/projects` | List all known projects and their instinct counts |
+
+## Configuration
+
+Edit `config.json` to control the background observer:
+
+```json
+{
+  "version": "2.1",
+  "observer": {
+    "enabled": false,
+    "run_interval_minutes": 5,
+    "min_observations_to_analyze": 20
+  }
+}
+```
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| `observer.enabled` | `false` | Enable the background observer agent |
+| `observer.run_interval_minutes` | `5` | How often the observer analyzes observations |
+| `observer.min_observations_to_analyze` | `20` | Minimum observations before analysis runs |
+
+Other behavior (observation capture, instinct thresholds, project scoping, promotion criteria) is configured via code defaults in `instinct-cli.py` and `observe.sh`.
+
+## File Structure
+
+```
+~/.claude/homunculus/
++-- identity.json           # Your profile, technical level
++-- projects.json           # Registry: project hash -> name/path/remote
++-- observations.jsonl      # Global observations (fallback)
++-- instincts/
+|   +-- personal/           # Global auto-learned instincts
+|   +-- inherited/          # Global imported instincts
++-- evolved/
+|   +-- agents/             # Global generated agents
+|   +-- skills/             # Global generated skills
+|   +-- commands/           # Global generated commands
++-- projects/
+    +-- a1b2c3d4e5f6/       # Project hash (from git remote URL)
+    |   +-- project.json    # Per-project metadata mirror (id/name/root/remote)
+    |   +-- observations.jsonl
+    |   +-- observations.archive/
+    |   +-- instincts/
+    |   |   +-- personal/   # Project-specific auto-learned
+    |   |   +-- inherited/  # Project-specific imported
+    |   +-- evolved/
+    |       +-- skills/
+    |       +-- commands/
+    |       +-- agents/
+    +-- f6e5d4c3b2a1/       # Another project
+        +-- ...
+```
+
+## Scope Decision Guide
+
+| Pattern Type | Scope | Examples |
+|-------------|-------|---------|
+| Language/framework conventions | **project** | "Use React hooks", "Follow Django REST patterns" |
+| File structure preferences | **project** | "Tests in `__tests__`/", "Components in src/components/" |
+| Code style | **project** | "Use functional style", "Prefer dataclasses" |
+| Error handling strategies | **project** | "Use Result type for errors" |
+| Security practices | **global** | "Validate user input", "Sanitize SQL" |
+| General best practices | **global** | "Write tests first", "Always handle errors" |
+| Tool workflow preferences | **global** | "Grep before Edit", "Read before Write" |
+| Git practices | **global** | "Conventional commits", "Small focused commits" |
+
+## Instinct Promotion (Project -> Global)
+
+When the same instinct appears in multiple projects with high confidence, it's a candidate for promotion to global scope.
+
+**Auto-promotion criteria:**
+- Same instinct ID in 2+ projects
+- Average confidence >= 0.8
+
+**How to promote:**
+
+```bash
+# Promote a specific instinct
+python3 instinct-cli.py promote prefer-explicit-errors
+
+# Auto-promote all qualifying instincts
+python3 instinct-cli.py promote
+
+# Preview without changes
+python3 instinct-cli.py promote --dry-run
+```
+
+The `/evolve` command also suggests promotion candidates.
+
+## Confidence Scoring
+
+Confidence evolves over time:
+
+| Score | Meaning | Behavior |
+|-------|---------|----------|
+| 0.3 | Tentative | Suggested but not enforced |
+| 0.5 | Moderate | Applied when relevant |
+| 0.7 | Strong | Auto-approved for application |
+| 0.9 | Near-certain | Core behavior |
+
+**Confidence increases** when:
+- Pattern is repeatedly observed
+- User doesn't correct the suggested behavior
+- Similar instincts from other sources agree
+
+**Confidence decreases** when:
+- User explicitly corrects the behavior
+- Pattern isn't observed for extended periods
+- Contradicting evidence appears
+
+## Why Hooks vs Skills for Observation?
+
+> "v1 relied on skills to observe. Skills are probabilistic -- they fire ~50-80% of the time based on Claude's judgment."
+
+Hooks fire **100% of the time**, deterministically. This means:
+- Every tool call is observed
+- No patterns are missed
+- Learning is comprehensive
+
+## Backward Compatibility
+
+v2.1 is fully compatible with v2.0 and v1:
+- Existing global instincts in `~/.claude/homunculus/instincts/` still work as global instincts
+- Existing `~/.claude/skills/learned/` skills from v1 still work
+- Stop hook still runs (but now also feeds into v2)
+- Gradual migration: run both in parallel
+
+## Privacy
+
+- Observations stay **local** on your machine
+- Project-scoped instincts are isolated per project
+- Only **instincts** (patterns) can be exported — not raw observations
+- No actual code or conversation content is shared
+- You control what gets exported and promoted
+
+## Related
+
+- [ECC-Tools GitHub App](https://github.com/apps/ecc-tools) - Generate instincts from repo history
+- Homunculus - Community project that inspired the v2 instinct-based architecture (atomic observations, confidence scoring, instinct evolution pipeline)
+- [The Longform Guide](https://x.com/affaanmustafa/status/2014040193557471352) - Continuous learning section
+
+---
+
+*Instinct-based learning: teaching Claude your patterns, one project at a time.*

package/skills/continuous-learning-v2/agents/observer-loop.sh

@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+# Continuous Learning v2 - Observer background loop
+#
+# Fix for #521: Added re-entrancy guard, cooldown throttle, and
+# tail-based sampling to prevent memory explosion from runaway
+# parallel Claude analysis processes.
+
+set +e
+unset CLAUDECODE
+
+SLEEP_PID=""
+USR1_FIRED=0
+ANALYZING=0
+LAST_ANALYSIS_EPOCH=0
+# Minimum seconds between analyses (prevents rapid re-triggering)
+ANALYSIS_COOLDOWN="${ECC_OBSERVER_ANALYSIS_COOLDOWN:-60}"
+
+cleanup() {
+  [ -n "$SLEEP_PID" ] && kill "$SLEEP_PID" 2>/dev/null
+  if [ -f "$PID_FILE" ] && [ "$(cat "$PID_FILE" 2>/dev/null)" = "$$" ]; then
+    rm -f "$PID_FILE"
+  fi
+  exit 0
+}
+trap cleanup TERM INT
+
+analyze_observations() {
+  if [ ! -f "$OBSERVATIONS_FILE" ]; then
+    return
+  fi
+
+  obs_count=$(wc -l < "$OBSERVATIONS_FILE" 2>/dev/null || echo 0)
+  if [ "$obs_count" -lt "$MIN_OBSERVATIONS" ]; then
+    return
+  fi
+
+  echo "[$(date)] Analyzing $obs_count observations for project ${PROJECT_NAME}..." >> "$LOG_FILE"
+
+  if [ "${CLV2_IS_WINDOWS:-false}" = "true" ] && [ "${ECC_OBSERVER_ALLOW_WINDOWS:-false}" != "true" ]; then
+    echo "[$(date)] Skipping claude analysis on Windows due to known non-interactive hang issue (#295). Set ECC_OBSERVER_ALLOW_WINDOWS=true to override." >> "$LOG_FILE"
+    return
+  fi
+
+  if ! command -v claude >/dev/null 2>&1; then
+    echo "[$(date)] claude CLI not found, skipping analysis" >> "$LOG_FILE"
+    return
+  fi
+
+  # session-guardian: gate observer cycle (active hours, cooldown, idle detection)
+  if ! bash "$(dirname "$0")/session-guardian.sh"; then
+    echo "[$(date)] Observer cycle skipped by session-guardian" >> "$LOG_FILE"
+    return
+  fi
+
+  # Sample recent observations instead of loading the entire file (#521).
+  # This prevents multi-MB payloads from being passed to the LLM.
+  MAX_ANALYSIS_LINES="${ECC_OBSERVER_MAX_ANALYSIS_LINES:-500}"
+  analysis_file="$(mktemp "${TMPDIR:-/tmp}/ecc-observer-analysis.XXXXXX.jsonl")"
+  tail -n "$MAX_ANALYSIS_LINES" "$OBSERVATIONS_FILE" > "$analysis_file"
+  analysis_count=$(wc -l < "$analysis_file" 2>/dev/null || echo 0)
+  echo "[$(date)] Using last $analysis_count of $obs_count observations for analysis" >> "$LOG_FILE"
+
+  prompt_file="$(mktemp "${TMPDIR:-/tmp}/ecc-observer-prompt.XXXXXX")"
+  cat > "$prompt_file" <<PROMPT
+Read ${analysis_file} and identify patterns for the project ${PROJECT_NAME} (user corrections, error resolutions, repeated workflows, tool preferences).
+If you find 3+ occurrences of the same pattern, create an instinct file in ${INSTINCTS_DIR}/<id>.md.
+
+CRITICAL: Every instinct file MUST use this exact format:
+
+---
+id: kebab-case-name
+trigger: when <specific condition>
+confidence: <0.3-0.85 based on frequency: 3-5 times=0.5, 6-10=0.7, 11+=0.85>
+domain: <one of: code-style, testing, git, debugging, workflow, file-patterns>
+source: session-observation
+scope: project
+project_id: ${PROJECT_ID}
+project_name: ${PROJECT_NAME}
+---
+
+# Title
+
+## Action
+<what to do, one clear sentence>
+
+## Evidence
+- Observed N times in session <id>
+- Pattern: <description>
+- Last observed: <date>
+
+Rules:
+- Be conservative, only clear patterns with 3+ observations
+- Use narrow, specific triggers
+- Never include actual code snippets, only describe patterns
+- If a similar instinct already exists in ${INSTINCTS_DIR}/, update it instead of creating a duplicate
+- The YAML frontmatter (between --- markers) with id field is MANDATORY
+- If a pattern seems universal (not project-specific), set scope to global instead of project
+- Examples of global patterns: always validate user input, prefer explicit error handling
+- Examples of project patterns: use React functional components, follow Django REST framework conventions
+PROMPT
+
+  timeout_seconds="${ECC_OBSERVER_TIMEOUT_SECONDS:-120}"
+  max_turns="${ECC_OBSERVER_MAX_TURNS:-10}"
+  exit_code=0
+
+  case "$max_turns" in
+    ''|*[!0-9]*)
+      max_turns=10
+      ;;
+  esac
+
+  if [ "$max_turns" -lt 4 ]; then
+    max_turns=10
+  fi
+
+  # Prevent observe.sh from recording this automated Haiku session as observations
+  ECC_SKIP_OBSERVE=1 ECC_HOOK_PROFILE=minimal claude --model haiku --max-turns "$max_turns" --print \
+    --allowedTools "Read,Write" \
+    < "$prompt_file" >> "$LOG_FILE" 2>&1 &
+  claude_pid=$!
+
+  (
+    sleep "$timeout_seconds"
+    if kill -0 "$claude_pid" 2>/dev/null; then
+      echo "[$(date)] Claude analysis timed out after ${timeout_seconds}s; terminating process" >> "$LOG_FILE"
+      kill "$claude_pid" 2>/dev/null || true
+    fi
+  ) &
+  watchdog_pid=$!
+
+  wait "$claude_pid"
+  exit_code=$?
+  kill "$watchdog_pid" 2>/dev/null || true
+  rm -f "$prompt_file" "$analysis_file"
+
+  if [ "$exit_code" -ne 0 ]; then
+    echo "[$(date)] Claude analysis failed (exit $exit_code)" >> "$LOG_FILE"
+  fi
+
+  if [ -f "$OBSERVATIONS_FILE" ]; then
+    archive_dir="${PROJECT_DIR}/observations.archive"
+    mkdir -p "$archive_dir"
+    mv "$OBSERVATIONS_FILE" "$archive_dir/processed-$(date +%Y%m%d-%H%M%S)-$$.jsonl" 2>/dev/null || true
+  fi
+}
+
+on_usr1() {
+  [ -n "$SLEEP_PID" ] && kill "$SLEEP_PID" 2>/dev/null
+  SLEEP_PID=""
+  USR1_FIRED=1
+
+  # Re-entrancy guard: skip if analysis is already running (#521)
+  if [ "$ANALYZING" -eq 1 ]; then
+    echo "[$(date)] Analysis already in progress, skipping signal" >> "$LOG_FILE"
+    return
+  fi
+
+  # Cooldown: skip if last analysis was too recent (#521)
+  now_epoch=$(date +%s)
+  elapsed=$(( now_epoch - LAST_ANALYSIS_EPOCH ))
+  if [ "$elapsed" -lt "$ANALYSIS_COOLDOWN" ]; then
+    echo "[$(date)] Analysis cooldown active (${elapsed}s < ${ANALYSIS_COOLDOWN}s), skipping" >> "$LOG_FILE"
+    return
+  fi
+
+  ANALYZING=1
+  analyze_observations
+  LAST_ANALYSIS_EPOCH=$(date +%s)
+  ANALYZING=0
+}
+trap on_usr1 USR1
+
+echo "$$" > "$PID_FILE"
+echo "[$(date)] Observer started for ${PROJECT_NAME} (PID: $$)" >> "$LOG_FILE"
+
+# Prune expired pending instincts before analysis
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+"${CLV2_PYTHON_CMD:-python3}" "${SCRIPT_DIR}/../scripts/instinct-cli.py" prune --quiet >> "$LOG_FILE" 2>&1 || echo "[$(date)] Warning: instinct prune failed (non-fatal)" >> "$LOG_FILE"
+
+while true; do
+  sleep "$OBSERVER_INTERVAL_SECONDS" &
+  SLEEP_PID=$!
+  wait "$SLEEP_PID" 2>/dev/null
+  SLEEP_PID=""
+
+  if [ "$USR1_FIRED" -eq 1 ]; then
+    USR1_FIRED=0
+  else
+    analyze_observations
+  fi
+done