arboris-cli 1.0.0 → 1.1.1

package/prisma/skills/continuous-learning-v2/agents/observer-loop.sh

@@ -0,0 +1,359 @@
+#!/usr/bin/env bash
+# Continuous Learning v2 - Observer background loop
+#
+# Fix for #521: Added re-entrancy guard, cooldown throttle, and
+# tail-based sampling to prevent memory explosion from runaway
+# parallel Claude analysis processes.
+
+set +e
+unset CLAUDECODE
+
+SLEEP_PID=""
+USR1_FIRED=0
+PENDING_ANALYSIS=0
+ANALYZING=0
+LAST_ANALYSIS_EPOCH=0
+# Minimum seconds between analyses (prevents rapid re-triggering)
+ANALYSIS_COOLDOWN="${ECC_OBSERVER_ANALYSIS_COOLDOWN:-60}"
+IDLE_TIMEOUT_SECONDS="${ECC_OBSERVER_IDLE_TIMEOUT_SECONDS:-1800}"
+SESSION_LEASE_DIR="${PROJECT_DIR}/.observer-sessions"
+ACTIVITY_FILE="${PROJECT_DIR}/.observer-last-activity"
+
+# Resolve this script's own directory so sibling scripts (session-guardian.sh)
+# and relative helpers (../scripts/instinct-cli.py) resolve correctly whether
+# this file is executed or sourced. $0 is the *caller* when sourced, so prefer
+# ${BASH_SOURCE[0]}, which always points at this file (#2370).
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+cleanup() {
+  [ -n "$SLEEP_PID" ] && kill "$SLEEP_PID" 2>/dev/null
+  if [ -f "$PID_FILE" ] && [ "$(cat "$PID_FILE" 2>/dev/null)" = "$$" ]; then
+    rm -f "$PID_FILE"
+  fi
+  exit 0
+}
+trap cleanup TERM INT
+
+file_mtime_epoch() {
+  local file="$1"
+  if [ ! -f "$file" ]; then
+    printf '0\n'
+    return
+  fi
+
+  if stat -c %Y "$file" >/dev/null 2>&1; then
+    stat -c %Y "$file" 2>/dev/null || printf '0\n'
+    return
+  fi
+
+  if stat -f %m "$file" >/dev/null 2>&1; then
+    stat -f %m "$file" 2>/dev/null || printf '0\n'
+    return
+  fi
+
+  printf '0\n'
+}
+
+has_active_session_leases() {
+  if [ ! -d "$SESSION_LEASE_DIR" ]; then
+    return 1
+  fi
+
+  find "$SESSION_LEASE_DIR" -type f -name '*.json' -print -quit 2>/dev/null | grep -q .
+}
+
+latest_activity_epoch() {
+  local observations_epoch activity_epoch
+  observations_epoch="$(file_mtime_epoch "$OBSERVATIONS_FILE")"
+  activity_epoch="$(file_mtime_epoch "$ACTIVITY_FILE")"
+
+  if [ "$activity_epoch" -gt "$observations_epoch" ] 2>/dev/null; then
+    printf '%s\n' "$activity_epoch"
+  else
+    printf '%s\n' "$observations_epoch"
+  fi
+}
+
+exit_if_idle_without_sessions() {
+  if has_active_session_leases; then
+    return
+  fi
+
+  local last_activity now_epoch idle_for
+  last_activity="$(latest_activity_epoch)"
+  now_epoch="$(date +%s)"
+  idle_for=$(( now_epoch - last_activity ))
+
+  if [ "$last_activity" -eq 0 ] || [ "$idle_for" -ge "$IDLE_TIMEOUT_SECONDS" ]; then
+    echo "[$(date)] Observer idle without active session leases for ${idle_for}s; exiting" >> "$LOG_FILE"
+    cleanup
+  fi
+}
+
+wait_for_claude_analysis() {
+  local child_pid="$1"
+  local wait_status=0
+
+  while true; do
+    wait "$child_pid"
+    wait_status=$?
+
+    if [ "$wait_status" -eq 0 ]; then
+      return 0
+    fi
+
+    # SIGUSR1 can interrupt wait while the Claude child is still running.
+    # Re-wait in that case so a signal is not logged as a false child failure.
+    if kill -0 "$child_pid" 2>/dev/null; then
+      continue
+    fi
+
+    return "$wait_status"
+  done
+}
+
+analyze_observations() {
+  if [ ! -f "$OBSERVATIONS_FILE" ]; then
+    return
+  fi
+
+  obs_count=$(wc -l < "$OBSERVATIONS_FILE" 2>/dev/null || echo 0)
+  if [ "$obs_count" -lt "$MIN_OBSERVATIONS" ]; then
+    return
+  fi
+
+  echo "[$(date)] Analyzing $obs_count observations for project ${PROJECT_NAME}..." >> "$LOG_FILE"
+
+  if [ "${CLV2_IS_WINDOWS:-false}" = "true" ] && [ "${ECC_OBSERVER_ALLOW_WINDOWS:-false}" != "true" ]; then
+    echo "[$(date)] Skipping claude analysis on Windows due to known non-interactive hang issue (#295). Set ECC_OBSERVER_ALLOW_WINDOWS=true to override." >> "$LOG_FILE"
+    return
+  fi
+
+  if ! command -v claude >/dev/null 2>&1; then
+    echo "[$(date)] claude CLI not found, skipping analysis" >> "$LOG_FILE"
+    return
+  fi
+
+  # session-guardian: gate observer cycle (active hours, cooldown, idle detection)
+  if ! bash "${SCRIPT_DIR}/session-guardian.sh"; then
+    echo "[$(date)] Observer cycle skipped by session-guardian" >> "$LOG_FILE"
+    return
+  fi
+
+  # Sample recent observations instead of loading the entire file (#521).
+  # This prevents multi-MB payloads from being passed to the LLM.
+  MAX_ANALYSIS_LINES="${ECC_OBSERVER_MAX_ANALYSIS_LINES:-500}"
+  observer_tmp_dir="${PROJECT_DIR}/.observer-tmp"
+  mkdir -p "$observer_tmp_dir"
+  analysis_file="$(mktemp "${observer_tmp_dir}/ecc-observer-analysis.XXXXXX.jsonl")"
+  tail -n "$MAX_ANALYSIS_LINES" "$OBSERVATIONS_FILE" > "$analysis_file"
+  analysis_count=$(wc -l < "$analysis_file" 2>/dev/null || echo 0)
+  echo "[$(date)] Using last $analysis_count of $obs_count observations for analysis" >> "$LOG_FILE"
+
+  # Use relative path from PROJECT_DIR for cross-platform compatibility (#842).
+  # On Windows (Git Bash/MSYS2), absolute paths from mktemp may use MSYS-style
+  # prefixes (e.g. /c/Users/...) that the Claude subprocess cannot resolve.
+  analysis_relpath=".observer-tmp/$(basename "$analysis_file")"
+
+  prompt_file="$(mktemp "${observer_tmp_dir}/ecc-observer-prompt.XXXXXX")"
+  cat > "$prompt_file" <<PROMPT
+IMPORTANT: You are running in non-interactive --print mode. You MUST use the Write tool directly to create files. Do NOT ask for permission, do NOT ask for confirmation, do NOT output summaries instead of writing. Just read, analyze, and write.
+
+Read ${analysis_relpath} and identify patterns for the project ${PROJECT_NAME} (user corrections, error resolutions, repeated workflows, tool preferences).
+If you find 3+ occurrences of the same pattern, you MUST write an instinct file directly to ${INSTINCTS_DIR}/<id>.md using the Write tool.
+Do NOT ask for permission to write files, do NOT describe what you would write, and do NOT stop at analysis when a qualifying pattern exists.
+
+CRITICAL: Every instinct file MUST use this exact format:
+
+---
+id: kebab-case-name
+trigger: when <specific condition>
+confidence: <0.3-0.85 based on frequency: 3-5 times=0.5, 6-10=0.7, 11+=0.85>
+domain: <one of: code-style, testing, git, debugging, workflow, file-patterns>
+source: session-observation
+scope: project
+project_id: ${PROJECT_ID}
+project_name: ${PROJECT_NAME}
+---
+
+# Title
+
+## Action
+<what to do, one clear sentence>
+
+## Evidence
+- Observed N times in session <id>
+- Pattern: <description>
+- Last observed: <date>
+
+Rules:
+- Be conservative, only clear patterns with 3+ observations
+- Use narrow, specific triggers
+- Never include actual code snippets, only describe patterns
+- When a qualifying pattern exists, write or update the instinct file in this run instead of asking for confirmation
+- If a similar instinct already exists in ${INSTINCTS_DIR}/, update it instead of creating a duplicate
+- The YAML frontmatter (between --- markers) with id field is MANDATORY
+- If a pattern seems universal (not project-specific), set scope to global instead of project
+- Examples of global patterns: always validate user input, prefer explicit error handling
+- Examples of project patterns: use React functional components, follow Django REST framework conventions
+PROMPT
+
+  # Read the prompt into memory before the Claude subprocess is spawned.
+  # On Windows/MSYS2, the mktemp path can differ from the shell's later path
+  # resolution, so relying on cat "$prompt_file" inside the claude invocation
+  # can fail even though the file was created successfully.
+  prompt_content="$(cat "$prompt_file" 2>/dev/null || true)"
+  rm -f "$prompt_file"
+  if [ -z "$prompt_content" ]; then
+    echo "[$(date)] Failed to load observer prompt content, skipping analysis" >> "$LOG_FILE"
+    rm -f "$analysis_file"
+    return
+  fi
+
+  timeout_seconds="${ECC_OBSERVER_TIMEOUT_SECONDS:-120}"
+  # Auto-scale max_turns proportional to analysis batch size when not explicitly set.
+  # The old hardcoded default of 20 is insufficient for the 500-line MAX_ANALYSIS_LINES
+  # default: Claude hits --max-turns before it can write all discovered instinct files.
+  # Formula: 1 turn per 10 analysis lines, floor 20, cap 100. (#2035)
+  if [ -n "${ECC_OBSERVER_MAX_TURNS:-}" ]; then
+    max_turns="${ECC_OBSERVER_MAX_TURNS}"
+  else
+    max_turns=$(( analysis_count / 10 ))
+    if [ "$max_turns" -lt 20 ]; then max_turns=20; fi
+    if [ "$max_turns" -gt 100 ]; then max_turns=100; fi
+  fi
+  exit_code=0
+
+  # Sanitize max_turns. The auto-scaled path above always yields a valid value >=20,
+  # but an explicit ECC_OBSERVER_MAX_TURNS override may be non-numeric, empty, or too
+  # small, so guard here and fall back to the safe default of 20.
+  case "$max_turns" in
+    ''|*[!0-9]*)
+      max_turns=20
+      ;;
+  esac
+
+  if [ "$max_turns" -lt 4 ]; then
+    max_turns=20
+  fi
+
+  # Ensure CWD is PROJECT_DIR so the relative analysis_relpath resolves correctly
+  # on all platforms, not just when the observer happens to be launched from the project root.
+  cd "$PROJECT_DIR" || { echo "[$(date)] Failed to cd to PROJECT_DIR ($PROJECT_DIR), skipping analysis" >> "$LOG_FILE"; rm -f "$analysis_file"; return; }
+
+  # Prevent observe.sh from recording this automated observer session as observations.
+  # Pass prompt via -p flag instead of stdin redirect for Windows compatibility (#842).
+  # prompt_content is already loaded in-memory so this no longer depends on the
+  # mktemp absolute path continuing to resolve after cwd changes (#1296).
+  # Model is configurable via ECC_OBSERVER_MODEL (defaults to haiku for cost efficiency);
+  # e.g. ECC_OBSERVER_MODEL=opus for higher-quality instinct extraction. Heavier models are
+  # slower — consider raising ECC_OBSERVER_TIMEOUT_SECONDS (default 120s) so the watchdog
+  # doesn't kill the analysis mid-run.
+  ECC_SKIP_OBSERVE=1 ECC_HOOK_PROFILE=minimal claude --model "${ECC_OBSERVER_MODEL:-haiku}" --max-turns "$max_turns" --print \
+    --allowedTools "Read,Write" \
+    -p "$prompt_content" >> "$LOG_FILE" 2>&1 &
+  claude_pid=$!
+
+  (
+    sleep "$timeout_seconds"
+    if kill -0 "$claude_pid" 2>/dev/null; then
+      echo "[$(date)] Claude analysis timed out after ${timeout_seconds}s; terminating process" >> "$LOG_FILE"
+      kill "$claude_pid" 2>/dev/null || true
+    fi
+  ) &
+  watchdog_pid=$!
+
+  wait_for_claude_analysis "$claude_pid"
+  exit_code=$?
+  kill "$watchdog_pid" 2>/dev/null || true
+  rm -f "$analysis_file"
+
+  if [ "$exit_code" -ne 0 ]; then
+    echo "[$(date)] Claude analysis failed (exit $exit_code); retaining observations for retry" >> "$LOG_FILE"
+    return
+  fi
+
+  # Archive observations only after a successful analysis. A transient
+  # failure (timeout, non-zero exit, rate limit) must not discard the batch
+  # before it has been turned into instincts, since the analyzer only ever
+  # reads the live observations file (#2370).
+  if [ -f "$OBSERVATIONS_FILE" ]; then
+    archive_dir="${PROJECT_DIR}/observations.archive"
+    mkdir -p "$archive_dir"
+    mv "$OBSERVATIONS_FILE" "$archive_dir/processed-$(date +%Y%m%d-%H%M%S)-$$.jsonl" 2>/dev/null || true
+  fi
+}
+
+on_usr1() {
+  [ -n "$SLEEP_PID" ] && kill "$SLEEP_PID" 2>/dev/null
+  SLEEP_PID=""
+
+  # Re-entrancy guard: defer the nudge so the main loop runs a follow-up
+  # analysis immediately after the current analysis finishes.
+  if [ "$ANALYZING" -eq 1 ]; then
+    PENDING_ANALYSIS=1
+    echo "[$(date)] Analysis already in progress, deferring signal" >> "$LOG_FILE"
+    return
+  fi
+
+  USR1_FIRED=1
+
+  # Cooldown: skip if last analysis was too recent (#521)
+  now_epoch=$(date +%s)
+  elapsed=$(( now_epoch - LAST_ANALYSIS_EPOCH ))
+  if [ "$elapsed" -lt "$ANALYSIS_COOLDOWN" ]; then
+    echo "[$(date)] Analysis cooldown active (${elapsed}s < ${ANALYSIS_COOLDOWN}s), skipping" >> "$LOG_FILE"
+    return
+  fi
+
+  ANALYZING=1
+  analyze_observations
+  LAST_ANALYSIS_EPOCH=$(date +%s)
+  ANALYZING=0
+}
+trap on_usr1 USR1
+
+# When this file is sourced (e.g. by tests/hooks/observer-loop-archive.test.js)
+# rather than executed, stop here so callers can invoke individual functions
+# such as analyze_observations without starting the observer loop. The only
+# production caller (start-observer.sh) executes the script, so $0 equals
+# BASH_SOURCE[0] there and this guard is a no-op (#2370).
+if [ "${BASH_SOURCE[0]}" != "${0}" ]; then
+  return 0 2>/dev/null || true
+fi
+
+echo "$$" > "$PID_FILE"
+echo "[$(date)] Observer started for ${PROJECT_NAME} (PID: $$)" >> "$LOG_FILE"
+
+# Prune expired pending instincts before analysis (SCRIPT_DIR resolved at top
+# via ${BASH_SOURCE[0]} so it is correct under both execution and sourcing).
+"${CLV2_PYTHON_CMD:-python3}" "${SCRIPT_DIR}/../scripts/instinct-cli.py" prune --quiet >> "$LOG_FILE" 2>&1 || echo "[$(date)] Warning: instinct prune failed (non-fatal)" >> "$LOG_FILE"
+
+while true; do
+  exit_if_idle_without_sessions
+
+  if [ "$PENDING_ANALYSIS" -eq 1 ]; then
+    PENDING_ANALYSIS=0
+    USR1_FIRED=0
+    ANALYZING=1
+    analyze_observations
+    LAST_ANALYSIS_EPOCH=$(date +%s)
+    ANALYZING=0
+    continue
+  fi
+
+  sleep "$OBSERVER_INTERVAL_SECONDS" &
+  SLEEP_PID=$!
+  wait "$SLEEP_PID" 2>/dev/null
+  SLEEP_PID=""
+
+  exit_if_idle_without_sessions
+  if [ "$USR1_FIRED" -eq 1 ]; then
+    USR1_FIRED=0
+  else
+    ANALYZING=1
+    analyze_observations
+    LAST_ANALYSIS_EPOCH=$(date +%s)
+    ANALYZING=0
+  fi
+done

package/prisma/skills/continuous-learning-v2/agents/observer.md

@@ -0,0 +1,189 @@
+---
+name: observer
+description: Background agent that analyzes session observations to detect patterns and create instincts. Uses Haiku for cost-efficiency. v2.1 adds project-scoped instincts.
+model: haiku
+---
+
+# Observer Agent
+
+A background agent that analyzes observations from Claude Code sessions to detect patterns and create instincts.
+
+## When to Run
+
+- After enough observations accumulate (configurable, default 20)
+- On a scheduled interval (configurable, default 5 minutes)
+- When triggered on demand via SIGUSR1 to the observer process
+
+## Input
+
+Reads observations from the **project-scoped** observations file:
+- Project: `${XDG_DATA_HOME:-~/.local/share}/ecc-homunculus/projects/<project-hash>/observations.jsonl`
+- Global fallback: `${XDG_DATA_HOME:-~/.local/share}/ecc-homunculus/observations.jsonl`
+
+```jsonl
+{"timestamp":"2025-01-22T10:30:00Z","event":"tool_start","session":"abc123","tool":"Edit","input":"...","project_id":"a1b2c3d4e5f6","project_name":"my-react-app"}
+{"timestamp":"2025-01-22T10:30:01Z","event":"tool_complete","session":"abc123","tool":"Edit","output":"...","project_id":"a1b2c3d4e5f6","project_name":"my-react-app"}
+{"timestamp":"2025-01-22T10:30:05Z","event":"tool_start","session":"abc123","tool":"Bash","input":"npm test","project_id":"a1b2c3d4e5f6","project_name":"my-react-app"}
+{"timestamp":"2025-01-22T10:30:10Z","event":"tool_complete","session":"abc123","tool":"Bash","output":"All tests pass","project_id":"a1b2c3d4e5f6","project_name":"my-react-app"}
+```
+
+## Pattern Detection
+
+Look for these patterns in observations:
+
+### 1. User Corrections
+When a user's follow-up message corrects Claude's previous action:
+- "No, use X instead of Y"
+- "Actually, I meant..."
+- Immediate undo/redo patterns
+
+→ Create instinct: "When doing X, prefer Y"
+
+### 2. Error Resolutions
+When an error is followed by a fix:
+- Tool output contains error
+- Next few tool calls fix it
+- Same error type resolved similarly multiple times
+
+→ Create instinct: "When encountering error X, try Y"
+
+### 3. Repeated Workflows
+When the same sequence of tools is used multiple times:
+- Same tool sequence with similar inputs
+- File patterns that change together
+- Time-clustered operations
+
+→ Create workflow instinct: "When doing X, follow steps Y, Z, W"
+
+### 4. Tool Preferences
+When certain tools are consistently preferred:
+- Always uses Grep before Edit
+- Prefers Read over Bash cat
+- Uses specific Bash commands for certain tasks
+
+→ Create instinct: "When needing X, use tool Y"
+
+## Output
+
+Creates/updates instincts in the **project-scoped** instincts directory:
+- Project: `${XDG_DATA_HOME:-~/.local/share}/ecc-homunculus/projects/<project-hash>/instincts/personal/`
+- Global: `${XDG_DATA_HOME:-~/.local/share}/ecc-homunculus/instincts/personal/` (for universal patterns)
+
+### Project-Scoped Instinct (default)
+
+```yaml
+---
+id: use-react-hooks-pattern
+trigger: "when creating React components"
+confidence: 0.65
+domain: "code-style"
+source: "session-observation"
+scope: project
+project_id: "a1b2c3d4e5f6"
+project_name: "my-react-app"
+---
+
+# Use React Hooks Pattern
+
+## Action
+Always use functional components with hooks instead of class components.
+
+## Evidence
+- Observed 8 times in session abc123
+- Pattern: All new components use useState/useEffect
+- Last observed: 2025-01-22
+```
+
+### Global Instinct (universal patterns)
+
+```yaml
+---
+id: always-validate-user-input
+trigger: "when handling user input"
+confidence: 0.75
+domain: "security"
+source: "session-observation"
+scope: global
+---
+
+# Always Validate User Input
+
+## Action
+Validate and sanitize all user input before processing.
+
+## Evidence
+- Observed across 3 different projects
+- Pattern: User consistently adds input validation
+- Last observed: 2025-01-22
+```
+
+## Scope Decision Guide
+
+When creating instincts, determine scope based on these heuristics:
+
+> **Scope Decision Guide** – See the canonical table in `skills/continuous-learning-v2/SKILL.md` (lines 271‑282).
+
+**When in doubt, default to `scope: project`** — it's safer to be project-specific and promote later than to contaminate the global space.
+
+## Confidence Calculation
+
+Initial confidence based on observation frequency:
+- 1-2 observations: 0.3 (tentative)
+- 3-5 observations: 0.5 (moderate)
+- 6-10 observations: 0.7 (strong)
+- 11+ observations: 0.85 (very strong)
+
+Confidence adjusts over time:
+- +0.05 for each confirming observation
+- -0.1 for each contradicting observation
+- -0.02 per week without observation (decay)
+
+## Instinct Promotion (Project → Global)
+
+An instinct should be promoted from project-scoped to global when:
+1. The **same pattern** (by id or similar trigger) exists in **2+ different projects**
+2. Each instance has confidence **>= 0.8**
+3. The domain is in the global-friendly list (security, general-best-practices, workflow)
+
+Promotion is handled by the `instinct-cli.py promote` command or the `/evolve` analysis.
+
+## Important Guidelines
+
+1. **Be Conservative**: Only create instincts for clear patterns (3+ observations)
+2. **Be Specific**: Narrow triggers are better than broad ones
+3. **Track Evidence**: Always include what observations led to the instinct
+4. **Respect Privacy**: Never include actual code snippets, only patterns
+5. **Merge Similar**: If a new instinct is similar to existing, update rather than duplicate
+6. **Default to Project Scope**: Unless the pattern is clearly universal, make it project-scoped
+7. **Include Project Context**: Always set `project_id` and `project_name` for project-scoped instincts
+
+## Example Analysis Session
+
+Given observations:
+```jsonl
+{"event":"tool_start","tool":"Grep","input":"pattern: useState","project_id":"a1b2c3","project_name":"my-app"}
+{"event":"tool_complete","tool":"Grep","output":"Found in 3 files","project_id":"a1b2c3","project_name":"my-app"}
+{"event":"tool_start","tool":"Read","input":"src/hooks/useAuth.ts","project_id":"a1b2c3","project_name":"my-app"}
+{"event":"tool_complete","tool":"Read","output":"[file content]","project_id":"a1b2c3","project_name":"my-app"}
+{"event":"tool_start","tool":"Edit","input":"src/hooks/useAuth.ts...","project_id":"a1b2c3","project_name":"my-app"}
+```
+
+Analysis:
+- Detected workflow: Grep → Read → Edit
+- Frequency: Seen 5 times this session
+- **Scope decision**: This is a general workflow pattern (not project-specific) → **global**
+- Create instinct:
+  - trigger: "when modifying code"
+  - action: "Search with Grep, confirm with Read, then Edit"
+  - confidence: 0.6
+  - domain: "workflow"
+  - scope: "global"
+
+## Integration with Skill Creator
+
+When instincts are imported from Skill Creator (repo analysis), they have:
+- `source: "repo-analysis"`
+- `source_repo: "https://github.com/..."`
+- `scope: "project"` (since they come from a specific repo)
+
+These should be treated as team/project conventions with higher initial confidence (0.7+).