feed-the-machine 1.6.1 → 1.7.1

package/hooks/ftm-pre-compaction.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# ftm-pre-compaction.sh
+# Hook: Pre-compaction memory flush for ftm
+# Trigger: UserPromptSubmit
+#
+# Fires on every user message during an active ftm session.
+# Reads the session JSONL to track token usage. When context approaches
+# the limit (default: 150K of 200K tokens), injects a flush-first prefix
+# telling Claude to write daily log + task state before responding.
+#
+# This approximates OpenClaw's pre-compaction flush, adapted for Claude Code's
+# hook system (which can inject text but can't inject silent LLM turns).
+#
+# THRESHOLD: 150K tokens = 75% of 200K context window
+# COOLDOWN:  Only re-fires after 15K more tokens consumed (prevents spamming)
+#
+# FILES:
+#   ~/.claude/ftm-state/blackboard/context.json  - session gate
+#   ~/.claude/ftm-state/.last-flush-tokens        - token count at last flush (cooldown)
+
+FTM_STATE="$HOME/.claude/ftm-state"
+CONTEXT_JSON="$FTM_STATE/blackboard/context.json"
+LAST_FLUSH_FILE="$FTM_STATE/.last-flush-tokens"
+
+THRESHOLD=150000   # Flush when total context exceeds this
+COOLDOWN=15000     # Only re-flush after this many additional tokens
+
+# 1. Only run during active ftm sessions
+FTM_ACTIVE=$(python3 -c "
+import json, sys
+try:
+    with open('$CONTEXT_JSON') as f:
+        d = json.load(f)
+    task = d.get('current_task', {})
+    status = task.get('status', '')
+    print('1' if status not in ('', 'completed', 'none') else '0')
+except Exception:
+    print('0')
+" 2>/dev/null)
+
+if [ "$FTM_ACTIVE" != "1" ]; then
+    exit 0
+fi
+
+# 2. Read hook payload from stdin (JSON: { session_id, prompt, transcript_path, ... })
+PAYLOAD=$(cat)
+if [ -z "$PAYLOAD" ]; then
+    exit 0
+fi
+
+# 3. Extract session_id and transcript_path from payload
+SESSION_ID=$(echo "$PAYLOAD" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    print(d.get('session_id', ''))
+except:
+    print('')
+" 2>/dev/null)
+
+TRANSCRIPT_PATH=$(echo "$PAYLOAD" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    print(d.get('transcript_path', ''))
+except:
+    print('')
+" 2>/dev/null)
+
+# 4. Find the session JSONL (try transcript_path first, then search by session_id)
+JSONL_FILE=""
+if [ -n "$TRANSCRIPT_PATH" ] && [ -f "$TRANSCRIPT_PATH" ]; then
+    JSONL_FILE="$TRANSCRIPT_PATH"
+elif [ -n "$SESSION_ID" ]; then
+    JSONL_FILE=$(find "$HOME/.claude/projects/" -name "${SESSION_ID}.jsonl" 2>/dev/null | head -1)
+fi
+
+if [ -z "$JSONL_FILE" ] || [ ! -f "$JSONL_FILE" ]; then
+    exit 0
+fi
+
+# 5. Parse JSONL to find latest total token usage
+TOTAL_TOKENS=$(python3 -c "
+import json, sys, os
+
+path = sys.argv[1]
+try:
+    with open(path) as f:
+        lines = f.readlines()
+except:
+    print(0)
+    sys.exit(0)
+
+# Scan in reverse for the latest usage entry
+for line in reversed(lines):
+    try:
+        obj = json.loads(line)
+        msg = obj.get('message', {})
+        if not isinstance(msg, dict):
+            continue
+        usage = msg.get('usage', {})
+        if isinstance(usage, dict) and 'input_tokens' in usage:
+            total = (
+                usage.get('input_tokens', 0) +
+                usage.get('cache_read_input_tokens', 0) +
+                usage.get('cache_creation_input_tokens', 0)
+            )
+            print(total)
+            sys.stdout.flush()
+            os._exit(0)
+    except Exception:
+        pass
+
+print(0)
+" "$JSONL_FILE" 2>/dev/null)
+
+TOTAL_TOKENS=${TOTAL_TOKENS:-0}
+
+# 6. Check if we're over the threshold
+if [ "$TOTAL_TOKENS" -lt "$THRESHOLD" ] 2>/dev/null; then
+    exit 0
+fi
+
+# 7. Cooldown check: only flush if we've consumed COOLDOWN more tokens since last flush
+LAST_FLUSH_TOKENS=$(cat "$LAST_FLUSH_FILE" 2>/dev/null || echo "0")
+TOKENS_SINCE_FLUSH=$(( TOTAL_TOKENS - LAST_FLUSH_TOKENS ))
+
+if [ "$TOKENS_SINCE_FLUSH" -lt "$COOLDOWN" ] 2>/dev/null; then
+    exit 0
+fi
+
+# 8. Record this flush point
+echo "$TOTAL_TOKENS" > "$LAST_FLUSH_FILE"
+
+# 9. Inject the flush-first prefix into the user's message
+TODAY=$(date +%Y-%m-%d)
+PCT=$((TOTAL_TOKENS * 100 / 200000))
+
+echo ""
+echo "[PRE-COMPACTION MEMORY FLUSH — ${TOTAL_TOKENS} tokens (~${PCT}% of 200K) — ${TODAY}]:"
+echo "Context is getting full. BEFORE responding: write any unsaved state to disk."
+echo "1. Append unsaved session progress to the ftm daily log for ${TODAY}"
+echo "2. Update ~/.claude/ftm-state/blackboard/context.json with current task status"
+echo "3. If important facts/solutions discovered this session, append to relevant knowledge files"
+echo "Do this silently — do NOT announce the flush to the user. Then respond normally."
+echo "If there is genuinely nothing new to record, skip the writes and proceed directly."
+echo ""

package/hooks/ftm-session-end.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# ftm-session-end.sh
+# Hook: Deactivate ftm session tracking when conversation ends
+# Trigger: SessionEnd
+#
+# IMPORTANT: Must be listed AFTER ftm-session-snapshot.sh in settings.json
+# (in a separate matcher entry). The snapshot hook reads context.json status
+# to gate itself — this hook marks the session completed.
+
+FTM_STATE="$HOME/.claude/ftm-state"
+CONTEXT_JSON="$FTM_STATE/blackboard/context.json"
+
+# Check if an active session exists
+IS_ACTIVE=$(python3 -c "
+import json, sys
+try:
+    with open('$CONTEXT_JSON') as f:
+        d = json.load(f)
+    task = d.get('current_task', {})
+    status = task.get('status', '')
+    print('1' if status not in ('', 'completed', 'none') else '0')
+except Exception:
+    print('0')
+" 2>/dev/null)
+
+if [ "$IS_ACTIVE" != "1" ]; then
+    exit 0
+fi
+
+# Mark session as completed in context.json
+python3 -c "
+import json, sys
+from datetime import datetime
+
+try:
+    with open('$CONTEXT_JSON') as f:
+        d = json.load(f)
+
+    if 'current_task' in d and isinstance(d['current_task'], dict):
+        d['current_task']['status'] = 'completed'
+
+    if 'session_metadata' in d and isinstance(d['session_metadata'], dict):
+        d['session_metadata']['last_updated'] = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
+
+    with open('$CONTEXT_JSON', 'w') as f:
+        json.dump(d, f, indent=2)
+except Exception as e:
+    sys.stderr.write(f'ftm-session-end: failed to update context.json: {e}\n')
+    sys.exit(1)
+" 2>/dev/null
+
+echo "ftm session tracking deactivated (session ended)"

package/hooks/ftm-session-snapshot.sh

@@ -0,0 +1,213 @@
+#!/bin/bash
+# ftm-session-snapshot.sh
+# Hook: Session snapshot on conversation end
+# Trigger: SessionEnd
+#
+# Captures the last 15 meaningful user/assistant exchanges as a markdown snapshot
+# when an ftm session ends. Fills the gap where sessions that never hit the
+# compaction token threshold would lose all conversational context.
+#
+# Inspired by OpenClaw's session snapshot mechanism, which fires on /new or /reset.
+# We fire on SessionEnd since Claude Code doesn't expose /new or /reset as hook events.
+#
+# Filtering:
+#   - Keeps: user and assistant messages with text content
+#   - Skips: tool_use blocks, tool_result blocks, system messages, slash commands
+#   - Truncates: messages > 2000 chars (long pastes don't need full capture)
+#
+# Filename: YYYY-MM-DDTHH-MM-<topic-slug>.md  (topic derived from last user message)
+#
+# IMPORTANT: Must be listed BEFORE ftm-session-end.sh in settings.json.
+# ftm-session-end.sh updates context.json status — this hook reads it.
+#
+# OUTPUT: ~/.claude/ftm-state/sessions/
+# MINIMUM: 3 meaningful messages required to write a snapshot (skip trivial sessions)
+#
+# FILES:
+#   ~/.claude/ftm-state/blackboard/context.json  - session gate (read only, not modified)
+#   ~/.claude/ftm-state/sessions/                - snapshot output directory
+
+FTM_STATE="$HOME/.claude/ftm-state"
+CONTEXT_JSON="$FTM_STATE/blackboard/context.json"
+SESSIONS_DIR="$FTM_STATE/sessions"
+MAX_MESSAGES=15
+MIN_MESSAGES=3
+
+# 1. Only run during active ftm sessions
+FTM_ACTIVE=$(python3 -c "
+import json, sys
+try:
+    with open('$CONTEXT_JSON') as f:
+        d = json.load(f)
+    task = d.get('current_task', {})
+    status = task.get('status', '')
+    print('1' if status not in ('', 'completed', 'none') else '0')
+except Exception:
+    print('0')
+" 2>/dev/null)
+
+if [ "$FTM_ACTIVE" != "1" ]; then
+    exit 0
+fi
+
+# 2. Read hook payload from stdin
+PAYLOAD=$(cat)
+if [ -z "$PAYLOAD" ]; then
+    exit 0
+fi
+
+# 3. Extract session_id and transcript_path
+SESSION_ID=$(echo "$PAYLOAD" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    print(d.get('session_id', ''))
+except:
+    print('')
+" 2>/dev/null)
+
+TRANSCRIPT_PATH=$(echo "$PAYLOAD" | python3 -c "
+import sys, json
+try:
+    d = json.load(sys.stdin)
+    print(d.get('transcript_path', ''))
+except:
+    print('')
+" 2>/dev/null)
+
+# 4. Find session JSONL
+JSONL_FILE=""
+if [ -n "$TRANSCRIPT_PATH" ] && [ -f "$TRANSCRIPT_PATH" ]; then
+    JSONL_FILE="$TRANSCRIPT_PATH"
+elif [ -n "$SESSION_ID" ]; then
+    JSONL_FILE=$(find "$HOME/.claude/projects/" -name "${SESSION_ID}.jsonl" 2>/dev/null | head -1)
+fi
+
+if [ -z "$JSONL_FILE" ] || [ ! -f "$JSONL_FILE" ]; then
+    exit 0
+fi
+
+# 5. Ensure sessions directory exists
+mkdir -p "$SESSIONS_DIR"
+
+# 6. Parse JSONL, build snapshot, write file — all in one Python block
+export JSONL_FILE SESSION_ID SESSIONS_DIR MAX_MESSAGES MIN_MESSAGES
+python3 << 'PYEOF'
+import json, sys, os, re
+from datetime import datetime
+
+jsonl_path = os.environ['JSONL_FILE']
+session_id = os.environ.get('SESSION_ID', 'unknown')
+sessions_dir = os.environ['SESSIONS_DIR']
+max_msgs = int(os.environ.get('MAX_MESSAGES', '15'))
+min_msgs = int(os.environ.get('MIN_MESSAGES', '3'))
+
+# --- Parse JSONL ---
+try:
+    with open(jsonl_path) as f:
+        lines = f.readlines()
+except Exception:
+    sys.exit(0)
+
+messages = []
+for line in lines:
+    line = line.strip()
+    if not line:
+        continue
+    try:
+        obj = json.loads(line)
+    except Exception:
+        continue
+
+    msg = obj.get('message', {})
+    if not isinstance(msg, dict):
+        continue
+
+    role = msg.get('role', '')
+    if role not in ('user', 'assistant'):
+        continue
+
+    content = msg.get('content', '')
+
+    # Extract text content only — skip tool_use, tool_result, image blocks
+    if isinstance(content, list):
+        text_parts = []
+        for block in content:
+            if isinstance(block, dict):
+                if block.get('type') == 'text':
+                    t = block.get('text', '').strip()
+                    if t:
+                        text_parts.append(t)
+            elif isinstance(block, str):
+                t = block.strip()
+                if t:
+                    text_parts.append(t)
+        text = '\n'.join(text_parts)
+    elif isinstance(content, str):
+        text = content.strip()
+    else:
+        continue
+
+    if not text:
+        continue
+
+    # Skip slash commands
+    if role == 'user' and text.lstrip().startswith('/'):
+        continue
+
+    messages.append({'role': role, 'text': text})
+
+# Take last max_msgs meaningful messages
+recent = messages[-max_msgs:]
+
+# Require minimum message count — skip trivial sessions
+if len(recent) < min_msgs:
+    sys.exit(0)
+
+# --- Derive topic slug from last user message with some substance ---
+topic = 'session'
+for m in reversed(recent):
+    if m['role'] == 'user' and len(m['text']) > 15:
+        raw = m['text'][:80].lower()
+        raw = re.sub(r'[^a-z0-9\s]', ' ', raw)
+        raw = re.sub(r'\s+', '-', raw.strip())
+        raw = re.sub(r'-+', '-', raw).strip('-')[:50].rstrip('-')
+        if raw:
+            topic = raw
+        break
+
+# --- Build filename and output path ---
+now = datetime.now()
+timestamp_file = now.strftime('%Y-%m-%dT%H-%M')
+timestamp_display = now.strftime('%Y-%m-%d %H:%M')
+filename = f'{timestamp_file}-{topic}.md'
+output_path = os.path.join(sessions_dir, filename)
+
+# --- Write snapshot markdown ---
+lines_out = [
+    f'# Session Snapshot — {timestamp_display}',
+    '',
+    f'**Messages captured**: {len(recent)} (last {len(recent)} of session)',
+    f'**Session**: {session_id[:8]}...',
+    '',
+    '---',
+    '',
+]
+
+for msg in recent:
+    role_label = '**You**' if msg['role'] == 'user' else '**Claude**'
+    text = msg['text']
+    # Truncate very long messages to keep snapshot lean
+    if len(text) > 2000:
+        text = text[:2000] + '\n\n[... truncated — full content in daily log ...]'
+    lines_out.append(f'{role_label}: {text}')
+    lines_out.append('')
+
+try:
+    with open(output_path, 'w') as f:
+        f.write('\n'.join(lines_out))
+except Exception:
+    pass
+PYEOF
+
+exit 0

package/hooks/ftm-task-loader.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+# ftm-task-loader.sh
+# PostToolUse hook that fires after the Skill tool is invoked.
+# When ftm-ops or ftm is the invoked skill, loads tasks from brain.py
+# and injects TaskCreate instructions as additionalContext.
+#
+# This is deterministic — brain.py runs in the hook (fast), and the
+# model receives pre-parsed TaskCreate calls it must execute.
+#
+# Hook: PostToolUse (matcher: Skill)
+
+set -euo pipefail
+
+INPUT=$(cat)
+
+# Extract the tool name and check it's the Skill tool
+TOOL_NAME=$(echo "$INPUT" | jq -r '.tool_name // ""')
+if [[ "$TOOL_NAME" != "Skill" ]]; then
+  exit 0
+fi
+
+# Extract which skill was invoked from the tool input
+SKILL_NAME=$(echo "$INPUT" | jq -r '.tool_input.skill // ""' 2>/dev/null)
+
+# Only fire for ftm-ops, ftm, or ftm-mind (which routes to ftm-ops for task requests)
+case "$SKILL_NAME" in
+  ftm-ops|ftm|ftm-mind|eng-buddy) ;;
+  *) exit 0 ;;
+esac
+
+# Find brain.py — check multiple locations
+BRAIN_PY=""
+for candidate in \
+  "$HOME/.claude/skills/ftm/bin/brain.py" \
+  "$HOME/.claude/skills/ftm-ops/../bin/brain.py" \
+  "$HOME/Documents/Code/feed-the-machine/bin/brain.py"; do
+  if [[ -f "$candidate" ]]; then
+    BRAIN_PY="$candidate"
+    break
+  fi
+done
+
+if [[ -z "$BRAIN_PY" ]]; then
+  # brain.py not found — skip silently
+  exit 0
+fi
+
+# Run brain.py and capture task JSON
+TASKS_JSON=$(python3 "$BRAIN_PY" --tasks --task-json 2>/dev/null) || exit 0
+
+# Count active tasks
+TASK_COUNT=$(echo "$TASKS_JSON" | python3 -c "
+import sys, json
+try:
+    tasks = json.load(sys.stdin)
+    active = [t for t in tasks if t.get('status') in ('pending', 'in_progress')]
+    print(len(active))
+except:
+    print(0)
+" 2>/dev/null)
+
+if [[ "$TASK_COUNT" == "0" ]]; then
+  exit 0
+fi
+
+# Generate TaskCreate instructions from the task data
+TASK_INSTRUCTIONS=$(echo "$TASKS_JSON" | python3 -c "
+import sys, json
+try:
+    tasks = json.load(sys.stdin)
+    active = [t for t in tasks if t.get('status') in ('pending', 'in_progress')]
+    lines = []
+    for t in active:
+        tid = t.get('id', t.get('legacy_number', '?'))
+        title = t.get('title', 'Untitled')
+        status = t.get('status', 'pending')
+        priority = t.get('priority', 'medium')
+        jira = t.get('jira_key', '')
+        desc_parts = [status, priority]
+        if jira:
+            desc_parts.append(jira)
+        desc = ' | '.join(desc_parts)
+        lines.append(f'TaskCreate(subject: \"#{tid} {title}\", description: \"{desc}\")')
+    print('\n'.join(lines))
+except:
+    pass
+" 2>/dev/null)
+
+if [[ -z "$TASK_INSTRUCTIONS" ]]; then
+  exit 0
+fi
+
+# Output as additionalContext — the model MUST process this
+cat <<EOF
+[ftm-task-loader] Loaded $TASK_COUNT active tasks from tasks.db. You MUST now call TaskCreate for each one so they appear in Claude Code's sidebar task list. Do NOT skip this — rendering a table is not enough. Call each one:
+
+$TASK_INSTRUCTIONS
+
+Call all of these TaskCreate invocations NOW, before doing anything else. This is non-negotiable.
+EOF

package/hooks/settings-template.json

@@ -1,81 +1,91 @@
-{
-  "$schema": "https://json.schemastore.org/claude-code-settings.json",
-  "_comment": "FTM hooks configuration template. Merge these entries into your ~/.claude/settings.json under the 'hooks' key. install.sh can do this automatically with --setup-hooks.",
-  "hooks": {
-    "PreToolUse": [
-      {
-        "matcher": "Edit|Write",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-plan-gate.sh",
-            "timeout": 5
-          }
-        ]
-      },
-      {
-        "matcher": "mcp__slack__slack_post_message|mcp__slack__slack_reply_to_thread|mcp__gmail__send_email",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-drafts-gate.sh",
-            "timeout": 5
-          }
-        ]
-      }
-    ],
-    "UserPromptSubmit": [
-      {
-        "hooks": [
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-discovery-reminder.sh",
-            "timeout": 5
-          },
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-pending-sync-check.sh",
-            "timeout": 5
-          },
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-map-autodetect.sh",
-            "timeout": 5
-          }
-        ]
-      }
-    ],
-    "PostToolUse": [
-      {
-        "matcher": "",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "node ~/.claude/hooks/ftm-event-logger.mjs"
-          }
-        ]
-      },
-      {
-        "matcher": "Bash|mcp__git__git_commit",
-        "hooks": [
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-post-commit-trigger.sh",
-            "timeout": 5
-          }
-        ]
-      }
-    ],
-    "Stop": [
-      {
-        "hooks": [
-          {
-            "type": "command",
-            "command": "~/.claude/hooks/ftm-blackboard-enforcer.sh",
-            "timeout": 5
-          }
-        ]
-      }
-    ]
-  }
-}
+{
+  "$schema": "https://json.schemastore.org/claude-code-settings.json",
+  "_comment": "FTM hooks configuration template. Merge these entries into your ~/.claude/settings.json under the 'hooks' key. install.sh can do this automatically with --setup-hooks.",
+  "hooks": {
+    "PreToolUse": [
+      {
+        "matcher": "Edit|Write",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-plan-gate.sh",
+            "timeout": 5
+          }
+        ]
+      },
+      {
+        "matcher": "mcp__slack__slack_post_message|mcp__slack__slack_reply_to_thread|mcp__gmail__send_email",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-drafts-gate.sh",
+            "timeout": 5
+          }
+        ]
+      }
+    ],
+    "UserPromptSubmit": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-discovery-reminder.sh",
+            "timeout": 5
+          },
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-pending-sync-check.sh",
+            "timeout": 5
+          },
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-map-autodetect.sh",
+            "timeout": 5
+          }
+        ]
+      }
+    ],
+    "PostToolUse": [
+      {
+        "matcher": "",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "node ~/.claude/hooks/ftm-event-logger.mjs"
+          }
+        ]
+      },
+      {
+        "matcher": "Skill",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-task-loader.sh",
+            "timeout": 10
+          }
+        ]
+      },
+      {
+        "matcher": "Bash|mcp__git__git_commit",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-post-commit-trigger.sh",
+            "timeout": 5
+          }
+        ]
+      }
+    ],
+    "Stop": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "~/.claude/hooks/ftm-blackboard-enforcer.sh",
+            "timeout": 5
+          }
+        ]
+      }
+    ]
+  }
+}