@jairodri/huckleberry-cli 1.0.2 → 1.1.0

package/skills/claw-compactor/scripts/lib/engram_prompts.py

@@ -0,0 +1,175 @@
+"""
+engram_prompts.py — System prompts for Engram Observer and Reflector agents.
+
+Both prompts produce structured, bilingual (EN/ZH), priority-annotated
+observation logs compatible with the claw-compactor pipeline.
+
+Part of claw-compactor / Engram layer. License: MIT.
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# Observer system prompt
+# ---------------------------------------------------------------------------
+
+OBSERVER_SYSTEM_PROMPT = """\
+You are the **Observer Agent** — a specialist in compressing raw conversation \
+history into a structured, high-signal observation log.
+
+## Your Mission
+Transform a batch of raw messages into a concise, structured observation log \
+that preserves all important information while drastically reducing token count.
+
+## Output Format
+
+Produce observations grouped by date, using the following exact structure:
+
+Date: YYYY-MM-DD
+- 🔴 HH:MM <critical observation — user goals, deadlines, blockers, key decisions>
+  - 🔴 HH:MM <sub-point — equally critical detail>
+  - 🟡 HH:MM <sub-point — important context>
+  - 🟢 HH:MM <sub-point — useful but lower priority>
+- 🟡 HH:MM <important observation — technical details, preferences, plans>
+- 🟢 HH:MM <useful observation — background info, mentions, soft context>
+
+## Priority Legend
+- 🔴 **Critical** — user goals, hard deadlines, blocking issues, key decisions, \
+important user preferences
+- 🟡 **Important** — technical details, ongoing work, significant context, \
+tool outputs summary, preferences
+- 🟢 **Useful** — background information, mentions, soft context, \
+non-blocking observations
+
+## Timestamp Rules
+- Use the actual timestamps from the conversation when available.
+- If no timestamp is present, use an approximate relative position (e.g., 00:01, 00:02…).
+- Each observation entry must have exactly ONE timestamp on the same line.
+
+## Three-Date Model
+- **Observation date**: The date the Observer is running (today).
+- **Referenced date**: The date the events actually occurred (from message timestamps).
+- **Relative date**: How far back the events are (e.g., "yesterday", "3 days ago").
+Use the referenced date in entries, not the observation date, unless they are the same.
+
+## Compression Targets
+- Plain text conversations: achieve **3–6× compression**
+- Tool call outputs / code blocks: achieve **5–40× compression** \
+(summarise results, not raw output)
+- **Never** omit critical information (🔴); minimise 🟢 freely.
+
+## Language
+- Write observations in **both Chinese and English** when the conversation is \
+bilingual, or match the dominant language of the conversation.
+- Technical terms, proper nouns, and code identifiers: keep in original language.
+
+## Important Rules
+1. Preserve ALL 🔴 critical items — no exceptions.
+2. Merge closely related consecutive items into one entry with sub-bullets.
+3. For tool outputs: summarise the outcome, not the raw data.
+4. For code blocks: note what the code does / its result, not the full code \
+(unless it's a critical snippet ≤5 lines).
+5. Dates come from the messages; if ambiguous use today's date.
+6. Output ONLY the observation log — no preamble, no explanation, no markdown fences.
+
+## Example Output
+Date: 2026-03-05
+- 🔴 12:10 User is building OpenCompress project; deadline within one week / \
+用户在构建 OpenCompress 项目，deadline 一周内
+  - 🔴 12:10 Using ModernBERT-large for inference / 使用 ModernBERT-large 做推理
+  - 🟡 12:12 Discussed training data annotation strategy / 讨论了训练数据标注策略
+  - 🟢 12:15 Mentioned benchmark results are promising / 提到 benchmark 结果不错
+- 🟡 12:30 Switched to discussing deployment pipeline on M3 Ultra
+- 🟢 12:45 User prefers concise, structured replies
+"""
+
+# ---------------------------------------------------------------------------
+# Reflector system prompt
+# ---------------------------------------------------------------------------
+
+REFLECTOR_SYSTEM_PROMPT = """\
+You are the **Reflector Agent** — a specialist in distilling and compressing \
+an accumulated observation log into a tighter, pattern-aware reflection log.
+
+## Your Mission
+Take a large observation log (previously produced by the Observer Agent) and:
+1. **Merge** related entries across dates into unified threads.
+2. **Promote** recurring patterns and long-term context to the top.
+3. **Prune** outdated or superseded information.
+4. **Preserve** all 🔴 critical items — never drop them.
+
+## Output Format
+
+Produce a two-section reflection log:
+
+## Persistent Context (long-term patterns & facts)
+- 🔴 <fact/pattern that spans multiple sessions or is permanently relevant>
+- 🟡 <recurring theme or preference>
+- 🟢 <background context>
+
+## Recent Events (chronological, compressed)
+Date: YYYY-MM-DD
+- 🔴 HH:MM <critical event>
+  - 🟡 HH:MM <important sub-detail>
+- 🟡 HH:MM <important event>
+- 🟢 HH:MM <useful event>
+
+Date: YYYY-MM-DD
+...
+
+## Priority Legend (same as Observer)
+- 🔴 **Critical** — user goals, hard deadlines, blocking issues, key decisions
+- 🟡 **Important** — technical details, ongoing work, significant context
+- 🟢 **Useful** — background, mentions, soft context
+
+## Reflection Rules
+1. **Never drop 🔴 items** — consolidate if possible, never delete.
+2. **Merge related items**: If the same topic appears across multiple dates, \
+merge into a single "Persistent Context" entry with a note like \
+"(repeated across 3 sessions, last: 2026-03-05)".
+3. **Mark superseded info**: If a later entry contradicts an earlier one, \
+keep only the latest and note it was updated.
+4. **Identify patterns**: If a user repeatedly asks about the same topic, \
+note it as a persistent interest.
+5. **Prune freely**: 🟢 items older than 7 days that are not referenced again \
+can be dropped. 🟡 items older than 30 days that are not part of a pattern \
+can be condensed to a one-liner.
+6. **Keep event structure**: Do NOT collapse everything into a blob summary. \
+The output must remain scannable and structured.
+
+## Compression Target
+- Achieve **2–4× compression** over the input observation log while retaining \
+all 🔴 items and key 🟡 items.
+
+## Language
+- Match the language style of the input (bilingual if input is bilingual).
+- Technical terms, proper nouns, code identifiers: keep in original language.
+
+## Output
+Output ONLY the reflection log — no preamble, no explanation, no markdown fences.
+"""
+
+# ---------------------------------------------------------------------------
+# User-turn templates
+# ---------------------------------------------------------------------------
+
+OBSERVER_USER_TEMPLATE = """\
+Please observe and compress the following conversation messages into a \
+structured observation log.
+
+Current date/time: {current_datetime}
+
+--- MESSAGES START ---
+{messages_text}
+--- MESSAGES END ---
+"""
+
+REFLECTOR_USER_TEMPLATE = """\
+Please reflect on and compress the following accumulated observation log.
+
+Current date/time: {current_datetime}
+
+--- OBSERVATIONS START ---
+{observations_text}
+--- OBSERVATIONS END ---
+"""

package/skills/claw-compactor/scripts/lib/engram_storage.py

@@ -0,0 +1,254 @@
+"""
+engram_storage.py — File-system storage backend for Engram (Observational Memory).
+
+Layout under base_path/memory/engram/{thread_id}/:
+    observations.md   — append-only observation log (Observer output)
+    reflections.md    — latest reflection (Reflector output, overwritten each run)
+    pending.jsonl     — raw pending messages not yet observed (JSONL, append-only)
+    meta.json         — per-thread statistics and timestamps
+
+All writes use atomic rename (tempfile + os.replace) to avoid partial-write
+corruption even on crash.
+
+Part of claw-compactor / Engram layer. License: MIT.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional
+
+
+class EngramStorage:
+    """
+    File-system storage for Engram's three-layer memory.
+
+    Args:
+        base_path: Workspace root directory. Engram data lives at
+                   ``{base_path}/memory/engram/{thread_id}/``.
+    """
+
+    def __init__(self, base_path: Path) -> None:
+        self.base_path = Path(base_path)
+
+    # ------------------------------------------------------------------
+    # Path helpers
+    # ------------------------------------------------------------------
+
+    def _thread_dir(self, thread_id: str) -> Path:
+        """Return (and create) the directory for a thread."""
+        d = self.base_path / "memory" / "engram" / thread_id
+        d.mkdir(parents=True, exist_ok=True)
+        return d
+
+    def _obs_path(self, thread_id: str) -> Path:
+        return self._thread_dir(thread_id) / "observations.md"
+
+    def _ref_path(self, thread_id: str) -> Path:
+        return self._thread_dir(thread_id) / "reflections.md"
+
+    def _pending_path(self, thread_id: str) -> Path:
+        return self._thread_dir(thread_id) / "pending.jsonl"
+
+    def _meta_path(self, thread_id: str) -> Path:
+        return self._thread_dir(thread_id) / "meta.json"
+
+    # ------------------------------------------------------------------
+    # Observations (append-only Markdown)
+    # ------------------------------------------------------------------
+
+    def append_observation(
+        self,
+        thread_id: str,
+        observation: str,
+        timestamp: Optional[str] = None,
+    ) -> None:
+        """
+        Append a new observation block to the thread's observation log.
+
+        A separator header is prepended so multiple Observer runs are
+        distinguishable.
+
+        Args:
+            thread_id:   Thread identifier.
+            observation: Observation text from the Observer LLM.
+            timestamp:   Optional ISO timestamp; defaults to UTC now.
+        """
+        ts = timestamp or _now_utc()
+        header = f"\n<!-- observed_at: {ts} -->\n"
+        content = header + observation.strip() + "\n"
+
+        path = self._obs_path(thread_id)
+        with path.open("a", encoding="utf-8") as f:
+            f.write(content)
+
+        self._update_meta(thread_id, last_observed_at=ts)
+
+    def read_observations(self, thread_id: str) -> str:
+        """Read the full observation log for a thread (empty string if none)."""
+        path = self._obs_path(thread_id)
+        if not path.exists():
+            return ""
+        return path.read_text(encoding="utf-8")
+
+    # ------------------------------------------------------------------
+    # Reflections (overwrite each run)
+    # ------------------------------------------------------------------
+
+    def write_reflection(
+        self,
+        thread_id: str,
+        reflection: str,
+        timestamp: Optional[str] = None,
+    ) -> None:
+        """
+        Write (overwrite) the reflection for a thread using atomic rename.
+
+        Args:
+            thread_id:  Thread identifier.
+            reflection: Reflection text from the Reflector LLM.
+            timestamp:  Optional ISO timestamp; defaults to UTC now.
+        """
+        ts = timestamp or _now_utc()
+        header = f"<!-- reflected_at: {ts} -->\n"
+        content = header + reflection.strip() + "\n"
+
+        path = self._ref_path(thread_id)
+        _atomic_write(path, content)
+        self._update_meta(thread_id, last_reflected_at=ts)
+
+    def read_reflection(self, thread_id: str) -> str:
+        """Read the latest reflection for a thread (empty string if none)."""
+        path = self._ref_path(thread_id)
+        if not path.exists():
+            return ""
+        return path.read_text(encoding="utf-8")
+
+    # ------------------------------------------------------------------
+    # Pending messages (JSONL, append-only)
+    # ------------------------------------------------------------------
+
+    def append_message(self, thread_id: str, message: dict) -> None:
+        """
+        Append a raw message dict to the pending queue.
+
+        Args:
+            thread_id: Thread identifier.
+            message:   Dict with at least ``"role"`` and ``"content"``.
+        """
+        path = self._pending_path(thread_id)
+        with path.open("a", encoding="utf-8") as f:
+            f.write(json.dumps(message, ensure_ascii=False) + "\n")
+
+    def read_pending(self, thread_id: str) -> List[dict]:
+        """
+        Read all pending messages for a thread.
+
+        Returns:
+            List of message dicts in append order.
+        """
+        path = self._pending_path(thread_id)
+        if not path.exists():
+            return []
+        messages: List[dict] = []
+        with path.open("r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    try:
+                        messages.append(json.loads(line))
+                    except json.JSONDecodeError:
+                        pass  # skip corrupted lines
+        return messages
+
+    def clear_pending(self, thread_id: str) -> None:
+        """
+        Truncate the pending queue (called after a successful observe run).
+
+        Args:
+            thread_id: Thread identifier.
+        """
+        path = self._pending_path(thread_id)
+        if path.exists():
+            path.write_text("", encoding="utf-8")
+        self._update_meta(thread_id, pending_count=0)
+
+    def pending_count(self, thread_id: str) -> int:
+        """Return the number of pending messages."""
+        return len(self.read_pending(thread_id))
+
+    # ------------------------------------------------------------------
+    # Metadata
+    # ------------------------------------------------------------------
+
+    def read_meta(self, thread_id: str) -> dict:
+        """
+        Read thread metadata.
+
+        Returns:
+            Metadata dict, or a minimal default dict if none exists.
+        """
+        path = self._meta_path(thread_id)
+        if not path.exists():
+            return {"thread_id": thread_id, "created_at": None}
+        try:
+            return json.loads(path.read_text(encoding="utf-8"))
+        except (json.JSONDecodeError, OSError):
+            return {"thread_id": thread_id}
+
+    def _update_meta(self, thread_id: str, **kwargs: object) -> None:
+        """Merge *kwargs* into thread metadata and persist atomically."""
+        meta = self.read_meta(thread_id)
+        if not meta.get("created_at"):
+            meta["created_at"] = datetime.now(timezone.utc).isoformat()
+        meta["thread_id"] = thread_id
+        meta.update(kwargs)
+        meta["updated_at"] = datetime.now(timezone.utc).isoformat()
+        _atomic_write(
+            self._meta_path(thread_id),
+            json.dumps(meta, ensure_ascii=False, indent=2),
+        )
+
+    # ------------------------------------------------------------------
+    # Thread discovery
+    # ------------------------------------------------------------------
+
+    def list_threads(self) -> List[str]:
+        """Return sorted list of all known thread IDs."""
+        engram_dir = self.base_path / "memory" / "engram"
+        if not engram_dir.exists():
+            return []
+        return sorted(
+            d.name
+            for d in engram_dir.iterdir()
+            if d.is_dir() and (d / "meta.json").exists()
+        )
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _now_utc() -> str:
+    return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
+
+
+def _atomic_write(path: Path, content: str) -> None:
+    """Write *content* to *path* atomically via tempfile + os.replace."""
+    dir_ = path.parent
+    dir_.mkdir(parents=True, exist_ok=True)
+    fd, tmp_path = tempfile.mkstemp(dir=dir_, prefix=".tmp_engram_")
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            f.write(content)
+        os.replace(tmp_path, path)
+    except Exception:
+        try:
+            os.unlink(tmp_path)
+        except OSError:
+            pass
+        raise

package/skills/claw-compactor/scripts/lib/exceptions.py

@@ -0,0 +1,24 @@
+"""Custom exceptions for claw-compactor.
+
+Part of claw-compactor. License: MIT.
+"""
+
+
+class MemCompressError(Exception):
+    """Base exception for claw-compactor operations."""
+    pass
+
+
+class FileNotFoundError_(MemCompressError):
+    """Raised when a required file or directory is not found."""
+    pass
+
+
+class ParseError(MemCompressError):
+    """Raised when input cannot be parsed (malformed markdown, JSON, etc.)."""
+    pass
+
+
+class TokenEstimationError(MemCompressError):
+    """Raised when token estimation fails."""
+    pass