codexkit 1.0.0

package/.codex/scripts/memory_search.sh

@@ -0,0 +1,292 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script: memory_search
+# Ranked search across durable memory and context docs using derived local indexes.
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+INDEX_FILE="$SCRIPT_DIR/memory/.index/memory-index.jsonl"
+DB_FILE="$SCRIPT_DIR/memory/.index/memory-index.db"
+QUERY=""
+LIMIT=5
+REBUILD=false
+DECISIONS_ONLY=false
+BACKEND="${CODEXKIT_MEMORY_BACKEND:-auto}"
+ADVANCED_MODE="${CODEXKIT_ADVANCED_MEMORY_MODE:-off}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --limit)
+      shift
+      LIMIT="${1:-5}"
+      ;;
+    --limit=*)
+      LIMIT="${1#--limit=}"
+      ;;
+    --rebuild)
+      REBUILD=true
+      ;;
+    --decisions-only)
+      DECISIONS_ONLY=true
+      ;;
+    *)
+      if [[ -z "$QUERY" ]]; then
+        QUERY="$1"
+      else
+        QUERY="$QUERY $1"
+      fi
+      ;;
+  esac
+  shift || true
+done
+
+if [[ -z "$QUERY" ]]; then
+  echo "Usage: $0 <query> [--limit <n>] [--rebuild]" >&2
+  exit 1
+fi
+
+if ! [[ "$LIMIT" =~ ^[0-9]+$ ]]; then
+  echo "limit must be an integer" >&2
+  exit 1
+fi
+
+if [[ ! "$BACKEND" =~ ^(auto|sqlite|jsonl)$ ]]; then
+  echo "CODEXKIT_MEMORY_BACKEND must be one of: auto, sqlite, jsonl" >&2
+  exit 1
+fi
+
+if [[ ! "$ADVANCED_MODE" =~ ^(off|basic-memory|bridge)$ ]]; then
+  echo "CODEXKIT_ADVANCED_MEMORY_MODE must be one of: off, basic-memory, bridge" >&2
+  exit 1
+fi
+
+SOURCE_CHANGED=false
+if [[ -f "$DB_FILE" ]] && find "$SCRIPT_DIR/context" "$SCRIPT_DIR/memory/project" "$SCRIPT_DIR/memory/research" -type f -name '*.md' -newer "$DB_FILE" -print -quit | grep -q .; then
+  SOURCE_CHANGED=true
+fi
+
+if $REBUILD || [[ ! -f "$INDEX_FILE" ]] || [[ ! -f "$DB_FILE" ]] || $SOURCE_CHANGED; then
+  "$SCRIPT_DIR/scripts/build_memory_index.sh" >/dev/null
+fi
+
+python3 - "$DB_FILE" "$INDEX_FILE" "$QUERY" "$LIMIT" "$BACKEND" "$ADVANCED_MODE" "$DECISIONS_ONLY" <<'PY'
+import json
+import re
+import sqlite3
+import sys
+from pathlib import Path
+
+db_file = Path(sys.argv[1])
+index_file = Path(sys.argv[2])
+query = sys.argv[3].strip()
+limit = int(sys.argv[4])
+backend = sys.argv[5]
+advanced_mode = sys.argv[6]
+decisions_only = sys.argv[7] == "true"
+query_lower = query.lower()
+tokens = [token for token in re.split(r"[^a-z0-9._/-]+", query_lower) if len(token) >= 2]
+
+
+def print_results(hits, index_path, index_type):
+    print("# Memory Search")
+    print()
+    print(f"- Query: {query}")
+    print(f"- Limit: {limit}")
+    print(f"- Index: {index_path}")
+    print(f"- Index Type: {index_type}")
+    print(f"- Backend Preference: {backend}")
+    print(f"- Advanced Memory Mode: {advanced_mode}")
+    print(f"- Decisions Only: {'yes' if decisions_only else 'no'}")
+    print()
+
+    if not hits:
+        print("No matches found.")
+        return
+
+    for idx, item in enumerate(hits[:limit], start=1):
+        print(f"{idx}. [score={item['score']}] {item['path']}")
+        print(f"   - Heading: {item['heading']}")
+        print(f"   - Source Type: {item.get('source_type', 'other')}")
+        print(f"   - Snippet: {item['snippet'][:180]}")
+        print(f"   - Why: {item['reasons']}")
+
+    decision_hits = [h for h in hits if h["path"].endswith("decision-log.md")]
+    if decision_hits:
+        print()
+        print("## Recent Related Decisions")
+        print()
+        for idx, item in enumerate(decision_hits[: min(3, limit)], start=1):
+            print(f"{idx}. [score={item['score']}] {item['heading']}")
+            print(f"   - Path: {item['path']}")
+            print(f"   - Snippet: {item['snippet'][:180]}")
+
+
+def sqlite_hits():
+    if backend == "jsonl" or not db_file.exists():
+        return None
+
+    conn = sqlite3.connect(db_file)
+    conn.row_factory = sqlite3.Row
+    try:
+        token_query = " OR ".join(f'"{token}"*' for token in tokens) if tokens else f'"{query_lower}"'
+        decisions_clause = "AND s.path = 'context/decision-log.md'" if decisions_only else ""
+        sql = """
+            SELECT
+                s.path,
+                s.heading,
+                s.level,
+                s.source_type,
+                s.snippet,
+                s.token_count,
+                s.mtime,
+                bm25(sections_fts, 3.0, 5.0, 2.0, 1.5) AS fts_rank
+            FROM sections_fts
+            JOIN sections s ON s.id = sections_fts.rowid
+            WHERE sections_fts MATCH ?
+            {decisions_clause}
+            ORDER BY fts_rank ASC, s.mtime DESC, s.path ASC, s.heading ASC
+            LIMIT ?
+        """.format(decisions_clause=decisions_clause)
+        rows = conn.execute(sql, (token_query, max(limit * 6, 20))).fetchall()
+        if not rows:
+            return []
+
+        hits = []
+        for row in rows:
+            score = max(1, int(round(100 - (row["fts_rank"] * 10))))
+            reasons = ["fts match"]
+            path = row["path"].lower()
+            heading = row["heading"].lower()
+            snippet = row["snippet"].lower()
+            source_type = row["source_type"]
+            token_count = max(1, int(row["token_count"]))
+            coverage_hits = sum(1 for token in tokens if token in heading or token in snippet or token in path)
+            if query_lower in heading:
+                score += 18
+                reasons.append("heading match")
+            if query_lower in snippet:
+                score += 12
+                reasons.append("exact phrase")
+            if query_lower in path:
+                score += 8
+                reasons.append("path match")
+            if coverage_hits:
+                score += int(round((coverage_hits / max(1, len(tokens))) * 20))
+                reasons.append(f"token coverage {coverage_hits}/{max(1, len(tokens))}")
+            if source_type == "decision-log":
+                score += 4
+            elif source_type == "project-state":
+                score += 3
+            elif source_type == "session-context":
+                score += 2
+            if "decision-log.md" in path:
+                score += 2
+            if "state.md" in path:
+                score += 1
+            if token_count < 80:
+                score += 1
+            hits.append(
+                {
+                    "path": row["path"],
+                    "heading": row["heading"],
+                    "level": row["level"],
+                    "source_type": source_type,
+                    "snippet": row["snippet"],
+                    "mtime": row["mtime"],
+                    "score": score,
+                    "reasons": ", ".join(dict.fromkeys(reasons)),
+                }
+            )
+
+        hits.sort(key=lambda item: (-item["score"], -item["mtime"], item["path"], item["heading"]))
+        return hits
+    finally:
+        conn.close()
+
+
+def jsonl_hits():
+    if not index_file.exists():
+        print("No derived index found. Run with --rebuild.", file=sys.stderr)
+        sys.exit(1)
+
+    def rank(record):
+        heading = record["heading"].lower()
+        path = record["path"].lower()
+        snippet = record["snippet"].lower()
+        source_type = record.get("source_type", "other")
+        token_count = max(1, int(record.get("token_count", 1)))
+        score = 0
+        reasons = []
+
+        phrase_hits = snippet.count(query_lower) + heading.count(query_lower)
+        if phrase_hits:
+            score += phrase_hits * 20
+            reasons.append(f"exact phrase x{phrase_hits}")
+
+        if query_lower in heading:
+            score += 30
+            reasons.append("heading match")
+
+        if query_lower in path:
+            score += 12
+            reasons.append("path match")
+
+        if "decision-log.md" in path:
+            score += 2
+
+        if "state.md" in path:
+            score += 1
+
+        coverage_hits = 0
+        for token in tokens:
+            if token in heading:
+                score += 8
+                coverage_hits += 1
+            if token in path:
+                score += 6
+            if token in snippet:
+                score += 3
+                coverage_hits += 1
+
+        if coverage_hits:
+            score += int(round((coverage_hits / max(1, len(tokens) * 2)) * 18))
+            reasons.append(f"token coverage {coverage_hits}")
+
+        if source_type == "decision-log":
+            score += 4
+        elif source_type == "project-state":
+            score += 3
+        elif source_type == "session-context":
+            score += 2
+
+        if token_count < 80:
+            score += 1
+
+        return score, reasons
+
+    hits = []
+    with index_file.open(encoding="utf-8") as fh:
+        for line in fh:
+            record = json.loads(line)
+            if decisions_only and record["path"] != "context/decision-log.md":
+                continue
+            score, reasons = rank(record)
+            if score <= 0:
+                continue
+            record["score"] = score
+            record["reasons"] = ", ".join(dict.fromkeys(reasons)) or "token overlap"
+            hits.append(record)
+
+    hits.sort(key=lambda item: (-item["score"], -item["mtime"], item["path"], item["heading"]))
+    return hits
+
+
+hits = None if backend == "jsonl" else sqlite_hits()
+if backend == "sqlite" and hits is None:
+    print("SQLite backend requested but no SQLite index is available.", file=sys.stderr)
+    sys.exit(1)
+if hits is not None:
+    print_results(hits, db_file, "sqlite-fts")
+else:
+    print_results(jsonl_hits(), index_file, "jsonl-fallback")
+PY

package/.codex/scripts/prepare_worklog_archive.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+set -euo pipefail
+
+# Script: prepare_worklog_archive
+# Creates non-destructive archive and retained-worklog drafts from older worklog entries.
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <archive-output-path> [keep_recent_entries] [--retained-output <path>]" >&2
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+WORKLOG="$SCRIPT_DIR/context/worklog.md"
+ARCHIVE_OUT=""
+KEEP_RECENT="20"
+RETAINED_OUT=""
+KEEP_RECENT_SET=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --retained-output)
+      shift
+      RETAINED_OUT="${1:-}"
+      if [[ -z "$RETAINED_OUT" ]]; then
+        echo "--retained-output requires a path" >&2
+        exit 1
+      fi
+      ;;
+    *)
+      if [[ -z "$ARCHIVE_OUT" ]]; then
+        ARCHIVE_OUT="$1"
+      elif ! $KEEP_RECENT_SET; then
+        KEEP_RECENT="$1"
+        KEEP_RECENT_SET=true
+      else
+        echo "Unexpected argument: $1" >&2
+        echo "Usage: $0 <archive-output-path> [keep_recent_entries] [--retained-output <path>]" >&2
+        exit 1
+      fi
+      ;;
+  esac
+  shift || true
+done
+
+if [[ -z "$ARCHIVE_OUT" ]]; then
+  echo "archive-output-path is required" >&2
+  exit 1
+fi
+
+if [[ ! -f "$WORKLOG" ]]; then
+  echo "worklog not found: $WORKLOG" >&2
+  exit 1
+fi
+
+if ! [[ "$KEEP_RECENT" =~ ^[0-9]+$ ]]; then
+  echo "keep_recent_entries must be an integer" >&2
+  exit 1
+fi
+
+entries=()
+while IFS= read -r line; do
+  entries+=("$line")
+done < <(grep '^-' "$WORKLOG" || true)
+total="${#entries[@]}"
+
+if [[ "$total" -le "$KEEP_RECENT" ]]; then
+  echo "Nothing to archive: worklog has $total entries and keep_recent_entries is $KEEP_RECENT."
+  exit 0
+fi
+
+archive_count=$((total - KEEP_RECENT))
+recent_start=$archive_count
+
+{
+  echo "# Worklog Archive Draft"
+  echo
+  echo "> Generated from \`.codex/context/worklog.md\`."
+  echo "> Review before committing."
+  echo
+  echo "## Archived Entries"
+  for ((i=0; i<archive_count; i++)); do
+    echo "${entries[$i]}"
+  done
+} > "$ARCHIVE_OUT"
+
+if [[ -n "$RETAINED_OUT" ]]; then
+  {
+    echo "# Worklog"
+    echo
+    echo "> Session-level human-readable trail. Update after significant milestones."
+    echo
+    echo "---"
+    echo
+    echo "## Entries"
+    for ((i=recent_start; i<total; i++)); do
+      echo "${entries[$i]}"
+    done
+  } > "$RETAINED_OUT"
+fi
+
+echo "Archive draft written to: $ARCHIVE_OUT"
+echo "Archived entries: $archive_count"
+echo "Recent entries left in active worklog: $KEEP_RECENT"
+if [[ -n "$RETAINED_OUT" ]]; then
+  echo "Retained worklog draft written to: $RETAINED_OUT"
+fi
+echo
+echo "Next step:"
+echo "- Review the archive draft."
+if [[ -n "$RETAINED_OUT" ]]; then
+  echo "- Review the retained worklog draft."
+  echo "- If both drafts look correct, replace .codex/context/worklog.md with the retained draft and store the archive draft in a reviewable location."
+else
+  echo "- If it looks correct, manually move those older entries out of .codex/context/worklog.md."
+fi

package/.codex/scripts/search_bead_context.sh

@@ -0,0 +1,214 @@
+#!/bin/bash
+set -euo pipefail
+# Script: search_bead_context
+# Ranked search across synced bead summaries and durable context docs.
+
+PROJECT_ROOT="${CODEXKIT_PROJECT_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)}"
+QUERY=""
+LIMIT=3
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --limit)
+      shift
+      LIMIT="${1:-3}"
+      ;;
+    --limit=*)
+      LIMIT="${1#--limit=}"
+      ;;
+    *)
+      if [[ -z "$QUERY" ]]; then
+        QUERY="$1"
+      else
+        QUERY="$QUERY $1"
+      fi
+      ;;
+  esac
+  shift || true
+done
+
+if [[ -z "$QUERY" ]]; then
+  echo "Usage: $0 <query> [--limit <n>]" >&2
+  exit 1
+fi
+
+python3 - "$PROJECT_ROOT" "$QUERY" "$LIMIT" <<'PY'
+import re
+import sys
+from pathlib import Path
+
+project_root = Path(sys.argv[1])
+query = sys.argv[2].strip()
+limit = int(sys.argv[3])
+query_lower = query.lower()
+tokens = [token for token in re.split(r"[^a-z0-9._/-]+", query_lower) if len(token) >= 3]
+
+summary_dir = project_root / ".codex" / "memory" / "research" / "beads"
+context_files = [
+    project_root / ".codex" / "context" / "worklog.md",
+    project_root / ".codex" / "context" / "decision-log.md",
+    project_root / ".codex" / "memory" / "project" / "state.md",
+]
+
+
+def read_text(path: Path) -> str:
+    if not path.exists() or not path.is_file():
+        return ""
+    return path.read_text(encoding="utf-8", errors="replace")
+
+
+def extract_section(text: str, heading: str) -> str:
+    pattern = re.compile(
+        rf"^##+\s+{re.escape(heading)}\s*$\n(.*?)(?=^##+\s+|\Z)",
+        re.MULTILINE | re.DOTALL,
+    )
+    match = pattern.search(text)
+    return match.group(1).strip() if match else ""
+
+
+def compact_line(text: str) -> str:
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        return re.sub(r"\s+", " ", line)[:180]
+    return ""
+
+
+def find_bead_field(text: str, label: str) -> str:
+    bead_section = extract_section(text, "Bead")
+    if not bead_section:
+        return ""
+    prefix = f"- {label}:"
+    for raw_line in bead_section.splitlines():
+        line = raw_line.strip()
+        if line.startswith(prefix):
+            return line
+    return ""
+
+
+def rank_summary(path: Path):
+    text = read_text(path)
+    if not text:
+        return None
+    lower = text.lower()
+    title = text.splitlines()[0] if text.splitlines() else path.stem
+    title_lower = title.lower()
+    keywords = extract_section(text, "Keywords").lower()
+    problem = extract_section(text, "Problem").lower()
+    outcome = extract_section(text, "Outcome").lower()
+    status_line = find_bead_field(text, "Status")
+
+    score = 0
+    reasons = []
+    phrase_hits = lower.count(query_lower)
+    if phrase_hits:
+        score += phrase_hits * 30
+        reasons.append(f"exact phrase x{phrase_hits}")
+
+    if query_lower in title_lower:
+        score += 40
+        reasons.append("title match")
+
+    for token in tokens:
+        if token in path.stem.lower():
+            score += 12
+        if token in title_lower:
+            score += 12
+        if token in keywords:
+            score += 10
+        if token in problem:
+            score += 6
+        if token in outcome:
+            score += 6
+        if token in lower:
+            score += 2
+
+    if score <= 0:
+        return None
+
+    return {
+        "score": score,
+        "path": path.relative_to(project_root).as_posix(),
+        "title": title.removeprefix("# ").strip(),
+        "status": status_line or "- Status: not captured",
+        "excerpt": compact_line(extract_section(text, "Outcome")) or compact_line(extract_section(text, "Problem")),
+        "reasons": ", ".join(dict.fromkeys(reasons)) or "token overlap",
+    }
+
+
+def best_excerpt(text: str) -> str:
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line:
+            continue
+        if query_lower in line.lower():
+            return re.sub(r"\s+", " ", line)[:180]
+    return compact_line(text)
+
+
+def rank_context(path: Path):
+    text = read_text(path)
+    if not text:
+        return None
+    lower = text.lower()
+    score = 0
+    phrase_hits = lower.count(query_lower)
+    if phrase_hits:
+        score += phrase_hits * 12
+    for token in tokens:
+        if token in lower:
+            score += 3
+    if score <= 0:
+        return None
+    return {
+        "score": score,
+        "path": path.relative_to(project_root).as_posix(),
+        "excerpt": best_excerpt(text),
+    }
+
+
+summary_hits = []
+if summary_dir.exists():
+    for path in sorted(summary_dir.glob("*.md")):
+        ranked = rank_summary(path)
+        if ranked:
+            summary_hits.append(ranked)
+summary_hits.sort(key=lambda item: (-item["score"], item["path"]))
+
+context_hits = []
+for path in context_files:
+    ranked = rank_context(path)
+    if ranked:
+        context_hits.append(ranked)
+context_hits.sort(key=lambda item: (-item["score"], item["path"]))
+
+print("# Bead Context Search")
+print()
+print(f"- Query: {query}")
+print(f"- Limit: {limit}")
+print()
+print("## Related Bead Summaries")
+print()
+if summary_hits:
+    for index, item in enumerate(summary_hits[:limit], start=1):
+        print(f"{index}. [score={item['score']}] {item['path']}")
+        print(f"   - Title: {item['title']}")
+        print(f"   - {item['status']}")
+        if item["excerpt"]:
+            print(f"   - Excerpt: {item['excerpt']}")
+        print(f"   - Why: {item['reasons']}")
+else:
+    print("- No related bead summaries found.")
+
+print()
+print("## Related Durable Context")
+print()
+if context_hits:
+    for index, item in enumerate(context_hits[:limit], start=1):
+        print(f"{index}. [score={item['score']}] {item['path']}")
+        if item["excerpt"]:
+            print(f"   - Excerpt: {item['excerpt']}")
+else:
+    print("- No related durable context files found.")
+PY