@heytherevibin/skillforge 0.2.1 → 0.7.0

package/python/app/chunking.py

@@ -0,0 +1,116 @@
+"""Split SKILL.md bodies into line-bounded chunks for RAG-style retrieval."""
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+def chunk_max_chars() -> int:
+    return max(400, int(os.getenv("SKILLFORGE_CHUNK_MAX_CHARS", "1200")))
+
+
+def chunk_overlap_chars() -> int:
+    return max(0, int(os.getenv("SKILLFORGE_CHUNK_OVERLAP", "200")))
+
+
+@dataclass
+class SkillChunk:
+    """One span of a skill body with 1-based inclusive line numbers (within the body text)."""
+
+    text: str
+    line_start: int
+    line_end: int
+
+
+def _split_long_segment(text: str, line_start: int, max_chars: int, overlap: int) -> list[SkillChunk]:
+    """Character windows with overlap; ``line_start`` is the body line of ``text[0]`` (1-based)."""
+    if not text:
+        return []
+    line_no = line_start
+    line_at_idx: list[int] = []
+    for ch in text:
+        line_at_idx.append(line_no)
+        if ch == "\n":
+            line_no += 1
+    n = len(text)
+    out: list[SkillChunk] = []
+    i = 0
+    while i < n:
+        end = min(i + max_chars, n)
+        piece = text[i:end].strip()
+        if piece:
+            ls = line_at_idx[i]
+            le = line_at_idx[end - 1]
+            out.append(SkillChunk(piece, ls, le))
+        if end >= n:
+            break
+        adv = max(1, end - i - overlap)
+        i += adv
+    if out:
+        return out
+    st = text.strip()
+    if not st:
+        return []
+    le_fallback = line_start + max(0, text.count("\n"))
+    return [SkillChunk(st, line_start, max(line_start, le_fallback))]
+
+
+def chunk_skill_body(body: str, *, max_chars: int | None = None, overlap: int | None = None) -> list[SkillChunk]:
+    """Chunk by markdown headings (lines starting with ``#``) then hard-split long sections.
+
+    Empty body yields no chunks (caller may treat as single empty).
+    """
+    mc = max_chars if max_chars is not None else chunk_max_chars()
+    ov = overlap if overlap is not None else chunk_overlap_chars()
+    b = body or ""
+    if not b.strip():
+        return []
+
+    lines = b.split("\n")
+    sections: list[tuple[str, int, int]] = []
+    cur: list[str] = []
+    cur_start = 1
+    for i, line in enumerate(lines):
+        ln = i + 1
+        if line.startswith("#") and cur:
+            sections.append(("\n".join(cur), cur_start, ln - 1))
+            cur = [line]
+            cur_start = ln
+        else:
+            cur.append(line)
+    if cur:
+        sections.append(("\n".join(cur), cur_start, len(lines)))
+
+    chunks: list[SkillChunk] = []
+    for text, ls, le in sections:
+        text = text.strip()
+        if not text:
+            continue
+        if len(text) <= mc:
+            chunks.append(SkillChunk(text, ls, le))
+        else:
+            chunks.extend(_split_long_segment(text, ls, mc, ov))
+    return chunks if chunks else [SkillChunk(b.strip(), 1, max(1, len(lines)))]
+
+
+def chunk_raw_document(
+    body: str,
+    *,
+    max_chars: int | None = None,
+    overlap: int | None = None,
+) -> list[SkillChunk]:
+    """Chunk arbitrary file text with line-bounded windows (no markdown section split).
+
+    Line numbers are 1-based within the normalized document (``\\r\\n`` → ``\\n``).
+    """
+    mc = max_chars if max_chars is not None else chunk_max_chars()
+    ov = overlap if overlap is not None else chunk_overlap_chars()
+    if not body:
+        return []
+    normalized = body.replace("\r\n", "\n")
+    if not normalized.strip():
+        return []
+    line_count = normalized.count("\n") + 1
+    if len(normalized) <= mc:
+        return [SkillChunk(normalized, 1, max(1, line_count))]
+    return _split_long_segment(normalized, 1, mc, ov)

package/python/app/context_fusion.py

@@ -0,0 +1,77 @@
+"""MMR-based selection to fuse skill + project chunks under one character budget."""
+from __future__ import annotations
+
+from typing import Any
+
+import numpy as np
+
+
+def mmr_select(
+    embeddings: np.ndarray,
+    relevance: np.ndarray,
+    text_lengths: np.ndarray,
+    *,
+    char_budget: int,
+    overhead_per_chunk: int | np.ndarray,
+    lambda_mult: float,
+) -> tuple[list[int], list[dict[str, Any]]]:
+    """Greedy MMR over normalized row embeddings.
+
+    Each step maximizes ``lambda_mult * rel[i] - (1 - lambda_mult) * max_{j in selected} sim(i, j)``.
+
+    Returns selected **indices** in pick order and a trace row per pick (for telemetry).
+    """
+    n = int(embeddings.shape[0])
+    if n == 0 or char_budget <= 0:
+        return [], []
+
+    lam = float(lambda_mult)
+    lam = max(0.0, min(1.0, lam))
+    rel = np.asarray(relevance, dtype=np.float64).reshape(-1)
+    lens = np.asarray(text_lengths, dtype=np.int64).reshape(-1)
+    emb = np.asarray(embeddings, dtype=np.float32)
+    if isinstance(overhead_per_chunk, int):
+        ovh = np.full(n, int(overhead_per_chunk), dtype=np.int64)
+    else:
+        ovh = np.asarray(overhead_per_chunk, dtype=np.int64).reshape(-1)
+    if emb.shape[0] != n or rel.shape[0] != n or lens.shape[0] != n or ovh.shape[0] != n:
+        raise ValueError("embeddings, relevance, text_lengths, and overheads must align")
+
+    selected: list[int] = []
+    trace: list[dict[str, Any]] = []
+    used = 0
+    remaining = set(range(n))
+
+    while remaining:
+        best_i: int | None = None
+        best_mmr = -1e18
+        for i in remaining:
+            need = int(lens[i]) + int(ovh[i])
+            if need <= 0 or used + need > char_budget:
+                continue
+            if not selected:
+                div = 0.0
+            else:
+                sims = emb[i] @ emb[np.array(selected, dtype=np.int64)].T
+                div = float(np.max(sims))
+            mmr = lam * float(rel[i]) - (1.0 - lam) * div
+            if mmr > best_mmr:
+                best_mmr = mmr
+                best_i = i
+        if best_i is None:
+            break
+        if selected:
+            sims = emb[best_i] @ emb[np.array(selected, dtype=np.int64)].T
+            div_used = float(np.max(sims))
+        else:
+            div_used = 0.0
+        selected.append(best_i)
+        used += int(lens[best_i]) + int(ovh[best_i])
+        remaining.remove(best_i)
+        trace.append({
+            "pool_index": best_i,
+            "mmr": round(float(best_mmr), 6),
+            "relevance": round(float(rel[best_i]), 6),
+            "max_sim_to_selected": round(div_used, 6),
+        })
+    return selected, trace

package/python/app/events_cli.py

@@ -123,7 +123,7 @@ def main() -> None:
     db_path = resolve_orchestrator_db(pr)
 
     if not db_path.exists():
-        print("No database yet — run skillforge mcp or skillforge start first (or route once with this project_root).")
+        print("No database yet — run skillforge mcp first (or route once with this project_root).")
         print(f"  Expected: {db_path}")
         return
 

package/python/app/index_cli.py

@@ -0,0 +1,89 @@
+"""CLI: index project files into ``<project>/.skillforge/orchestrator.db`` for project RAG."""
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+from app.db_paths import resolve_orchestrator_db
+from app.main import build_router_and_skills, init_db
+from app.project_index import index_project, project_index_stats
+
+
+def _parse_args(argv: list[str] | None) -> argparse.Namespace:
+    p = argparse.ArgumentParser(
+        description=(
+            "Chunk and embed text files under project_root into the per-repo orchestrator DB. "
+            "Use with MCP route_skills/include_project_rag or skillforge route --include-project-rag."
+        ),
+    )
+    p.add_argument(
+        "--project-root",
+        required=True,
+        help="Repository root directory to index (writes .skillforge/orchestrator.db).",
+    )
+    p.add_argument(
+        "--reset",
+        action="store_true",
+        help="Clear all project_chunks rows before re-indexing.",
+    )
+    p.add_argument(
+        "--stats-only",
+        action="store_true",
+        help="Print index metadata from DB and exit (no scan/embed).",
+    )
+    p.add_argument(
+        "--quiet",
+        action="store_true",
+        help="Skip progress messages on stderr from skill loading.",
+    )
+    return p.parse_args(argv)
+
+
+async def _run(args: argparse.Namespace) -> int:
+    root_s = args.project_root.strip()
+    if not root_s:
+        print("skillforge index: --project-root is required.", file=sys.stderr)
+        return 2
+    root = Path(root_s).expanduser().resolve()
+    db_path = resolve_orchestrator_db(str(root))
+    db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    con = init_db(db_path)
+    try:
+        if args.stats_only:
+            print(json.dumps({"db": str(db_path), **project_index_stats(con)}, indent=2))
+            return 0
+
+        router, _ = await asyncio.to_thread(
+            build_router_and_skills,
+            log=not args.quiet,
+            log_prefix="[skillforge-index]",
+        )
+        stats = await asyncio.to_thread(
+            index_project,
+            con,
+            root,
+            router.embed_model,
+            reset=args.reset,
+        )
+        print(
+            json.dumps(
+                {"db": str(db_path), "index_state": project_index_stats(con), **stats},
+                indent=2,
+            )
+        )
+        return 0
+    finally:
+        con.close()
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = _parse_args(argv)
+    raise SystemExit(asyncio.run(_run(args)))
+
+
+if __name__ == "__main__":
+    main()