@luizsantiago/spec-guardrails 3.5.0 → 3.6.0

package/README.md

@@ -11,7 +11,7 @@
 | **Solution** | One kit, two deliberate modes: **Process** (Node only) for a flexible spec-driven workflow; **Brakes** (Node + Python) for the **full product** — structural gates that exit non-zero when paperwork or evidence is missing. You approve specs/tasks in both. |
 | **Result** | Traceable `.specs/` memory, fewer fake finishes, cheaper turns (~70% less skill text on planning). Choose Process for light ceremony; add Python when you want the [Guarantees matrix](#guarantees-matrix) enforced automatically. |
 
-npm: [`@luizsantiago/spec-guardrails`](https://www.npmjs.com/package/@luizsantiago/spec-guardrails) **3.5.x**
+npm: [`@luizsantiago/spec-guardrails`](https://www.npmjs.com/package/@luizsantiago/spec-guardrails) **3.6.x**
 
 ---
 
@@ -217,6 +217,7 @@ npx @luizsantiago/spec-guardrails install
 
 | Version | What you gain |
 | --- | --- |
+| **3.6.x** | Hybrid retrieval (`memory-retrieve`); chunk FTS; optional semantic embed |
 | **3.5.x** | Solution exploration (`solution-explore`) — compare candidates from approved spec |
 | **3.4.x** | Contextual guards (`context-guard`); FTS memory search (`memory-search`) |
 | **3.3.x** | Intent/effect policy — `check-path --op read|write|delete` and `effects` config block |

package/index.js

@@ -103,12 +103,16 @@ Commands:
   execution-policy record-retry <task>  Increment retry counter for a task id (blocks at limit)
   execution-policy record-run        Increment agent-run counter (blocks at budget)
   memory-index rebuild               Rebuild SQLite memory index from .specs/ artifacts
+  memory-index embed [--force]       Optional semantic embeddings (requires config + provider)
   memory-query --from <id>           Bounded context package from the knowledge graph
     [--depth N]                      Traversal depth (default 2)
     [--json]                         Machine-readable output
-  memory-search <query>              Full-text search over the memory index (FTS5)
+  memory-search <query>              Full-text search over indexed artifact chunks (FTS5)
     [--limit N]                      Max results (default 10)
     [--json]                         Machine-readable output
+  memory-retrieve "<query>"          Hybrid retrieval (FTS + graph + optional semantic)
+    [--mode fts|hybrid|semantic]     Strategy (default: hybrid)
+    [--json]                         Machine-readable output
   context-guard status               Execute readiness from STATE + tasks.md
     [--json]                         Machine-readable output
   context-guard check-edit <path>    Contextual guard before editing a file

package/lib/constants.js

@@ -112,6 +112,9 @@ export const SCRIPT_ASSETS = [
   { file: "memory_index.py", remotePath: "scripts/memory_index.py" },
   { file: "memory_query.py", remotePath: "scripts/memory_query.py" },
   { file: "memory_search.py", remotePath: "scripts/memory_search.py" },
+  { file: "memory_retrieve.py", remotePath: "scripts/memory_retrieve.py" },
+  { file: "_memory_config.py", remotePath: "scripts/_memory_config.py" },
+  { file: "_memory_embed.py", remotePath: "scripts/_memory_embed.py" },
 ];
 
 /** @type {{ file: string, remotePath: string }[]} */

package/lib/gates.js

@@ -44,6 +44,7 @@ const AUX_SCRIPTS = {
   "memory-index": "memory_index.py",
   "memory-query": "memory_query.py",
   "memory-search": "memory_search.py",
+  "memory-retrieve": "memory_retrieve.py",
 };
 
 const GUARDRAILS_SCRIPTS = { ...GATE_SCRIPTS, ...AUX_SCRIPTS };

package/lib/memory-index.js

@@ -16,6 +16,42 @@ export async function rebuildMemoryIndex(options = {}) {
   return runGuardrailsScript("memory-index", ["rebuild"], { cwd });
 }
 
+/**
+ * Build optional semantic embeddings for indexed chunks.
+ *
+ * @param {{ force?: boolean, json?: boolean, cwd?: string }} [options]
+ * @returns {Promise<number>} exit code
+ */
+export async function embedMemoryIndex(options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const args = ["embed"];
+  if (options.force) {
+    args.push("--force");
+  }
+  if (options.json) {
+    args.push("--json");
+  }
+  return runGuardrailsScript("memory-index", args, { cwd });
+}
+
+/**
+ * Hybrid retrieval over indexed artifacts.
+ *
+ * @param {{ query: string, mode?: string, json?: boolean, cwd?: string }} options
+ * @returns {Promise<number>} exit code
+ */
+export async function retrieveMemory(options) {
+  const cwd = options.cwd ?? process.cwd();
+  const args = [options.query];
+  if (options.mode) {
+    args.push("--mode", options.mode);
+  }
+  if (options.json) {
+    args.push("--json");
+  }
+  return runGuardrailsScript("memory-retrieve", args, { cwd });
+}
+
 /**
  * Query the knowledge graph for a bounded context package.
  *

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@luizsantiago/spec-guardrails",
-  "version": "3.5.0",
+  "version": "3.6.0",
   "description": "Keep AI coding agents honest — specify the work, prove each step, verify independently. Process mode (Node) for flexibility; Brakes mode (Node + Python) for structural gates and a Guarantees matrix. Progressive loading, independent verify — any AI agent.",
   "type": "module",
   "bin": {

package/scripts/_memory_config.py

@@ -0,0 +1,97 @@
+"""Shared memory retrieval config parsed from `.specs/config.yaml`."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+SPECS_DIR = Path(".specs")
+FEATURES_DIR = SPECS_DIR / "features"
+CONFIG_PATH = SPECS_DIR / "config.yaml"
+MEMORY_DIR = SPECS_DIR / "memory"
+DB_PATH = MEMORY_DIR / "memory.db"
+
+DEFAULT_RETRIEVAL = {
+    "semantic": False,
+    "provider": "none",
+    "model": "text-embedding-3-small",
+    "max_chunks": 20,
+    "fts_weight": 0.6,
+    "semantic_weight": 0.4,
+    "graph_depth": 1,
+}
+
+BOOL = {"true", "false", "yes", "no", "on", "off"}
+
+
+def _parse_scalar(raw: str):
+    value = raw.strip().strip("'\"")
+    lower = value.lower()
+    if lower in {"true", "yes", "on"}:
+        return True
+    if lower in {"false", "no", "off"}:
+        return False
+    if re.fullmatch(r"-?\d+", value):
+        return int(value)
+    if re.fullmatch(r"-?\d+\.\d+", value):
+        return float(value)
+    return value
+
+
+def load_memory_retrieval_config() -> dict:
+    """Return memory.retrieval settings with defaults."""
+
+    config = dict(DEFAULT_RETRIEVAL)
+    if not CONFIG_PATH.is_file():
+        return config
+
+    lines = CONFIG_PATH.read_text(encoding="utf-8").splitlines()
+    in_memory = False
+    in_retrieval = False
+    memory_indent = 0
+    retrieval_indent = 0
+
+    for line in lines:
+        if not line.strip() or line.lstrip().startswith("#"):
+            continue
+
+        indent = len(line) - len(line.lstrip())
+        stripped = line.strip()
+
+        if stripped == "memory:":
+            in_memory = True
+            in_retrieval = False
+            memory_indent = indent
+            continue
+
+        if not in_memory:
+            continue
+
+        if indent <= memory_indent and stripped != "memory:":
+            in_memory = False
+            in_retrieval = False
+            continue
+
+        if stripped == "retrieval:":
+            in_retrieval = True
+            retrieval_indent = indent
+            continue
+
+        if in_retrieval and indent <= retrieval_indent and not stripped.startswith("retrieval:"):
+            in_retrieval = False
+
+        if not in_retrieval:
+            continue
+
+        match = re.match(r"^([a-z_]+):\s*(.*)$", stripped)
+        if not match:
+            continue
+
+        key, raw_value = match.group(1), match.group(2)
+        if key not in config:
+            continue
+        if raw_value == "":
+            continue
+        config[key] = _parse_scalar(raw_value)
+
+    return config

package/scripts/_memory_embed.py

@@ -0,0 +1,116 @@
+"""Optional chunk embeddings for semantic retrieval."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import struct
+import urllib.error
+import urllib.request
+from typing import Callable
+
+HASH_DIMS = 64
+OPENAI_DIMS = 256
+
+
+def pack_vector(values: list[float]) -> bytes:
+    return struct.pack(f"{len(values)}f", *values)
+
+
+def unpack_vector(blob: bytes) -> list[float]:
+    count = len(blob) // 4
+    return list(struct.unpack(f"{count}f", blob))
+
+
+def cosine_similarity(a: list[float], b: list[float]) -> float:
+    if len(a) != len(b) or not a:
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    norm_a = sum(x * x for x in a) ** 0.5
+    norm_b = sum(y * y for y in b) ** 0.5
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+def hash_embed(text: str, dims: int = HASH_DIMS) -> list[float]:
+    digest = hashlib.sha256(text.encode("utf-8")).digest()
+    values: list[float] = []
+    counter = 0
+    while len(values) < dims:
+        block = hashlib.sha256(digest + counter.to_bytes(2, "big")).digest()
+        for i in range(0, len(block), 4):
+            if len(values) >= dims:
+                break
+            chunk = block[i : i + 4]
+            if len(chunk) < 4:
+                break
+            integer = int.from_bytes(chunk, "big", signed=False)
+            values.append((integer / 2**31) - 1.0)
+        counter += 1
+    return values
+
+
+def openai_embed(text: str, model: str) -> list[float]:
+    api_key = os.environ.get("OPENAI_API_KEY", "").strip()
+    if not api_key:
+        raise RuntimeError("OPENAI_API_KEY is required for provider openai")
+
+    payload = json.dumps({"input": text, "model": model}).encode("utf-8")
+    request = urllib.request.Request(
+        "https://api.openai.com/v1/embeddings",
+        data=payload,
+        headers={
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        },
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=60) as response:
+            body = json.loads(response.read().decode("utf-8"))
+    except urllib.error.HTTPError as err:
+        detail = err.read().decode("utf-8", errors="replace")
+        raise RuntimeError(f"OpenAI embeddings failed: {detail}") from err
+
+    vector = body["data"][0]["embedding"]
+    if len(vector) > OPENAI_DIMS:
+        vector = vector[:OPENAI_DIMS]
+    return [float(value) for value in vector]
+
+
+def ollama_embed(text: str, model: str) -> list[float]:
+    host = os.environ.get("OLLAMA_HOST", "http://127.0.0.1:11434").rstrip("/")
+    payload = json.dumps({"model": model, "prompt": text}).encode("utf-8")
+    request = urllib.request.Request(
+        f"{host}/api/embeddings",
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=120) as response:
+            body = json.loads(response.read().decode("utf-8"))
+    except urllib.error.URLError as err:
+        raise RuntimeError(f"Ollama embeddings failed: {err}") from err
+
+    vector = body.get("embedding") or []
+    if not vector:
+        raise RuntimeError("Ollama returned an empty embedding vector")
+    if len(vector) > OPENAI_DIMS:
+        vector = vector[:OPENAI_DIMS]
+    return [float(value) for value in vector]
+
+
+def get_embed_fn(provider: str, model: str) -> Callable[[str], list[float]]:
+    normalized = provider.strip().lower()
+    if normalized in {"none", ""}:
+        raise RuntimeError("semantic retrieval is disabled (provider none)")
+    if normalized == "hash":
+        return lambda text: hash_embed(text)
+    if normalized == "openai":
+        return lambda text: openai_embed(text, model)
+    if normalized == "ollama":
+        return lambda text: ollama_embed(text, model)
+    raise RuntimeError(f"Unknown embedding provider: {provider}")

package/scripts/memory_index.py

@@ -5,6 +5,7 @@ Source of truth remains markdown under `.specs/`. The database is a derived inde
 
     python3 memory_index.py rebuild
     python3 memory_index.py rebuild --json
+    python3 memory_index.py embed [--force]
 
 Exit codes: 0 ok, 1 failure, 2 usage error.
 """
@@ -12,6 +13,7 @@ Exit codes: 0 ok, 1 failure, 2 usage error.
 from __future__ import annotations
 
 import argparse
+import hashlib
 import json
 import re
 import sqlite3
@@ -20,12 +22,10 @@ from datetime import datetime, timezone
 from pathlib import Path
 
 from _common import EXIT_FAILED, EXIT_OK, EXIT_USAGE, requirement_ids
+from _memory_config import DB_PATH, FEATURES_DIR, MEMORY_DIR, SPECS_DIR, load_memory_retrieval_config
+from _memory_embed import get_embed_fn, pack_vector
 
 GATE = "memory-index"
-SPECS_DIR = Path(".specs")
-FEATURES_DIR = SPECS_DIR / "features"
-MEMORY_DIR = SPECS_DIR / "memory"
-DB_PATH = MEMORY_DIR / "memory.db"
 
 TASK_HEADING = re.compile(
     r"^#{2,6}\s*(?P<id>T\d{1,6})\s*[:\-–]?\s*(?P<title>.*)$",
@@ -35,7 +35,11 @@ TASK_FIELD = re.compile(
     r"^\s*[-*]?\s*\*{0,2}(?P<key>[A-Za-z][A-Za-z ]+?)\*{0,2}\s*:\s*(?P<value>.+?)\s*$",
     re.MULTILINE,
 )
+REQ_HEADING = re.compile(r"^#{2,6}\s*(?P<id>REQ-\d+)\b.*$", re.MULTILINE | re.IGNORECASE)
 REQ_REF = re.compile(r"\b[A-Z][A-Z0-9]{1,9}-\d{2,4}\b")
+SECTION_HEADING = re.compile(r"^##\s+(.+)$", re.MULTILINE)
+
+LESSON_STATUSES = {"approved", "graduated", "confirmed"}
 
 
 def fail(message: str, code: int = EXIT_FAILED) -> int:
@@ -53,6 +57,10 @@ def utc_now() -> str:
     return datetime.now(timezone.utc).replace(microsecond=0).isoformat()
 
 
+def text_hash(text: str) -> str:
+    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
 def init_schema(conn: sqlite3.Connection) -> None:
     conn.executescript(
         """
@@ -76,6 +84,31 @@ def init_schema(conn: sqlite3.Connection) -> None:
             source_path,
             tokenize='porter'
         );
+        CREATE TABLE IF NOT EXISTS chunks (
+            id TEXT PRIMARY KEY,
+            entity_id TEXT,
+            kind TEXT NOT NULL,
+            source_path TEXT,
+            text TEXT NOT NULL,
+            text_hash TEXT NOT NULL,
+            updated_at TEXT NOT NULL
+        );
+        CREATE VIRTUAL TABLE IF NOT EXISTS chunk_fts USING fts5(
+            id UNINDEXED,
+            kind,
+            source_path,
+            text,
+            tokenize='porter'
+        );
+        CREATE TABLE IF NOT EXISTS embeddings (
+            chunk_id TEXT PRIMARY KEY,
+            model TEXT NOT NULL,
+            provider TEXT NOT NULL,
+            dims INTEGER NOT NULL,
+            text_hash TEXT NOT NULL,
+            vector BLOB NOT NULL,
+            updated_at TEXT NOT NULL
+        );
         """
     )
 
@@ -107,6 +140,41 @@ def upsert_entity(
     )
 
 
+def upsert_chunk(
+    conn: sqlite3.Connection,
+    chunk_id: str,
+    entity_id: str | None,
+    kind: str,
+    source_path: str,
+    text: str,
+    now: str,
+) -> None:
+    normalized = text.strip()
+    if len(normalized) < 8:
+        return
+
+    digest = text_hash(normalized)
+    conn.execute(
+        """
+        INSERT INTO chunks (id, entity_id, kind, source_path, text, text_hash, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?)
+        ON CONFLICT(id) DO UPDATE SET
+            entity_id=excluded.entity_id,
+            kind=excluded.kind,
+            source_path=excluded.source_path,
+            text=excluded.text,
+            text_hash=excluded.text_hash,
+            updated_at=excluded.updated_at
+        """,
+        (chunk_id, entity_id, kind, source_path, normalized, digest, now),
+    )
+    conn.execute("DELETE FROM chunk_fts WHERE id = ?", (chunk_id,))
+    conn.execute(
+        "INSERT INTO chunk_fts (id, kind, source_path, text) VALUES (?, ?, ?, ?)",
+        (chunk_id, kind, source_path, normalized),
+    )
+
+
 def upsert_relation(conn: sqlite3.Connection, from_id: str, to_id: str, kind: str) -> None:
     conn.execute(
         """
@@ -170,27 +238,94 @@ def task_file_paths(tasks_text: str) -> dict[str, set[str]]:
     return mapping
 
 
-def index_lessons(conn: sqlite3.Connection, now: str) -> int:
+def chunk_spec_requirements(
+    conn: sqlite3.Connection,
+    feature_id: str,
+    spec_path: Path,
+    spec_text: str,
+    now: str,
+) -> int:
+    count = 0
+    matches = list(REQ_HEADING.finditer(spec_text))
+    for index, match in enumerate(matches):
+        req_id = match.group("id").upper()
+        start = match.end()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(spec_text)
+        body = spec_text[start:end].strip()
+        if not body:
+            continue
+        chunk_id = f"chunk:{feature_id}:requirement:{req_id}"
+        upsert_chunk(conn, chunk_id, req_id, "requirement", str(spec_path), body, now)
+        count += 1
+    return count
+
+
+def chunk_markdown_sections(
+    conn: sqlite3.Connection,
+    feature_id: str,
+    path: Path,
+    text: str,
+    kind: str,
+    entity_id: str | None,
+    now: str,
+) -> int:
+    count = 0
+    matches = list(SECTION_HEADING.finditer(text))
+    if not matches:
+        chunk_id = f"chunk:{feature_id}:{kind}:body"
+        upsert_chunk(conn, chunk_id, entity_id, kind, str(path), text, now)
+        return 1
+
+    for index, match in enumerate(matches):
+        title = match.group(1).strip()
+        slug = re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") or "section"
+        start = match.end()
+        end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
+        body = f"## {title}\n{text[start:end]}".strip()
+        chunk_id = f"chunk:{feature_id}:{kind}:{slug}"
+        upsert_chunk(conn, chunk_id, entity_id, kind, str(path), body, now)
+        count += 1
+    return count
+
+
+def index_lessons(conn: sqlite3.Connection, now: str) -> tuple[int, int]:
     lessons_path = SPECS_DIR / "lessons.json"
     if not lessons_path.is_file():
-        return 0
+        return 0, 0
 
     payload = json.loads(lessons_path.read_text(encoding="utf-8"))
-    count = 0
+    entity_count = 0
+    chunk_count = 0
     for lesson in payload.get("lessons") or []:
         lesson_id = str(lesson.get("id") or "").strip()
         if not lesson_id:
             continue
+        status = str(lesson.get("status") or "").strip().lower()
+        title = str(lesson.get("title") or lesson_id)
         upsert_entity(
             conn,
             lesson_id,
             "lesson",
-            str(lesson.get("title") or lesson_id),
+            title,
             str(lesson.get("source") or ""),
             now,
         )
-        count += 1
-    return count
+        entity_count += 1
+
+        if status not in LESSON_STATUSES:
+            continue
+
+        body_parts = [title]
+        for key in ("summary", "detail", "evidence", "recommendation"):
+            value = lesson.get(key)
+            if value:
+                body_parts.append(str(value))
+        body = "\n".join(body_parts).strip()
+        chunk_id = f"chunk:lesson:{lesson_id}"
+        upsert_chunk(conn, chunk_id, lesson_id, "lesson", str(lessons_path), body, now)
+        chunk_count += 1
+
+    return entity_count, chunk_count
 
 
 def rebuild(json_output: bool = False) -> int:
@@ -206,9 +341,13 @@ def rebuild(json_output: bool = False) -> int:
         conn.execute("DELETE FROM relations")
         conn.execute("DELETE FROM entities")
         conn.execute("DELETE FROM entity_fts")
+        conn.execute("DELETE FROM chunks")
+        conn.execute("DELETE FROM chunk_fts")
+        conn.execute("DELETE FROM embeddings")
 
         entity_count = 0
         relation_count = 0
+        chunk_count = 0
 
         if FEATURES_DIR.is_dir():
             for feature_dir in sorted(FEATURES_DIR.iterdir()):
@@ -241,6 +380,35 @@ def rebuild(json_output: bool = False) -> int:
                         entity_count += 1
                         upsert_relation(conn, feature_id, req_id, "contains")
                         relation_count += 1
+                    chunk_count += chunk_spec_requirements(
+                        conn, feature_id, spec_path, spec_text, now
+                    )
+
+                validation_path = feature_dir / "validation.md"
+                if validation_path.is_file():
+                    validation_text = validation_path.read_text(encoding="utf-8")
+                    chunk_count += chunk_markdown_sections(
+                        conn,
+                        feature_id,
+                        validation_path,
+                        validation_text,
+                        "validation",
+                        feature_id,
+                        now,
+                    )
+
+                exploration_path = feature_dir / "exploration.md"
+                if exploration_path.is_file():
+                    exploration_text = exploration_path.read_text(encoding="utf-8")
+                    chunk_count += chunk_markdown_sections(
+                        conn,
+                        feature_id,
+                        exploration_path,
+                        exploration_text,
+                        "exploration",
+                        feature_id,
+                        now,
+                    )
 
                 tasks_path = feature_dir / "tasks.md"
                 if tasks_path.is_file():
@@ -280,13 +448,16 @@ def rebuild(json_output: bool = False) -> int:
                             upsert_relation(conn, task_id, file_entity, "touches")
                             relation_count += 1
 
-        entity_count += index_lessons(conn, now)
+        lesson_entities, lesson_chunks = index_lessons(conn, now)
+        entity_count += lesson_entities
+        chunk_count += lesson_chunks
         conn.commit()
 
         summary = {
             "database": str(DB_PATH),
             "entities": entity_count,
             "relations": relation_count,
+            "chunks": chunk_count,
             "updated_at": now,
         }
 
@@ -294,13 +465,86 @@ def rebuild(json_output: bool = False) -> int:
             print(json.dumps(summary, indent=2))
         else:
             return ok(
-                f"indexed {entity_count} entities and {relation_count} relations -> {DB_PATH}"
+                f"indexed {entity_count} entities, {relation_count} relations, "
+                f"{chunk_count} chunks -> {DB_PATH}"
             )
         return EXIT_OK
     finally:
         conn.close()
 
 
+def embed_chunks(force: bool = False, json_output: bool = False) -> int:
+    config = load_memory_retrieval_config()
+    if not config.get("semantic"):
+        print(f"[{GATE}] semantic retrieval disabled — embed skipped (set memory.retrieval.semantic: true)")
+        return EXIT_OK
+
+    provider = str(config.get("provider") or "none")
+    model = str(config.get("model") or "text-embedding-3-small")
+
+    if not DB_PATH.is_file():
+        return fail(f"{DB_PATH} not found — run `memory-index rebuild` first")
+
+    try:
+        embed_fn = get_embed_fn(provider, model)
+    except RuntimeError as err:
+        return fail(str(err))
+
+    now = utc_now()
+    conn = sqlite3.connect(DB_PATH)
+    embedded = 0
+    skipped = 0
+    try:
+        init_schema(conn)
+        rows = conn.execute(
+            "SELECT id, text, text_hash FROM chunks ORDER BY id"
+        ).fetchall()
+
+        for chunk_id, text, digest in rows:
+            if not force:
+                existing = conn.execute(
+                    """
+                    SELECT text_hash FROM embeddings
+                    WHERE chunk_id = ? AND provider = ? AND model = ?
+                    """,
+                    (chunk_id, provider, model),
+                ).fetchone()
+                if existing and existing[0] == digest:
+                    skipped += 1
+                    continue
+
+            vector = embed_fn(text)
+            conn.execute(
+                """
+                INSERT INTO embeddings (chunk_id, model, provider, dims, text_hash, vector, updated_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(chunk_id) DO UPDATE SET
+                    model=excluded.model,
+                    provider=excluded.provider,
+                    dims=excluded.dims,
+                    text_hash=excluded.text_hash,
+                    vector=excluded.vector,
+                    updated_at=excluded.updated_at
+                """,
+                (chunk_id, model, provider, len(vector), digest, pack_vector(vector), now),
+            )
+            embedded += 1
+
+        conn.commit()
+    finally:
+        conn.close()
+
+    summary = {"embedded": embedded, "skipped": skipped, "provider": provider, "model": model}
+    if json_output:
+        print(json.dumps(summary, indent=2))
+    else:
+        print(
+            f"[{GATE}] embedded {embedded} chunk(s), skipped {skipped} "
+            f"({provider}/{model})"
+        )
+    return EXIT_OK
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(description="Rebuild SQLite memory index from .specs/")
     sub = parser.add_subparsers(dest="command")
@@ -309,6 +553,11 @@ def build_parser() -> argparse.ArgumentParser:
     rebuild_cmd.add_argument("--json", action="store_true")
     rebuild_cmd.set_defaults(func=lambda args: rebuild(json_output=args.json))
 
+    embed_cmd = sub.add_parser("embed", help="build optional semantic embeddings for chunks")
+    embed_cmd.add_argument("--force", action="store_true")
+    embed_cmd.add_argument("--json", action="store_true")
+    embed_cmd.set_defaults(func=lambda args: embed_chunks(force=args.force, json_output=args.json))
+
     return parser
 
 

package/scripts/memory_retrieve.py

@@ -0,0 +1,289 @@
+#!/usr/bin/env python3
+"""Hybrid memory retrieval: FTS chunks + graph expansion (+ optional semantic).
+
+    python3 memory_retrieve.py "silent session expiry"
+    python3 memory_retrieve.py "auth failure" --mode hybrid --json
+
+Exit codes: 0 ok, 1 failure, 2 usage error.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sqlite3
+import sys
+from collections import deque
+
+from _common import EXIT_FAILED, EXIT_OK, EXIT_USAGE
+from _memory_config import DB_PATH, load_memory_retrieval_config
+from _memory_embed import cosine_similarity, get_embed_fn, unpack_vector
+from memory_search import search, search_chunks
+
+GATE = "memory-retrieve"
+
+
+def fail(message: str, code: int = EXIT_FAILED) -> int:
+    print(f"[{GATE}] FAIL - {DB_PATH}")
+    print(f"  error   {message}")
+    return code
+
+
+def fetch_neighbors(conn: sqlite3.Connection, entity_id: str) -> list[dict]:
+    rows = conn.execute(
+        """
+        SELECT r.kind, e.id, e.kind, e.label, e.source_path
+        FROM relations r
+        JOIN entities e ON e.id = r.to_id
+        WHERE r.from_id = ?
+        UNION
+        SELECT r.kind, e.id, e.kind, e.label, e.source_path
+        FROM relations r
+        JOIN entities e ON e.id = r.from_id
+        WHERE r.to_id = ?
+        """,
+        (entity_id, entity_id),
+    ).fetchall()
+
+    return [
+        {
+            "relation": row[0],
+            "entity": {
+                "id": row[1],
+                "kind": row[2],
+                "label": row[3],
+                "source_path": row[4],
+            },
+        }
+        for row in rows
+    ]
+
+
+def expand_graph(entity_ids: list[str], depth: int) -> dict:
+    if not DB_PATH.is_file() or depth <= 0 or not entity_ids:
+        return {"entities": [], "relations": []}
+
+    conn = sqlite3.connect(DB_PATH)
+    visited: set[str] = set()
+    entities: dict[str, dict] = {}
+    relations: list[dict] = []
+    queue: deque[tuple[str, int]] = deque()
+
+    try:
+        for entity_id in entity_ids:
+            if entity_id and entity_id not in visited:
+                visited.add(entity_id)
+                queue.append((entity_id, 0))
+
+        while queue:
+            current_id, current_depth = queue.popleft()
+            if current_depth >= depth:
+                continue
+
+            row = conn.execute(
+                "SELECT id, kind, label, source_path, updated_at FROM entities WHERE id = ?",
+                (current_id,),
+            ).fetchone()
+            if row:
+                entities[current_id] = {
+                    "id": row[0],
+                    "kind": row[1],
+                    "label": row[2],
+                    "source_path": row[3],
+                    "updated_at": row[4],
+                }
+
+            for neighbor in fetch_neighbors(conn, current_id):
+                target = neighbor["entity"]
+                target_id = target["id"]
+                relations.append(
+                    {"from": current_id, "to": target_id, "kind": neighbor["relation"]}
+                )
+                if target_id not in entities:
+                    entities[target_id] = target
+                if target_id not in visited:
+                    visited.add(target_id)
+                    queue.append((target_id, current_depth + 1))
+    finally:
+        conn.close()
+
+    return {"entities": list(entities.values()), "relations": relations}
+
+
+def semantic_search(query: str, limit: int, provider: str, model: str) -> list[dict]:
+    embed_fn = get_embed_fn(provider, model)
+    query_vector = embed_fn(query)
+
+    conn = sqlite3.connect(DB_PATH)
+    try:
+        rows = conn.execute(
+            """
+            SELECT c.id, c.entity_id, c.kind, c.source_path, c.text, e.vector
+            FROM embeddings e
+            JOIN chunks c ON c.id = e.chunk_id
+            WHERE e.provider = ? AND e.model = ?
+            """,
+            (provider, model),
+        ).fetchall()
+    finally:
+        conn.close()
+
+    scored: list[tuple[float, dict]] = []
+    for chunk_id, entity_id, kind, source_path, text, blob in rows:
+        vector = unpack_vector(blob)
+        score = cosine_similarity(query_vector, vector)
+        if score <= 0:
+            continue
+        snippet = text if len(text) <= 240 else text[:237] + "..."
+        scored.append(
+            (
+                score,
+                {
+                    "chunk_id": chunk_id,
+                    "entity_id": entity_id,
+                    "kind": kind,
+                    "source_path": source_path,
+                    "snippet": snippet,
+                    "score": round(score, 4),
+                    "source": "semantic",
+                },
+            )
+        )
+
+    scored.sort(key=lambda item: item[0], reverse=True)
+    return [item[1] for item in scored[:limit]]
+
+
+def merge_hybrid(
+    fts_results: list[dict],
+    semantic_results: list[dict],
+    fts_weight: float,
+    semantic_weight: float,
+    max_chunks: int,
+) -> list[dict]:
+    merged: dict[str, dict] = {}
+
+    for rank, item in enumerate(fts_results):
+        key = item.get("chunk_id") or item.get("entity_id") or f"fts-{rank}"
+        score = fts_weight * (1.0 / (rank + 1))
+        merged[key] = {**item, "score": round(score, 4), "sources": [item.get("source", "fts")]}
+
+    for rank, item in enumerate(semantic_results):
+        key = item.get("chunk_id") or item.get("entity_id") or f"sem-{rank}"
+        bonus = semantic_weight * (item.get("score") or (1.0 / (rank + 1)))
+        if key in merged:
+            merged[key]["score"] = round(merged[key]["score"] + bonus, 4)
+            merged[key]["sources"].append("semantic")
+        else:
+            merged[key] = {**item, "score": round(bonus, 4), "sources": ["semantic"]}
+
+    ordered = sorted(merged.values(), key=lambda item: item.get("score", 0), reverse=True)
+    return ordered[:max_chunks]
+
+
+def retrieve(query: str, mode: str) -> dict:
+    if not DB_PATH.is_file():
+        raise FileNotFoundError(
+            f"{DB_PATH} not found — run `memory-index rebuild` after install"
+        )
+
+    config = load_memory_retrieval_config()
+    max_chunks = int(config.get("max_chunks") or 20)
+    graph_depth = int(config.get("graph_depth") or 1)
+    fts_weight = float(config.get("fts_weight") or 0.6)
+    semantic_weight = float(config.get("semantic_weight") or 0.4)
+    semantic_enabled = bool(config.get("semantic"))
+    provider = str(config.get("provider") or "none")
+    model = str(config.get("model") or "text-embedding-3-small")
+
+    fts_results = search_chunks(query, max_chunks)
+    if not fts_results:
+        fts_results = search(query, max_chunks)
+
+    semantic_results: list[dict] = []
+    if mode in {"hybrid", "semantic"} and semantic_enabled and provider not in {"", "none"}:
+        try:
+            semantic_results = semantic_search(query, max_chunks, provider, model)
+        except RuntimeError:
+            if mode == "semantic":
+                raise
+            semantic_results = []
+
+    if mode == "semantic" and semantic_results:
+        results = semantic_results[:max_chunks]
+    elif mode == "hybrid" and semantic_results:
+        results = merge_hybrid(
+            fts_results, semantic_results, fts_weight, semantic_weight, max_chunks
+        )
+    else:
+        results = fts_results[:max_chunks]
+
+    seed_ids = []
+    for item in results:
+        if item.get("entity_id"):
+            seed_ids.append(item["entity_id"])
+    graph = expand_graph(seed_ids, graph_depth if mode != "fts" else 0)
+
+    return {
+        "query": query,
+        "mode": mode,
+        "semantic_enabled": semantic_enabled,
+        "results": results,
+        "graph_expansion": {
+            "entities": len(graph["entities"]),
+            "relations": len(graph["relations"]),
+            "entities_detail": graph["entities"],
+            "relations_detail": graph["relations"],
+        },
+    }
+
+
+def cmd_retrieve(args: argparse.Namespace) -> int:
+    try:
+        package = retrieve(args.query, args.mode)
+    except FileNotFoundError as err:
+        return fail(str(err))
+    except RuntimeError as err:
+        return fail(str(err))
+
+    if args.json:
+        print(json.dumps(package, indent=2))
+    else:
+        print(
+            f"[{GATE}] {len(package['results'])} result(s) for {args.query!r} "
+            f"(mode={package['mode']})"
+        )
+        for item in package["results"]:
+            sources = ",".join(item.get("sources") or [item.get("source", "fts")])
+            label = item.get("snippet") or item.get("entity_id")
+            print(f"  {item.get('chunk_id') or item.get('entity_id')} [{sources}]: {label}")
+        graph = package["graph_expansion"]
+        print(
+            f"  graph: {graph['entities']} entities, {graph['relations']} relations"
+        )
+
+    return EXIT_OK
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Hybrid retrieval over the memory index")
+    parser.add_argument("query", help="natural language or keyword query")
+    parser.add_argument(
+        "--mode",
+        choices=["fts", "hybrid", "semantic"],
+        default="hybrid",
+        help="retrieval strategy (default: hybrid)",
+    )
+    parser.add_argument("--json", action="store_true")
+    parser.set_defaults(func=cmd_retrieve)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/memory_search.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Full-text search over the SQLite memory index (entity_fts).
+"""Full-text search over the SQLite memory index (chunks + entity metadata).
 
     python3 memory_search.py "authentication route"
     python3 memory_search.py "REQ-001" --limit 5 --json
@@ -17,9 +17,9 @@ import sys
 from pathlib import Path
 
 from _common import EXIT_FAILED, EXIT_OK, EXIT_USAGE
+from _memory_config import DB_PATH
 
 GATE = "memory-search"
-DB_PATH = Path(".specs/memory/memory.db")
 DEFAULT_LIMIT = 10
 MAX_LIMIT = 50
 
@@ -31,26 +31,56 @@ def fail(message: str, code: int = EXIT_FAILED) -> int:
 
 
 def sanitize_fts_query(raw: str) -> str:
-    """Return an FTS5-safe query string (token AND chain)."""
-
     tokens = re.findall(r"[A-Za-z0-9_\-]+", raw)
     if not tokens:
         raise ValueError("query must contain at least one searchable token")
     return " AND ".join(f'"{token}"' for token in tokens)
 
 
-def search(query: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
-    if not DB_PATH.is_file():
-        raise FileNotFoundError(
-            f"{DB_PATH} not found — run `memory-index rebuild` after install"
+def search_chunks(query: str, limit: int) -> list[dict]:
+    fts_query = sanitize_fts_query(query)
+    conn = sqlite3.connect(DB_PATH)
+    try:
+        rows = conn.execute(
+            """
+            SELECT c.id, c.entity_id, c.kind, c.source_path, c.text, c.updated_at, f.rank
+            FROM chunk_fts f
+            JOIN chunks c ON c.id = f.id
+            WHERE chunk_fts MATCH ?
+            ORDER BY rank
+            LIMIT ?
+            """,
+            (fts_query, limit),
+        ).fetchall()
+    finally:
+        conn.close()
+
+    results = []
+    for row in rows:
+        text = row[4]
+        snippet = text if len(text) <= 240 else text[:237] + "..."
+        results.append(
+            {
+                "chunk_id": row[0],
+                "entity_id": row[1],
+                "kind": row[2],
+                "source_path": row[3],
+                "snippet": snippet,
+                "updated_at": row[5],
+                "score": float(-row[6]) if row[6] is not None else 0.0,
+                "source": "fts-chunk",
+            }
         )
+    return results
 
+
+def search_entities(query: str, limit: int) -> list[dict]:
     fts_query = sanitize_fts_query(query)
     conn = sqlite3.connect(DB_PATH)
     try:
         rows = conn.execute(
             """
-            SELECT e.id, e.kind, e.label, e.source_path, e.updated_at
+            SELECT e.id, e.kind, e.label, e.source_path, e.updated_at, f.rank
             FROM entity_fts f
             JOIN entities e ON e.id = f.id
             WHERE entity_fts MATCH ?
@@ -64,16 +94,32 @@ def search(query: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
 
     return [
         {
-            "id": row[0],
+            "chunk_id": None,
+            "entity_id": row[0],
             "kind": row[1],
-            "label": row[2],
             "source_path": row[3],
+            "snippet": row[2],
             "updated_at": row[4],
+            "score": float(-row[5]) if row[5] is not None else 0.0,
+            "source": "fts-entity",
         }
         for row in rows
     ]
 
 
+def search(query: str, limit: int = DEFAULT_LIMIT) -> list[dict]:
+    if not DB_PATH.is_file():
+        raise FileNotFoundError(
+            f"{DB_PATH} not found — run `memory-index rebuild` after install"
+        )
+
+    chunk_hits = search_chunks(query, limit)
+    if chunk_hits:
+        return chunk_hits
+
+    return search_entities(query, limit)
+
+
 def cmd_search(args: argparse.Namespace) -> int:
     try:
         results = search(args.query, limit=args.limit)
@@ -89,7 +135,9 @@ def cmd_search(args: argparse.Namespace) -> int:
     else:
         print(f"[{GATE}] {len(results)} match(es) for {args.query!r}")
         for item in results:
-            print(f"  {item['id']} ({item['kind']}): {item['label']}")
+            label = item.get("snippet") or item.get("entity_id")
+            kind = item.get("kind")
+            print(f"  {item.get('chunk_id') or item.get('entity_id')} ({kind}): {label}")
             if item.get("source_path"):
                 print(f"           {item['source_path']}")
 

package/skills/agent-architecture.md

@@ -52,7 +52,8 @@ Structural gates run **before** owner review, so they cannot drift when the mode
 | Before claiming feature complete | `npx @luizsantiago/spec-guardrails context-guard check-complete [feature]` |
 | Solution exploration (explicit) | `npx @luizsantiago/spec-guardrails solution-explore init <feature> --candidates A,B` |
 | Before exploration decision | `npx @luizsantiago/spec-guardrails solution-explore validate [feature]` |
-| Retrieve related context | `npx @luizsantiago/spec-guardrails memory-search "<terms>"` |
+| Retrieve related context | `npx @luizsantiago/spec-guardrails memory-retrieve "<query>"` |
+| Rebuild / embed memory index | `npx @luizsantiago/spec-guardrails memory-index rebuild` · `memory-index embed` |
 | On gate retry (Execute playbook) | `npx @luizsantiago/spec-guardrails execution-policy record-retry Tn` |
 | On each commit | `python3 .specs/guardrails/scripts/check_commit.py --message "<message>"` |
 | Before declaring a feature done | `python3 .specs/guardrails/scripts/validate_state.py [feature]` |

package/skills/references/memory.md

@@ -93,6 +93,38 @@ Lessons are owned by `lessons.py`, not by hand. See `lessons.md`.
 - Lazy artifacts: never scaffold empty files to look organized.
 - `STATE.md` has exactly one "Next Step" item. A list of five is not a handoff.
 
+## Retrieval ladder
+
+Rebuild the derived index after meaningful `.specs/` changes:
+
+```bash
+npx @luizsantiago/spec-guardrails memory-index rebuild
+```
+
+| Need | Command |
+| --- | --- |
+| Exact entity + neighbors | `memory-query --from T1 --depth 2` |
+| Keyword search in artifact bodies | `memory-search "oauth session"` |
+| Ranked hybrid package (FTS + graph + optional semantic) | `memory-retrieve "silent session expiry"` |
+
+Optional semantic retrieval stays **off** by default in `.specs/config.yaml`. Enable only when FTS + graph are not enough:
+
+```yaml
+memory:
+  retrieval:
+    semantic: true
+    provider: openai   # or ollama | hash (local/testing)
+    model: text-embedding-3-small
+```
+
+Then embed after rebuild:
+
+```bash
+npx @luizsantiago/spec-guardrails memory-index embed
+```
+
+Markdown under `.specs/` remains the source of truth; the SQLite index is rebuildable.
+
 ## Next
 
 `git-handoff.md` — what to stage, how to word the commit, and why no auto-push.

package/templates/config.yaml.example

@@ -49,6 +49,17 @@ effects:
   warn_read: []
   warn_delete: []
 
+# Memory retrieval (optional — hybrid FTS + graph + semantic)
+memory:
+  retrieval:
+    semantic: false
+    provider: none
+    model: text-embedding-3-small
+    max_chunks: 20
+    fts_weight: 0.6
+    semantic_weight: 0.4
+    graph_depth: 1
+
 # Project-specific overrides (appended on top of preset + rules above):
 # overrides:
 #   rules: