@a13xu/lucid 1.4.0 → 1.9.0

package/README.md

@@ -1,8 +1,8 @@
 # @a13xu/lucid
 
-Memory, code indexing, and validation for Claude Code agents — backed by **SQLite + FTS5**.
+Token-efficient memory, code indexing, and validation for Claude Code agents — backed by **SQLite + FTS5**.
 
-Stores a persistent knowledge graph (entities, relations, observations), indexes source files as compressed binary with change detection, and validates code for LLM drift patterns.
+Stores a persistent knowledge graph (entities, relations, observations), indexes source files as compressed binary with change detection, retrieves minimal relevant context via TF-IDF or Qdrant, and validates code for LLM drift patterns.
 
 ## Install
 
@@ -35,13 +35,15 @@ Default DB path: `~/.claude/memory.db`
 ## Quick start
 
 ```
-1. "Index this project" → init_project() → scans CLAUDE.md, package.json, src/
-2. Write code          → sync_file(path) → compressed + hashed in DB
-3. "Where is X used?"  → grep_code("X") → matching lines only, no full file
-4. "What do we know?"  → recall("query") → knowledge graph search
+1. "Index this project" → init_project()        → scans CLAUDE.md, package.json, src/**
+2. Write code          → sync_file(path)         → compressed + hashed + diff stored
+3. "What's relevant?"  → get_context("auth flow") → TF-IDF ranked skeletons, ~500 tokens
+4. "What changed?"     → get_recent(hours=2)      → line diffs of recent edits
+5. "Where is X used?"  → grep_code("X")           → matching lines only, ~30 tokens
+6. "What do we know?"  → recall("query")          → knowledge graph search
 ```
 
-## Tools (13)
+## Tools (15)
 
 ### Memory
 | Tool | Description |
@@ -56,11 +58,17 @@ Default DB path: `~/.claude/memory.db`
 ### Code indexing
 | Tool | Description |
 |---|---|
-| `init_project` | Scan project directory and bootstrap knowledge graph. Reads `CLAUDE.md`, `package.json`/`pyproject.toml`, `README.md`, `.mcp.json`, `logic-guardian.yaml`, source files. Also installs a Claude Code hook for auto-sync. |
-| `sync_file` | Index or re-index a single file after writing/editing. Stores compressed binary (zlib-9), skips instantly if SHA-256 hash unchanged. |
+| `init_project` | Scan project directory recursively and bootstrap knowledge graph. Reads `CLAUDE.md`, `package.json`/`pyproject.toml`, `README.md`, `.mcp.json`, `logic-guardian.yaml`, all source files. Installs a Claude Code hook for auto-sync. |
+| `sync_file` | Index or re-index a single file after writing/editing. Stores compressed binary (zlib-9), skips instantly if SHA-256 hash unchanged. Stores line-level diff from previous version. |
 | `sync_project` | Re-index entire project incrementally. Reports compression ratio. |
 | `grep_code` | Regex search across all indexed files. Decompresses binary on-the-fly, returns only matching lines with context — ~20-50 tokens vs reading full files. |
 
+### Token optimization
+| Tool | Description |
+|---|---|
+| `get_context` | **Smart context retrieval.** Ranks all indexed files by TF-IDF relevance (or Qdrant vector search if `QDRANT_URL` is set), applies recency boost, returns skeletons (imports + signatures only) for large files. Respects `maxContextTokens` budget. |
+| `get_recent` | Return files modified in the last N hours with line-level diffs. |
+
 ### Logic Guardian
 | Tool | Description |
 |---|---|
@@ -68,18 +76,112 @@ Default DB path: `~/.claude/memory.db`
 | `check_drift` | Analyze a code snippet inline without saving to disk. |
 | `get_checklist` | Return the full 5-pass validation protocol (Logic Trace, Contract Verification, Stupid Mistakes, Integration Sanity, Explain It). |
 
-## Why no vectors?
+## Token optimization in depth
+
+### How `get_context` works
+
+```
+query: "auth middleware"
+         ↓
+  1. TF-IDF score all indexed files against query
+     (or Qdrant top-k if QDRANT_URL is set)
+         ↓
+  2. Boost recently-modified files (+0.3 score)
+         ↓
+  3. Apply whitelist dirs filter (if configured)
+         ↓
+  4. For each file within token budget:
+       file < maxTokensPerFile  → return full source
+       file > maxTokensPerFile  → return skeleton only
+                                   (imports + signatures + TODOs)
+                                   + relevant fragments around query terms
+         ↓
+  output: ~500–2000 tokens  vs  5000–20000 for reading full files
+```
+
+### Skeleton pruning (AST-based)
+
+Large files are replaced with their structural skeleton:
+
+```typescript
+// src/middleware/auth.ts [skeleton]
+// Validates JWT tokens and attaches user to request context
+
+import { Request, Response, NextFunction } from "express"
+import { verifyToken } from "../services/jwt.js"
+
+// — exports —
+export function authMiddleware(req: Request, res: Response, next: NextFunction): void { … }
+export function requireRole(role: string): RequestHandler { … }
+export type AuthenticatedRequest = Request & { user: User }
+```
+
+vs reading the full 200-line file.
+
+### Qdrant vector search (optional)
+
+Set env vars to enable semantic search instead of TF-IDF:
+
+```bash
+QDRANT_URL=http://localhost:6333
+QDRANT_API_KEY=your-key          # optional
+OPENAI_API_KEY=sk-...            # for embeddings
+EMBEDDING_MODEL=text-embedding-3-small  # optional
+```
+
+Or in `.mcp.json`:
+```json
+{
+  "mcpServers": {
+    "lucid": {
+      "command": "npx", "args": ["-y", "@a13xu/lucid"],
+      "env": {
+        "QDRANT_URL": "http://localhost:6333",
+        "OPENAI_API_KEY": "sk-..."
+      }
+    }
+  }
+}
+```
+
+Falls back to TF-IDF automatically if Qdrant is unreachable.
+
+### Configuration (`lucid.config.json`)
+
+Create in your project root to customize behavior:
+
+```json
+{
+  "whitelistDirs": ["src", "backend", "api"],
+  "blacklistDirs": ["migrations", "fixtures"],
+  "maxTokensPerFile": 400,
+  "maxContextTokens": 6000,
+  "recentWindowHours": 12
+}
+```
+
+| Key | Default | Description |
+|---|---|---|
+| `whitelistDirs` | — | Only index/return files from these dirs |
+| `blacklistDirs` | — | Extra dirs to skip (merged with built-in skips) |
+| `maxTokensPerFile` | `400` | Files above this get skeleton treatment |
+| `maxContextTokens` | `4000` | Total token budget for `get_context` |
+| `recentWindowHours` | `24` | "Recently touched" threshold |
+
+## Why no vectors by default?
 
-Code has explicit structure — no NLP needed:
+Code has explicit structure — no NLP needed for most queries:
 
 | Need | Approach | Tokens |
 |---|---|---|
 | "Where is X defined?" | `grep_code("export.*X")` | ~30 |
 | "What does auth.ts export?" | `recall("auth.ts")` | ~50 |
+| "What changed recently?" | `get_recent(hours=2)` | ~200 |
+| "Context for this task" | `get_context("auth flow")` | ~500 |
 | "Project conventions?" | `recall("CLAUDE.md conventions")` | ~80 |
-| Read full file | `Read tool` | ~500-2000 |
+| Read full file | `Read tool` | ~500–2000 |
 
-Source files are stored as **zlib-deflate level 9 BLOBs** (~70% smaller than plain text). Change detection via SHA-256 means `sync_file` is instant on unchanged files.
+TF-IDF is fast, deterministic, and requires zero external services. Qdrant is available when you need semantic similarity across large codebases.
 
 ## Why SQLite + FTS5?
 
@@ -90,6 +192,7 @@ Source files are stored as **zlib-deflate level 9 BLOBs** (~70% smaller than pla
 | Concurrent reads | Lock entire file | WAL mode |
 | Code storage | Plain text | Compressed BLOB + hash |
 | Change detection | Manual diff | SHA-256 per file |
+| Diff history | None | Line-level diffs per file |
 
 ## Entity types
 `person` · `project` · `decision` · `pattern` · `tool` · `config` · `bug` · `convention`
@@ -104,7 +207,7 @@ echo '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"capabilities":{},
   | npx @a13xu/lucid
 ```
 
-In Claude Code: run `/mcp` — you should see `lucid` with 13 tools.
+In Claude Code: run `/mcp` — you should see `lucid` with 15 tools.
 
 ## Tech stack
 
@@ -113,5 +216,6 @@ In Claude Code: run `/mcp` — you should see `lucid` with 13 tools.
 - **Database:** `better-sqlite3` (synchronous, WAL mode)
 - **Compression:** Node.js built-in `zlib` (deflate level 9)
 - **Hashing:** SHA-256 via `crypto` (change detection)
+- **Ranking:** TF-IDF (built-in) or Qdrant (optional, via REST)
 - **Validation:** `zod`
 - **Transport:** stdio

package/build/config.d.ts

@@ -0,0 +1,37 @@
+import type { SecurityConfig } from "./security/guard.js";
+export interface LucidConfig {
+    /** Only index/return files from these directories (e.g. ["src", "lib", "backend"]) */
+    whitelistDirs?: string[];
+    /** Additional dirs to skip (merged with built-in SKIP_DIRS) */
+    blacklistDirs?: string[];
+    /** Max estimated tokens to return per file in get_context (default 400) */
+    maxTokensPerFile?: number;
+    /** Total token budget for get_context response (default 4000) */
+    maxContextTokens?: number;
+    /** "Recently touched" = modified within N hours (default 24) */
+    recentWindowHours?: number;
+    /** Security guard configuration */
+    security?: SecurityConfig;
+    /** Optional Qdrant vector search (falls back to TF-IDF if not configured) */
+    qdrant?: {
+        url: string;
+        apiKey?: string;
+        collection?: string;
+        /** Embedding endpoint — must be OpenAI-compatible */
+        embeddingUrl?: string;
+        embeddingApiKey?: string;
+        embeddingModel?: string;
+        vectorDim?: number;
+    };
+}
+export interface Defaults {
+    maxTokensPerFile: number;
+    maxContextTokens: number;
+    recentWindowHours: number;
+}
+export declare const DEFAULTS: Defaults;
+export type ResolvedConfig = Defaults & LucidConfig;
+export declare function loadConfig(projectDir?: string): ResolvedConfig;
+/** Qdrant config from lucid.config.json or env vars */
+export declare function getQdrantConfig(cfg: ResolvedConfig): ResolvedConfig["qdrant"] | null;
+export declare function resetConfigCache(): void;

package/build/config.js

@@ -0,0 +1,45 @@
+import { existsSync, readFileSync } from "fs";
+import { join } from "path";
+export const DEFAULTS = {
+    maxTokensPerFile: 400,
+    maxContextTokens: 4000,
+    recentWindowHours: 24,
+};
+let _cached = null;
+export function loadConfig(projectDir) {
+    if (_cached)
+        return _cached;
+    const dirs = [projectDir, process.cwd()].filter(Boolean);
+    for (const dir of dirs) {
+        const cfgPath = join(dir, "lucid.config.json");
+        if (existsSync(cfgPath)) {
+            try {
+                const raw = JSON.parse(readFileSync(cfgPath, "utf-8"));
+                _cached = { ...DEFAULTS, ...raw };
+                return _cached;
+            }
+            catch { /* malformed — skip */ }
+        }
+    }
+    _cached = { ...DEFAULTS };
+    return _cached;
+}
+/** Qdrant config from lucid.config.json or env vars */
+export function getQdrantConfig(cfg) {
+    // Env vars override config file
+    const url = process.env["QDRANT_URL"] ?? cfg.qdrant?.url;
+    if (!url)
+        return null;
+    return {
+        url,
+        apiKey: process.env["QDRANT_API_KEY"] ?? cfg.qdrant?.apiKey,
+        collection: process.env["QDRANT_COLLECTION"] ?? cfg.qdrant?.collection ?? "lucid",
+        embeddingUrl: process.env["EMBEDDING_URL"] ?? cfg.qdrant?.embeddingUrl ?? "https://api.openai.com/v1/embeddings",
+        embeddingApiKey: process.env["EMBEDDING_API_KEY"] ?? process.env["OPENAI_API_KEY"] ?? cfg.qdrant?.embeddingApiKey,
+        embeddingModel: process.env["EMBEDDING_MODEL"] ?? cfg.qdrant?.embeddingModel ?? "text-embedding-3-small",
+        vectorDim: cfg.qdrant?.vectorDim ?? 1536,
+    };
+}
+export function resetConfigCache() {
+    _cached = null;
+}

package/build/database.d.ts

@@ -13,16 +13,43 @@ export interface FileContentRow {
     language: string;
     indexed_at: number;
 }
+export interface FileDiffRow {
+    filepath: string;
+    prev_hash: string;
+    diff_text: string;
+    changed_at: number;
+}
+export interface ExperienceRow {
+    id: number;
+    query: string;
+    query_terms: string;
+    context_fps: string;
+    strategy: string;
+    reward: number;
+    feedback: string | null;
+    created_at: number;
+    rewarded_at: number | null;
+}
+export interface FileRewardRow {
+    filepath: string;
+    total_reward: number;
+    use_count: number;
+    last_rewarded: number | null;
+}
 export interface Statements {
     getFileByPath: Stmt<[string], FileContentRow>;
     upsertFile: WriteStmt<[string, Buffer, string, number, number, string]>;
-    getAllFiles: Stmt<[], Pick<FileContentRow, "filepath" | "content" | "language" | "content_hash">>;
+    getAllFiles: Stmt<[], Pick<FileContentRow, "filepath" | "content" | "language" | "content_hash" | "indexed_at">>;
+    getRecentFiles: Stmt<[number], Pick<FileContentRow, "filepath" | "language" | "indexed_at">>;
     deleteFile: WriteStmt<[string]>;
     fileStorageStats: Stmt<[], {
         count: number;
         total_original: number;
         total_compressed: number;
     }>;
+    upsertDiff: WriteStmt<[string, string, string]>;
+    getDiff: Stmt<[string], FileDiffRow>;
+    getRecentDiffs: Stmt<[number], FileDiffRow>;
     getEntityByName: Stmt<[string], EntityRow>;
     insertEntity: WriteStmt<[string, string, string]>;
     updateEntity: WriteStmt<[string, number]>;
@@ -48,6 +75,14 @@ export interface Statements {
         from_name: string;
         to_name: string;
     }>;
+    insertExperience: WriteStmt<[string, string, string, string]>;
+    updateExperienceReward: WriteStmt<[number, string | null, number]>;
+    getExperienceById: Stmt<[number], ExperienceRow>;
+    getTopExperiences: Stmt<[number], ExperienceRow>;
+    searchExperiencesFTS: Stmt<[string, number], ExperienceRow>;
+    upsertFileReward: WriteStmt<[string, number]>;
+    getFileRewards: Stmt<[], FileRewardRow>;
+    getTopFileRewards: Stmt<[number], FileRewardRow>;
 }
 export declare function prepareStatements(db: Database.Database): Statements;
 export {};

package/build/database.js

@@ -92,6 +92,57 @@ function createSchema(db) {
     );
     CREATE INDEX IF NOT EXISTS idx_fc_filepath ON file_contents(filepath);
     CREATE INDEX IF NOT EXISTS idx_fc_hash     ON file_contents(content_hash);
+    CREATE INDEX IF NOT EXISTS idx_fc_indexed  ON file_contents(indexed_at);
+
+    -- Diffs între versiuni consecutive (pentru get_recent)
+    CREATE TABLE IF NOT EXISTS file_diffs (
+      filepath    TEXT PRIMARY KEY,
+      prev_hash   TEXT NOT NULL,
+      diff_text   TEXT NOT NULL,
+      changed_at  INTEGER DEFAULT (unixepoch())
+    );
+    CREATE INDEX IF NOT EXISTS idx_fd_changed ON file_diffs(changed_at);
+
+    -- Experiences: logged get_context calls for RL reward system
+    CREATE TABLE IF NOT EXISTS experiences (
+      id          INTEGER PRIMARY KEY,
+      query       TEXT NOT NULL,
+      query_terms TEXT NOT NULL DEFAULT '',
+      context_fps TEXT NOT NULL DEFAULT '[]',
+      strategy    TEXT NOT NULL DEFAULT 'tfidf',
+      reward      REAL NOT NULL DEFAULT 0.0,
+      feedback    TEXT,
+      created_at  INTEGER NOT NULL DEFAULT (unixepoch()),
+      rewarded_at INTEGER
+    );
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS experiences_fts USING fts5(
+      query,
+      feedback,
+      content='experiences',
+      content_rowid='id',
+      tokenize='porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS experiences_ai AFTER INSERT ON experiences BEGIN
+      INSERT INTO experiences_fts(rowid, query, feedback)
+      VALUES (new.id, new.query, COALESCE(new.feedback, ''));
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS experiences_au AFTER UPDATE ON experiences BEGIN
+      INSERT INTO experiences_fts(experiences_fts, rowid, query, feedback)
+      VALUES('delete', old.id, old.query, COALESCE(old.feedback, ''));
+      INSERT INTO experiences_fts(rowid, query, feedback)
+      VALUES (new.id, new.query, COALESCE(new.feedback, ''));
+    END;
+
+    -- Denormalized reward cache per file (updated on every reward/penalize/implicit)
+    CREATE TABLE IF NOT EXISTS file_rewards (
+      filepath      TEXT PRIMARY KEY,
+      total_reward  REAL NOT NULL DEFAULT 0.0,
+      use_count     INTEGER NOT NULL DEFAULT 0,
+      last_rewarded INTEGER
+    );
   `);
 }
 export function prepareStatements(db) {
@@ -107,8 +158,17 @@ export function prepareStatements(db) {
          compressed_size = excluded.compressed_size,
          language = excluded.language,
          indexed_at = unixepoch()`),
-        getAllFiles: db.prepare("SELECT filepath, content, language, content_hash FROM file_contents"),
+        getAllFiles: db.prepare("SELECT filepath, content, language, content_hash, indexed_at FROM file_contents"),
+        getRecentFiles: db.prepare("SELECT filepath, language, indexed_at FROM file_contents WHERE indexed_at >= ? ORDER BY indexed_at DESC"),
         deleteFile: db.prepare("DELETE FROM file_contents WHERE filepath = ?"),
+        upsertDiff: db.prepare(`INSERT INTO file_diffs (filepath, prev_hash, diff_text)
+       VALUES (?, ?, ?)
+       ON CONFLICT(filepath) DO UPDATE SET
+         prev_hash  = excluded.prev_hash,
+         diff_text  = excluded.diff_text,
+         changed_at = unixepoch()`),
+        getDiff: db.prepare("SELECT * FROM file_diffs WHERE filepath = ?"),
+        getRecentDiffs: db.prepare("SELECT * FROM file_diffs WHERE changed_at >= ? ORDER BY changed_at DESC"),
         fileStorageStats: db.prepare("SELECT COUNT(*) as count, SUM(original_size) as total_original, SUM(compressed_size) as total_compressed FROM file_contents"),
         // entities
         getEntityByName: db.prepare("SELECT * FROM entities WHERE name = ? COLLATE NOCASE"),
@@ -139,5 +199,29 @@ export function prepareStatements(db) {
        FROM relations r
        JOIN entities ef ON r.from_entity = ef.id
        JOIN entities et ON r.to_entity = et.id`),
+        // experiences
+        insertExperience: db.prepare(`INSERT INTO experiences (query, query_terms, context_fps, strategy)
+       VALUES (?, ?, ?, ?)`),
+        updateExperienceReward: db.prepare(`UPDATE experiences SET
+         reward = reward + ?,
+         rewarded_at = unixepoch(),
+         feedback = COALESCE(?, feedback)
+       WHERE id = ?`),
+        getExperienceById: db.prepare("SELECT * FROM experiences WHERE id = ?"),
+        getTopExperiences: db.prepare("SELECT * FROM experiences ORDER BY reward DESC LIMIT ?"),
+        searchExperiencesFTS: db.prepare(`SELECT e.* FROM experiences_fts
+       JOIN experiences e ON experiences_fts.rowid = e.id
+       WHERE experiences_fts MATCH ?
+       ORDER BY rank
+       LIMIT ?`),
+        // file_rewards
+        upsertFileReward: db.prepare(`INSERT INTO file_rewards (filepath, total_reward, use_count, last_rewarded)
+       VALUES (?, ?, 1, unixepoch())
+       ON CONFLICT(filepath) DO UPDATE SET
+         total_reward  = total_reward + excluded.total_reward,
+         use_count     = use_count + 1,
+         last_rewarded = unixepoch()`),
+        getFileRewards: db.prepare("SELECT * FROM file_rewards"),
+        getTopFileRewards: db.prepare("SELECT * FROM file_rewards WHERE total_reward > 0 ORDER BY total_reward DESC LIMIT ?"),
     };
 }

package/build/guardian/coding-analyzer.d.ts

@@ -0,0 +1,11 @@
+export type QualitySeverity = "high" | "medium" | "low";
+export interface QualityIssue {
+    file: string;
+    line: number;
+    severity: QualitySeverity;
+    ruleId: string;
+    message: string;
+    suggestion: string;
+}
+export declare function analyzeCodeQuality(filepath: string, source: string, lang?: string): QualityIssue[];
+export declare function formatQualityReport(filepath: string, issues: QualityIssue[]): string;