@a13xu/lucid 1.13.0 → 1.16.0

package/build/retrieval/qdrant.js

@@ -2,6 +2,7 @@
 // Only active when QDRANT_URL is set (via env var or lucid.config.json)
 // Falls back silently to TF-IDF when unavailable
 import { safeFetch } from "../security/ssrf.js";
+import { tryCompressTextSemantic } from "../compression/semantic.js";
 // ---------------------------------------------------------------------------
 // Embedding generation (OpenAI-compatible endpoint)
 // ---------------------------------------------------------------------------
@@ -85,16 +86,24 @@ function stableId(s) {
 // Public API
 // ---------------------------------------------------------------------------
 /** Index one file into Qdrant (called by sync_file when Qdrant is configured). */
-export async function indexFileInQdrant(filepath, text, cfg) {
+export async function indexFileInQdrant(filepath, text, cfg, compressionCfg) {
     await ensureCollection(cfg);
     const chunks = chunkFile(filepath, text);
     if (chunks.length === 0)
         return;
+    const compressForEmbedding = compressionCfg?.enabled && compressionCfg.applyToEmbeddings !== false;
     // Batch embed (max 96 texts per request for most providers)
     const BATCH = 32;
     for (let b = 0; b < chunks.length; b += BATCH) {
         const batch = chunks.slice(b, b + BATCH);
-        const vectors = await embed(batch.map((c) => c.text), cfg);
+        let textsToEmbed;
+        if (compressForEmbedding) {
+            textsToEmbed = await Promise.all(batch.map((c) => tryCompressTextSemantic(c.text, compressionCfg.ratio ?? 0.5, compressionCfg.minLength ?? 300)));
+        }
+        else {
+            textsToEmbed = batch.map((c) => c.text);
+        }
+        const vectors = await embed(textsToEmbed, cfg);
         const points = batch.map((c, idx) => ({
             id: c.id,
             vector: vectors[idx],

package/build/tools/compress.d.ts

@@ -0,0 +1,15 @@
+import { z } from "zod";
+export declare const CompressTextSchema: z.ZodObject<{
+    text: z.ZodString;
+    ratio: z.ZodOptional<z.ZodNumber>;
+    min_length: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    text: string;
+    ratio?: number | undefined;
+    min_length?: number | undefined;
+}, {
+    text: string;
+    ratio?: number | undefined;
+    min_length?: number | undefined;
+}>;
+export declare function handleCompressText(args: z.infer<typeof CompressTextSchema>): Promise<string>;

package/build/tools/compress.js

@@ -0,0 +1,18 @@
+import { z } from "zod";
+import { compressTextSemantic } from "../compression/semantic.js";
+export const CompressTextSchema = z.object({
+    text: z.string().min(1).describe("Text to compress"),
+    ratio: z.number().min(0.1).max(0.9).optional().describe("Target compression ratio: 0.3 = keep 30%, 0.5 = keep 50% (default: 0.5)"),
+    min_length: z.number().int().optional().describe("Skip compression for texts shorter than this in chars (default: 300)"),
+});
+export async function handleCompressText(args) {
+    const result = await compressTextSemantic(args.text, args.ratio ?? 0.5, args.min_length ?? 300);
+    return JSON.stringify({
+        compressed: result.compressed,
+        original_length: result.originalLength,
+        compressed_length: result.compressedLength,
+        ratio_kept: result.ratio,
+        method: result.method,
+        tokens_saved: Math.ceil((result.originalLength - result.compressedLength) / 4),
+    }, null, 2);
+}

package/build/tools/init.js

@@ -1,5 +1,6 @@
 import { z } from "zod";
 import { resolve, join } from "path";
+import { homedir } from "os";
 import { existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from "fs";
 import { fileURLToPath } from "url";
 import { indexProject } from "../indexer/project.js";
@@ -35,7 +36,7 @@ const LUCID_HOOK = {
     hooks: [
         {
             type: "command",
-            command: `echo '🔄 ${LUCID_MARKER}(path) to keep knowledge graph up to date'`,
+            command: `lucid-sync 2>/dev/null || echo '🔄 ${LUCID_MARKER}(path) — install lucid globally: npm i -g @a13xu/lucid'`,
         },
     ],
 };
@@ -167,6 +168,17 @@ export async function handleInitProject(stmts, input) {
     else if (skillsResult.skipped.length > 0) {
         lines.push(`📚 Skills: already installed (${skillsResult.skipped.length} skill(s))`);
     }
+    // ── Global skills (~/.claude/skills/) ────────────────────────────────────
+    const globalSkillsResult = installGlobalSkills();
+    if (globalSkillsResult.installed.length > 0) {
+        lines.push(`🌐 Global skills installed in ~/.claude/skills/:`);
+        for (const s of globalSkillsResult.installed) {
+            lines.push(`   • /${s} (available in all projects)`);
+        }
+    }
+    else if (globalSkillsResult.skipped.length > 0) {
+        lines.push(`🌐 Global skills: already installed (${globalSkillsResult.skipped.length} skill(s))`);
+    }
     // ── CLAUDE.md injection ───────────────────────────────────────────────────
     const injected = injectClaudeMdInstruction(dir);
     if (injected) {
@@ -277,6 +289,9 @@ function installSkills(projectDir) {
     }
     return result;
 }
+function installGlobalSkills() {
+    return installSkills(homedir());
+}
 function buildChannelSummary(cfg) {
     const channels = [];
     if (cfg.adminEmail && cfg.smtpHost)

package/build/tools/model-advisor.d.ts

@@ -0,0 +1,9 @@
+import { z } from "zod";
+export declare const SuggestModelSchema: z.ZodObject<{
+    task_description: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    task_description: string;
+}, {
+    task_description: string;
+}>;
+export declare function handleSuggestModel(args: z.infer<typeof SuggestModelSchema>): string;

package/build/tools/model-advisor.js

@@ -0,0 +1,30 @@
+import { z } from "zod";
+export const SuggestModelSchema = z.object({
+    task_description: z.string().min(1).describe("Natural language description of the task you are about to perform"),
+});
+const HAIKU_TRIGGERS = [
+    "list", "show", "find", "search", "where", "what is",
+    "recall", "get recent", "status",
+];
+const MODEL_IDS = {
+    haiku: "claude-haiku-4-5-20251001",
+    sonnet: "claude-sonnet-4-6",
+};
+const CONTEXT_BUDGETS = {
+    haiku: 2000,
+    sonnet: 8000,
+};
+export function handleSuggestModel(args) {
+    const lower = args.task_description.toLowerCase();
+    const haikuTrigger = HAIKU_TRIGGERS.find((t) => lower.includes(t));
+    const model = haikuTrigger ? "haiku" : "sonnet";
+    const reasoning = haikuTrigger
+        ? `Task matches retrieval/lookup pattern ("${haikuTrigger}") — Haiku is faster for read-only queries.`
+        : "No simple retrieval trigger detected — defaulting to Sonnet for reasoning, code generation, and analysis.";
+    return JSON.stringify({
+        model,
+        model_id: MODEL_IDS[model],
+        reasoning,
+        context_budget: CONTEXT_BUDGETS[model],
+    }, null, 2);
+}

package/build/tools/smart-context.d.ts

@@ -0,0 +1,16 @@
+import { z } from "zod";
+import type { Statements } from "../database.js";
+export declare const SmartContextSchema: z.ZodObject<{
+    query: z.ZodString;
+    task_type: z.ZodOptional<z.ZodEnum<["simple", "moderate", "complex"]>>;
+    dirs: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    query: string;
+    dirs?: string[] | undefined;
+    task_type?: "simple" | "moderate" | "complex" | undefined;
+}, {
+    query: string;
+    dirs?: string[] | undefined;
+    task_type?: "simple" | "moderate" | "complex" | undefined;
+}>;
+export declare function handleSmartContext(stmts: Statements, args: z.infer<typeof SmartContextSchema>): Promise<string>;

package/build/tools/smart-context.js

@@ -0,0 +1,54 @@
+import { z } from "zod";
+import { assembleContext } from "../retrieval/context.js";
+import { recall } from "./recall.js";
+import { loadConfig } from "../config.js";
+import { createExperience } from "../memory/experience.js";
+export const SmartContextSchema = z.object({
+    query: z.string().min(1).describe("What you are working on — used for both code retrieval and knowledge graph search"),
+    task_type: z.enum(["simple", "moderate", "complex"]).optional().describe("Token budget: simple=2000, moderate=6000 (default), complex=12000"),
+    dirs: z.array(z.string()).optional().describe("Whitelist: only return files from these directories"),
+});
+const TASK_BUDGETS = {
+    simple: 2000,
+    moderate: 6000,
+    complex: 12000,
+};
+export async function handleSmartContext(stmts, args) {
+    const cfg = loadConfig();
+    const maxTokens = TASK_BUDGETS[args.task_type ?? "moderate"] ?? 6000;
+    // 1. Knowledge graph entities (synchronous)
+    const recallResult = recall(stmts, { query: args.query });
+    // 2. Code context with adaptive budget (async)
+    const contextResult = await assembleContext(args.query, stmts, cfg, {
+        maxTokens,
+        dirs: args.dirs,
+    });
+    // 3. Log experience so reward()/penalize() work after this call
+    const expId = createExperience(args.query, contextResult.files.map((f) => f.filepath), contextResult.strategy, stmts);
+    const budgetUsedPct = Math.round((contextResult.totalTokens / maxTokens) * 100);
+    const sections = [
+        "## Knowledge Context (entities)",
+        recallResult,
+        "",
+        "## Code Context (files)",
+    ];
+    if (contextResult.files.length === 0) {
+        sections.push("No relevant files found. Run init_project() or sync_project() first.");
+    }
+    else {
+        for (const f of contextResult.files) {
+            sections.push(`// ─── ${f.filepath} [${f.language}] ~${f.tokens}t (${f.reason}) ───`);
+            sections.push(f.content);
+            sections.push("");
+        }
+        if (contextResult.truncated) {
+            sections.push(`// ⚠️  Truncated — ${contextResult.skippedFiles} files skipped. Use task_type="complex" for more.`);
+        }
+    }
+    sections.push("", "---");
+    sections.push(`Strategy: ${contextResult.strategy}`);
+    sections.push(`Files: ${contextResult.files.length} files, ${contextResult.totalTokens} tokens`);
+    sections.push(`Budget used: ${budgetUsedPct}%`);
+    sections.push(`Experience #${expId} logged. Call reward() if helpful, penalize() if not.`);
+    return sections.join("\n");
+}

package/build/tools/sync.js

@@ -6,6 +6,8 @@ import { indexProject } from "../indexer/project.js";
 import { computeDiff } from "../retrieval/context.js";
 import { decompress } from "../store/content.js";
 import { implicitRewardFromSync } from "../memory/experience.js";
+import { indexFileInQdrant } from "../retrieval/qdrant.js";
+import { loadConfig, getQdrantConfig } from "../config.js";
 const SUPPORTED_EXTS = new Set([".ts", ".tsx", ".js", ".jsx", ".vue", ".py", ".go", ".rs"]);
 // ---------------------------------------------------------------------------
 // sync_file
@@ -51,6 +53,12 @@ export function handleSyncFile(stmts, args) {
     const implicitRewarded = implicitRewardFromSync(filepath, stmts);
     if (implicitRewarded)
         lines.push(`   🎯 Implicit reward +0.3 (file was in recent context)`);
+    // Qdrant vector indexing — fire-and-forget (Qdrant is optional, silent on failure)
+    const cfg = loadConfig();
+    const qdrantCfg = getQdrantConfig(cfg);
+    if (qdrantCfg) {
+        void indexFileInQdrant(filepath, source, qdrantCfg, cfg.semanticCompression).catch(() => { });
+    }
     return lines.join("\n");
 }
 // ---------------------------------------------------------------------------

package/package.json

@@ -1,59 +1,64 @@
-{
-  "name": "@a13xu/lucid",
-  "version": "1.13.0",
-  "description": "Token-efficient memory, code indexing, and validation for Claude Code agents — SQLite + FTS5, TF-IDF + Qdrant retrieval, AST skeleton pruning, diff-aware context, Logic Guardian drift detection",
-  "type": "module",
-  "bin": {
-    "lucid": "./build/index.js"
-  },
-  "files": [
-    "build/**/*.js",
-    "build/**/*.d.ts",
-    "skills/**/*.md",
-    "README.md"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "prepublishOnly": "npm run build"
-  },
-  "keywords": [
-    "mcp",
-    "claude",
-    "claude-code",
-    "memory",
-    "sqlite",
-    "knowledge-graph",
-    "code-indexing",
-    "logic-guardian",
-    "drift-detection",
-    "tfidf",
-    "qdrant",
-    "token-optimization",
-    "context-pruning",
-    "ai",
-    "anthropic"
-  ],
-  "author": "a13xu",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/a13xu/lucid.git"
-  },
-  "homepage": "https://github.com/a13xu/lucid#readme",
-  "publishConfig": {
-    "access": "public"
-  },
-  "engines": {
-    "node": ">=18"
-  },
-  "dependencies": {
-    "@modelcontextprotocol/sdk": "^1.0.0",
-    "better-sqlite3": "^12.0.0",
-    "zod": "^3.23.8"
-  },
-  "devDependencies": {
-    "@types/better-sqlite3": "^7.6.11",
-    "@types/node": "^22.0.0",
-    "typescript": "^5.4.0"
-  }
-}
+{
+  "name": "@a13xu/lucid",
+  "version": "1.16.0",
+  "description": "Token-efficient memory, code indexing, and validation for Claude Code agents — SQLite + FTS5, TF-IDF + Qdrant retrieval, AST skeleton pruning, diff-aware context, Logic Guardian drift detection",
+  "type": "module",
+  "bin": {
+    "lucid": "./build/index.js",
+    "lucid-sync": "./build/lucid-sync.js"
+  },
+  "files": [
+    "build/**/*.js",
+    "build/**/*.d.ts",
+    "skills/**/*.md",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "mcp",
+    "claude",
+    "claude-code",
+    "memory",
+    "sqlite",
+    "knowledge-graph",
+    "code-indexing",
+    "logic-guardian",
+    "drift-detection",
+    "tfidf",
+    "qdrant",
+    "token-optimization",
+    "context-pruning",
+    "ai",
+    "anthropic"
+  ],
+  "author": "a13xu",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/a13xu/lucid.git"
+  },
+  "homepage": "https://github.com/a13xu/lucid#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "dependencies": {
+    "@huggingface/transformers": "^4.0.0",
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "better-sqlite3": "^12.0.0",
+    "chokidar": "^4.0.3",
+    "express": "^5.2.1",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.11",
+    "@types/express": "^5.0.6",
+    "@types/node": "^22.0.0",
+    "typescript": "^5.4.0"
+  }
+}

package/skills/lucid-audit/SKILL.md

@@ -1,53 +1,73 @@
----
-name: lucid-audit
-description: Run after writing or modifying code — validates logic correctness (Logic Guardian 5 passes) and code quality (25 Golden Rules) before marking work as done.
-argument-hint: "[file path or 'all']"
----
-
-# Lucid Code Audit
-
-Run this skill BEFORE marking any implementation as complete. It runs two complementary validators:
-
-- **Logic Guardian** (`validate_file`) — detects LLM drift: logic inversions, null propagation, off-by-one, copy-paste mistakes
-- **Code Quality Guard** (`check_code_quality`) — detects structural issues: file/function size, vague naming, deep nesting, prop explosion, inline styles
-
-## Steps
-
-### 1. Validate logic correctness
-```
-validate_file(path="<file you wrote or modified>")
-```
-Fix any 🔴 CRITICAL issues before continuing.
-
-### 2. Validate code quality
-```
-check_code_quality(path="<same file>")
-```
-Fix any 🔴 HIGH severity issues. Address 🟠 MEDIUM where practical.
-
-### 3. If unsure about a snippet before writing it to disk
-```
-check_drift(code="<your code>", language="typescript")
-```
-
-### 4. Get the full 5-pass mental checklist
-```
-get_checklist()
-```
-Use this when changes are complex or involve critical business logic.
-
-## Severity guide
-
-| Icon | Level | Action |
-|---|---|---|
-| 🔴 | Critical/High | Fix immediately — do not ship |
-| 🟠 | Medium/Warning | Fix if not risky refactor |
-| 🔵 | Low/Info | Note for future cleanup |
-
-## What each tool catches
-
-| Tool | Catches |
-|---|---|
-| `validate_file` | Logic inversions, silent exceptions, null propagation, type confusion, stale closures |
-| `check_code_quality` | Files >500 lines, functions >100 lines, vague names, nesting >4 levels, dead code, React/Vue component anti-patterns |
-| `check_drift` | Same as validate_file but on inline snippets — use before writing |
+---
+name: lucid-audit
+description: MANDATORY before marking any task done — runs Logic Guardian + Code Quality checks. HARD-GATE blocks completion without validation.
+argument-hint: "[file path or 'all changed files']"
+---
+
+<HARD-GATE>
+You are about to say "done", "fixed", "complete", or "implemented".
+STOP. You have NOT verified the code yet.
+
+Do NOT mark any task as done, do NOT commit, do NOT say the work is complete
+until you have run BOTH validators below and fixed all 🔴 CRITICAL issues.
+
+"It looks correct" is not verification. Run the tools.
+</HARD-GATE>
+
+## When to invoke
+
+**INVOKE when:** about to say done/fixed/complete/implemented, before committing, before creating a PR
+**DO NOT INVOKE for:** read-only tasks, pure research, config changes with no logic
+
+## Steps
+
+```dot
+digraph lucid_audit {
+    "About to say done?" -> "validate_file(path)";
+    "validate_file(path)" -> "Critical issues?";
+    "Critical issues?" -> "Fix issues" [label="yes 🔴"];
+    "Fix issues" -> "validate_file(path)";
+    "Critical issues?" -> "check_code_quality(path)" [label="no ✓"];
+    "check_code_quality(path)" -> "HIGH issues?";
+    "HIGH issues?" -> "Fix if safe" [label="yes 🟠"];
+    "Fix if safe" -> "Mark done ✓";
+    "HIGH issues?" -> "Mark done ✓" [label="no ✓"];
+}
+```
+
+### 0. Get model recommendation
+```
+suggest_model(task_description="<paste the file path or description of what was written>")
+```
+Say: **"Using [model] — [reasoning]"** then proceed.
+
+### 1. Validate logic correctness
+```
+validate_file(path="<file you wrote or modified>")
+```
+Fix every 🔴 CRITICAL issue. Re-run until clean.
+
+### 2. Validate code quality
+```
+check_code_quality(path="<same file>")
+```
+Fix 🔴 HIGH severity issues. Address 🟠 MEDIUM where practical.
+
+### 3. For complex logic — get the full checklist
+```
+get_checklist()
+```
+Run all 5 mental passes before marking done.
+
+### 4. Pre-write validation (before writing to disk)
+```
+check_drift(code="<your code snippet>", language="typescript")
+```
+
+## Severity guide
+
+| Icon | Level | Action |
+|---|---|---|
+| 🔴 | Critical/High | Fix immediately — do not proceed |
+| 🟠 | Medium | Fix if the refactor is safe |
+| 🔵 | Low/Info | Note for future cleanup |

package/skills/lucid-context/SKILL.md

@@ -1,35 +1,69 @@
----
-name: lucid-context
-description: Use before starting any coding task — retrieves minimal relevant context via Lucid's TF-IDF retrieval, then rewards or penalizes based on usefulness.
-argument-hint: "[what you are working on]"
----
-
-# Lucid Context Retrieval
-
-Use this skill at the START of every coding task to get only the relevant files instead of reading the whole codebase.
-
-## Steps
-
-1. **Call `get_context`** with a concise description of what you're working on:
-   ```
-   get_context(query="<what you are working on>", maxTokens=4000)
-   ```
-
-2. **Review the returned skeletons/files.** If they are relevant → call `reward()`. If they missed important files → call `penalize()` and note what was missing.
-
-3. **Start coding** using the context you received.
-
-## Tips
-
-- Use `dirs` to narrow scope: `get_context(query="...", dirs=["src/api"])`
-- Use `get_recent(hours=2)` after a git pull or when resuming a session to see what changed
-- Use `grep_code(pattern="...")` to locate specific function usages without reading full files
-- After finishing, call `sync_file(path="<modified file>")` to keep the knowledge graph current
-
-## When to reward vs penalize
-
-| Situation | Action |
-|---|---|
-| Context included the files that helped solve the task | `reward()` |
-| Context missed key files you had to find manually | `penalize(note="missed: src/utils/auth.ts")` |
-| Context was partially useful | No action needed |
+---
+name: lucid-context
+description: Use BEFORE starting any coding task — retrieves relevant context via smart_context (code + knowledge graph). HARD-GATE: do not read files manually before calling smart_context.
+argument-hint: "[what you are working on]"
+---
+
+<HARD-GATE>
+Do NOT open any source file, read any code, or start implementation
+until you have called smart_context and reviewed the result.
+Reading files manually when Lucid is available wastes tokens and misses context.
+</HARD-GATE>
+
+## When to invoke this skill
+
+**INVOKE when:** about to work on a feature, fix a bug, understand a module, or any coding task
+**DO NOT INVOKE for:** pure conversation, reading docs, non-code questions
+
+## Steps
+
+```dot
+digraph lucid_context {
+    "Describe task" -> "suggest_model";
+    "suggest_model" -> "call smart_context";
+    "call smart_context" -> "Result relevant?";
+    "Result relevant?" -> "call reward()" [label="yes"];
+    "Result relevant?" -> "call penalize()" [label="no — note what was missing"];
+    "reward()" -> "Start coding";
+    "penalize()" -> "Start coding";
+}
+```
+
+### 0. Get model recommendation
+```
+suggest_model(task_description="<concise description of what you are working on>")
+```
+Say: **"Using [model] — [reasoning]"**
+
+### 1. Call smart_context
+```
+smart_context(query="<concise description of what you are working on>", task_type="moderate")
+```
+
+Use `dirs` to narrow scope and `task_type` to adjust budget:
+```
+smart_context(query="...", dirs=["src/api"], task_type="simple")
+```
+
+### 2. Review results and give feedback
+
+| Result quality | Action |
+|---|---|
+| Included the files you needed | `reward()` |
+| Missed important files you had to find manually | `penalize(note="missed: src/path/file.ts")` |
+| Partially useful | no action |
+
+### 3. Supplement if needed
+
+```
+grep_code(pattern="functionName")          # locate specific usages
+get_recent(hours=2)                        # after git pull — see what changed
+recall(query="<topic>")                    # search accumulated knowledge
+```
+
+### 4. After finishing — sync
+
+After every Write/Edit:
+```
+sync_file(path="<modified file>")
+```