@a13xu/lucid 1.4.0 → 1.9.1

package/build/retrieval/context.js

@@ -0,0 +1,219 @@
+// Smart context assembly — TF-IDF + recency boost + AST skeleton pruning
+// Falls back gracefully: Qdrant → TF-IDF → recency-only
+import { decompress } from "../store/content.js";
+import { rankByRelevance } from "./tfidf.js";
+import { extractSkeleton, renderSkeleton } from "../indexer/ast.js";
+import { searchQdrant } from "./qdrant.js";
+import { getQdrantConfig } from "../config.js";
+import { getFileRewardsMap } from "../memory/experience.js";
+// ---------------------------------------------------------------------------
+// Token estimation (1 token ≈ 4 chars is the standard heuristic)
+// ---------------------------------------------------------------------------
+export function estimateTokens(text) {
+    return Math.ceil(text.length / 4);
+}
+// ---------------------------------------------------------------------------
+// Relevant fragment extraction (lines around query matches)
+// ---------------------------------------------------------------------------
+export function extractFragments(source, query, contextLines = 3) {
+    const lines = source.split("\n");
+    const terms = query.toLowerCase().split(/\s+/).filter((t) => t.length > 2);
+    const hitLines = new Set();
+    for (let i = 0; i < lines.length; i++) {
+        const lower = lines[i].toLowerCase();
+        if (terms.some((t) => lower.includes(t))) {
+            for (let j = Math.max(0, i - contextLines); j <= Math.min(lines.length - 1, i + contextLines); j++) {
+                hitLines.add(j);
+            }
+        }
+    }
+    if (hitLines.size === 0)
+        return "";
+    const sorted = [...hitLines].sort((a, b) => a - b);
+    const out = [];
+    let prev = -2;
+    for (const n of sorted) {
+        if (n > prev + 1)
+            out.push("…");
+        out.push(`${n + 1}: ${lines[n]}`);
+        prev = n;
+    }
+    return out.join("\n");
+}
+// ---------------------------------------------------------------------------
+// Simple line-level diff (no external deps)
+// ---------------------------------------------------------------------------
+export function computeDiff(prev, curr, maxChanges = 40) {
+    const pLines = prev.split("\n");
+    const cLines = curr.split("\n");
+    const out = [];
+    let changes = 0;
+    const maxLen = Math.max(pLines.length, cLines.length);
+    for (let i = 0; i < maxLen; i++) {
+        if (changes >= maxChanges) {
+            out.push(`[… +${Math.abs(cLines.length - pLines.length)} more line changes, truncated]`);
+            break;
+        }
+        const p = pLines[i];
+        const c = cLines[i];
+        if (p === c)
+            continue;
+        if (p === undefined) {
+            out.push(`+${i + 1}: ${c}`);
+        }
+        else if (c === undefined) {
+            out.push(`-${i + 1}: ${p}`);
+        }
+        else {
+            out.push(`-${i + 1}: ${p}`);
+            out.push(`+${i + 1}: ${c}`);
+        }
+        changes++;
+    }
+    return out.length > 0 ? out.join("\n") : "[no line changes]";
+}
+// ---------------------------------------------------------------------------
+// Main function
+// ---------------------------------------------------------------------------
+export async function assembleContext(query, stmts, cfg, opts = {}) {
+    const maxTokens = opts.maxTokens ?? cfg.maxContextTokens;
+    const maxPerFile = opts.maxTokensPerFile ?? cfg.maxTokensPerFile;
+    const recentHours = opts.recentHours ?? cfg.recentWindowHours;
+    const topK = opts.topK ?? 10;
+    const allRows = stmts.getAllFiles.all();
+    if (!Array.isArray(allRows) || allRows.length === 0) {
+        return { files: [], totalTokens: 0, strategy: "tfidf", truncated: false, skippedFiles: 0 };
+    }
+    // Apply whitelist dirs filter
+    const dirs = opts.dirs ?? cfg.whitelistDirs;
+    const filtered = dirs && dirs.length > 0
+        ? allRows.filter((r) => dirs.some((d) => r.filepath.replace(/\\/g, "/").includes(d)))
+        : allRows;
+    // Recency cutoff
+    const nowSec = Math.floor(Date.now() / 1000);
+    const cutoffSec = nowSec - recentHours * 3600;
+    const recentSet = new Set(filtered.filter((r) => (r.indexed_at ?? 0) >= cutoffSec).map((r) => r.filepath));
+    // If recentOnly, skip files outside the window
+    const candidates = opts.recentOnly
+        ? filtered.filter((r) => recentSet.has(r.filepath))
+        : filtered;
+    if (candidates.length === 0) {
+        return { files: [], totalTokens: 0, strategy: opts.recentOnly ? "recent" : "tfidf", truncated: false, skippedFiles: filtered.length };
+    }
+    // Decompress all candidates
+    const decompressed = candidates.map((r) => ({
+        filepath: r.filepath,
+        language: r.language,
+        indexedAt: r.indexed_at ?? 0,
+        text: decompress(r.content),
+    }));
+    // ---------------------------------------------------------------------------
+    // Ranking strategy
+    // ---------------------------------------------------------------------------
+    let strategy = "tfidf";
+    let ranked;
+    // Load experience-based rewards for ranking boost (decayed, normalized to +0.25 max)
+    const fileRewards = getFileRewardsMap(stmts);
+    const qdrantCfg = getQdrantConfig(cfg);
+    if (qdrantCfg && !opts.recentOnly) {
+        // Try Qdrant first
+        try {
+            const chunks = await searchQdrant(query, topK * 3, qdrantCfg);
+            if (chunks.length > 0) {
+                strategy = "qdrant";
+                // Deduplicate by filepath, preserve order
+                const seen = new Set();
+                const qdrantOrder = [];
+                for (const c of chunks) {
+                    if (!seen.has(c.filepath)) {
+                        seen.add(c.filepath);
+                        qdrantOrder.push(c.filepath);
+                    }
+                }
+                // Place Qdrant matches first, then remaining by TF-IDF + reward boost
+                const tfidfRanked = rankByRelevance(query, decompressed);
+                const tfidfOrder = tfidfRanked.map((s) => s.filepath).filter((fp) => !seen.has(fp));
+                const orderedFps = [...qdrantOrder, ...tfidfOrder];
+                const fpToDoc = new Map(decompressed.map((d) => [d.filepath, d]));
+                ranked = orderedFps.map((fp) => fpToDoc.get(fp)).filter(Boolean);
+            }
+            else {
+                ranked = rankAndBoost(query, decompressed, recentSet, fileRewards);
+            }
+        }
+        catch {
+            // Qdrant unreachable — fall back to TF-IDF
+            ranked = rankAndBoost(query, decompressed, recentSet, fileRewards);
+        }
+    }
+    else {
+        ranked = rankAndBoost(query, decompressed, recentSet, fileRewards);
+        if (opts.recentOnly)
+            strategy = "recent";
+    }
+    // ---------------------------------------------------------------------------
+    // Assemble context with token budget
+    // ---------------------------------------------------------------------------
+    const result = [];
+    let totalTokens = 0;
+    let truncated = false;
+    let skippedFiles = 0;
+    for (const file of ranked) {
+        if (totalTokens >= maxTokens) {
+            truncated = true;
+            break;
+        }
+        const remaining = maxTokens - totalTokens;
+        const fullTokens = estimateTokens(file.text);
+        const isRecent = recentSet.has(file.filepath);
+        let content;
+        let reason;
+        if (opts.skeletonOnly || fullTokens > maxPerFile) {
+            const sk = extractSkeleton(file.text, file.language);
+            const skText = renderSkeleton(sk, file.filepath);
+            const fragments = query ? extractFragments(file.text, query) : "";
+            content = fragments
+                ? `${skText}\n\n// — relevant fragments —\n${fragments}`
+                : skText;
+            reason = opts.skeletonOnly ? "skeleton" : `skeleton (${fullTokens} tokens > limit ${maxPerFile})`;
+        }
+        else {
+            content = file.text;
+            reason = "full";
+        }
+        if (isRecent)
+            reason += " +recent";
+        const contentTokens = estimateTokens(content);
+        if (contentTokens < 10) {
+            skippedFiles++;
+            continue;
+        }
+        // Truncate to remaining budget
+        const usedTokens = Math.min(contentTokens, remaining);
+        const finalContent = usedTokens < contentTokens
+            ? content.slice(0, usedTokens * 4) + "\n… [truncated]"
+            : content;
+        result.push({ filepath: file.filepath, language: file.language, tokens: usedTokens, content: finalContent, reason });
+        totalTokens += usedTokens;
+    }
+    skippedFiles += ranked.length - result.length - (truncated ? 0 : 0);
+    return { files: result, totalTokens, strategy, truncated, skippedFiles };
+}
+// ---------------------------------------------------------------------------
+// TF-IDF + recency boost ranking
+// ---------------------------------------------------------------------------
+function rankAndBoost(query, docs, recentSet, fileRewards) {
+    const scored = rankByRelevance(query, docs);
+    const scoreMap = new Map(scored.map((s) => [s.filepath, s.score]));
+    // Compute normalizer for experience boost (max 0.25, avoids dominating TF-IDF)
+    const maxReward = fileRewards && fileRewards.size > 0
+        ? Math.max(...fileRewards.values())
+        : 0;
+    return [...docs].sort((a, b) => {
+        const rewardBoostA = maxReward > 0 ? ((fileRewards.get(a.filepath) ?? 0) / maxReward) * 0.25 : 0;
+        const rewardBoostB = maxReward > 0 ? ((fileRewards.get(b.filepath) ?? 0) / maxReward) * 0.25 : 0;
+        const sA = (scoreMap.get(a.filepath) ?? 0) + (recentSet.has(a.filepath) ? 0.3 : 0) + rewardBoostA;
+        const sB = (scoreMap.get(b.filepath) ?? 0) + (recentSet.has(b.filepath) ? 0.3 : 0) + rewardBoostB;
+        return sB - sA;
+    });
+}

package/build/retrieval/qdrant.d.ts

@@ -0,0 +1,16 @@
+import type { ResolvedConfig } from "../config.js";
+type QdrantCfg = NonNullable<ResolvedConfig["qdrant"]>;
+export interface VectorChunk {
+    id: number;
+    filepath: string;
+    chunkIndex: number;
+    text: string;
+    score: number;
+}
+/** Index one file into Qdrant (called by sync_file when Qdrant is configured). */
+export declare function indexFileInQdrant(filepath: string, text: string, cfg: QdrantCfg): Promise<void>;
+/** Top-k semantic search across all indexed chunks. */
+export declare function searchQdrant(query: string, topK: number, cfg: QdrantCfg): Promise<VectorChunk[]>;
+/** Check if Qdrant collection exists and is reachable. */
+export declare function pingQdrant(cfg: QdrantCfg): Promise<boolean>;
+export {};

package/build/retrieval/qdrant.js

@@ -0,0 +1,135 @@
+// Qdrant vector search — via direct REST API calls (no npm dependency)
+// Only active when QDRANT_URL is set (via env var or lucid.config.json)
+// Falls back silently to TF-IDF when unavailable
+import { safeFetch } from "../security/ssrf.js";
+// ---------------------------------------------------------------------------
+// Embedding generation (OpenAI-compatible endpoint)
+// ---------------------------------------------------------------------------
+async function embed(texts, cfg) {
+    if (!cfg.embeddingApiKey) {
+        throw new Error("No embedding API key (set OPENAI_API_KEY or embeddingApiKey in lucid.config.json)");
+    }
+    const url = cfg.embeddingUrl ?? "https://api.openai.com/v1/embeddings";
+    const res = await safeFetch(url, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${cfg.embeddingApiKey}`,
+        },
+        body: JSON.stringify({ model: cfg.embeddingModel ?? "text-embedding-3-small", input: texts }),
+    });
+    if (!res.ok) {
+        const body = await res.text();
+        throw new Error(`Embedding API ${res.status}: ${body.slice(0, 200)}`);
+    }
+    const json = await res.json();
+    return json.data.map((d) => d.embedding);
+}
+// ---------------------------------------------------------------------------
+// Qdrant REST helpers
+// ---------------------------------------------------------------------------
+async function qdrantRequest(cfg, method, path, body) {
+    const url = `${cfg.url.replace(/\/$/, "")}${path}`;
+    const headers = { "Content-Type": "application/json" };
+    if (cfg.apiKey)
+        headers["api-key"] = cfg.apiKey;
+    const res = await safeFetch(url, {
+        method,
+        headers,
+        body: body !== undefined ? JSON.stringify(body) : undefined,
+    });
+    if (!res.ok) {
+        const text = await res.text();
+        throw new Error(`Qdrant ${method} ${path} → ${res.status}: ${text.slice(0, 200)}`);
+    }
+    return res.json();
+}
+async function ensureCollection(cfg) {
+    const col = cfg.collection;
+    try {
+        await qdrantRequest(cfg, "GET", `/collections/${col}`);
+    }
+    catch {
+        // Create collection
+        await qdrantRequest(cfg, "PUT", `/collections/${col}`, {
+            vectors: { size: cfg.vectorDim ?? 1536, distance: "Cosine" },
+        });
+    }
+}
+// ---------------------------------------------------------------------------
+// Chunking
+// ---------------------------------------------------------------------------
+const CHUNK_LINES = 80;
+function chunkFile(filepath, text) {
+    const lines = text.split("\n");
+    const chunks = [];
+    for (let i = 0; i < lines.length; i += CHUNK_LINES) {
+        const chunkText = lines.slice(i, i + CHUNK_LINES).join("\n");
+        const chunkIndex = Math.floor(i / CHUNK_LINES);
+        // Deterministic integer ID from filepath + chunk index
+        const id = stableId(`${filepath}::${chunkIndex}`);
+        chunks.push({ id, text: chunkText, chunkIndex });
+    }
+    return chunks;
+}
+function stableId(s) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    // Ensure positive 32-bit int
+    return (h >>> 0) % 2_000_000_000;
+}
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/** Index one file into Qdrant (called by sync_file when Qdrant is configured). */
+export async function indexFileInQdrant(filepath, text, cfg) {
+    await ensureCollection(cfg);
+    const chunks = chunkFile(filepath, text);
+    if (chunks.length === 0)
+        return;
+    // Batch embed (max 96 texts per request for most providers)
+    const BATCH = 32;
+    for (let b = 0; b < chunks.length; b += BATCH) {
+        const batch = chunks.slice(b, b + BATCH);
+        const vectors = await embed(batch.map((c) => c.text), cfg);
+        const points = batch.map((c, idx) => ({
+            id: c.id,
+            vector: vectors[idx],
+            payload: { filepath, chunkIndex: c.chunkIndex, text: c.text },
+        }));
+        await qdrantRequest(cfg, "PUT", `/collections/${cfg.collection}/points`, {
+            points,
+        });
+    }
+}
+/** Top-k semantic search across all indexed chunks. */
+export async function searchQdrant(query, topK, cfg) {
+    const [queryVec] = await embed([query], cfg);
+    if (!queryVec)
+        return [];
+    const result = await qdrantRequest(cfg, "POST", `/collections/${cfg.collection}/points/search`, {
+        vector: queryVec,
+        limit: topK,
+        with_payload: true,
+    });
+    return result.result.map((r) => ({
+        id: r.id,
+        filepath: r.payload["filepath"],
+        chunkIndex: r.payload["chunkIndex"],
+        text: r.payload["text"],
+        score: r.score,
+    }));
+}
+/** Check if Qdrant collection exists and is reachable. */
+export async function pingQdrant(cfg) {
+    try {
+        await qdrantRequest(cfg, "GET", `/collections/${cfg.collection}`);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/build/retrieval/tfidf.d.ts

@@ -0,0 +1,14 @@
+export declare function tokenize(text: string): string[];
+export interface ScoredFile {
+    filepath: string;
+    score: number;
+    matchedTerms: string[];
+}
+/**
+ * Rank files by TF-IDF relevance to a query.
+ * Returns all files sorted by score descending (score=0 files included at bottom).
+ */
+export declare function rankByRelevance(query: string, files: Array<{
+    filepath: string;
+    text: string;
+}>): ScoredFile[];

package/build/retrieval/tfidf.js

@@ -0,0 +1,64 @@
+// TF-IDF scoring — pure JS, no external deps
+// Used as the default relevance engine when Qdrant is not configured
+const STOPWORDS = new Set([
+    "the", "and", "for", "are", "but", "not", "you", "all", "can", "had",
+    "her", "was", "one", "our", "out", "day", "get", "has", "him", "his",
+    "how", "its", "let", "may", "new", "now", "old", "own", "say", "she",
+    "too", "use", "way", "who", "will", "with", "that", "this", "from",
+    "they", "been", "have", "their", "said", "each", "which", "what",
+    // code keywords (too common to be discriminative)
+    "return", "const", "import", "export", "function", "class", "type",
+    "interface", "string", "number", "boolean", "void", "null", "undefined",
+    "async", "await", "true", "false", "default", "module", "require",
+    "self", "def", "pass", "else", "elif", "then", "end", "var", "let",
+]);
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^a-z0-9_]/g, " ")
+        .split(/\s+/)
+        .filter((t) => t.length >= 3 && !STOPWORDS.has(t));
+}
+/**
+ * Rank files by TF-IDF relevance to a query.
+ * Returns all files sorted by score descending (score=0 files included at bottom).
+ */
+export function rankByRelevance(query, files) {
+    if (files.length === 0)
+        return [];
+    const queryTerms = tokenize(query);
+    if (queryTerms.length === 0) {
+        return files.map((f) => ({ filepath: f.filepath, score: 0, matchedTerms: [] }));
+    }
+    const N = files.length;
+    // Compute per-doc term frequencies + document frequencies
+    const df = new Map();
+    const docTF = [];
+    for (const file of files) {
+        const tokens = tokenize(file.text);
+        const tf = new Map();
+        for (const t of tokens)
+            tf.set(t, (tf.get(t) ?? 0) + 1);
+        docTF.push(tf);
+        for (const term of tf.keys())
+            df.set(term, (df.get(term) ?? 0) + 1);
+    }
+    const results = [];
+    for (let i = 0; i < files.length; i++) {
+        const tf = docTF[i];
+        const totalTokens = Math.max([...tf.values()].reduce((a, b) => a + b, 0), 1);
+        let score = 0;
+        const matched = [];
+        for (const qt of queryTerms) {
+            const freq = tf.get(qt) ?? 0;
+            if (freq > 0) {
+                const tfScore = freq / totalTokens;
+                const idf = Math.log((N + 1) / ((df.get(qt) ?? 0) + 1)) + 1;
+                score += tfScore * idf;
+                matched.push(qt);
+            }
+        }
+        results.push({ filepath: files[i].filepath, score, matchedTerms: matched });
+    }
+    return results.sort((a, b) => b.score - a.score);
+}

package/build/security/alerts.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Security alert dispatcher.
+ *
+ * Channels (all optional, configured via lucid-admin.json + env vars):
+ *   - Webhook  (generic HTTP POST, HMAC-SHA256 signed)
+ *   - Slack    (incoming webhook)
+ *   - Email    (SMTP via smtp.ts)
+ *
+ * Sensitive values MUST come from environment variables:
+ *   LUCID_SMTP_PASS        — SMTP password
+ *   LUCID_WEBHOOK_SECRET   — HMAC signing secret for webhook
+ *
+ * Config is stored in  <project>/.claude/lucid-admin.json  (non-sensitive fields only).
+ */
+import type { Severity } from "./waf.js";
+export interface AlertEvent {
+    severity: Severity;
+    rule: string;
+    tool: string;
+    detail: string;
+    timestamp: string;
+    projectDir?: string;
+}
+export interface AdminConfig {
+    adminName?: string;
+    adminEmail?: string;
+    smtpHost?: string;
+    smtpPort?: number;
+    smtpUser?: string;
+    smtpFrom?: string;
+    webhookUrl?: string;
+    slackWebhookUrl?: string;
+    /** Severities that trigger an alert (default: ["critical", "high"]) */
+    alertOn?: Severity[];
+    projectName?: string;
+}
+export declare const ADMIN_CONFIG_FILE = "lucid-admin.json";
+export declare function loadAdminConfig(projectDir: string): AdminConfig;
+export declare function saveAdminConfig(projectDir: string, cfg: AdminConfig): void;
+export declare function getAdminConfig(): AdminConfig;
+export declare function isAdminConfigured(): boolean;
+export declare function sendAlert(event: AlertEvent): Promise<void>;
+/** Send a test alert to verify all configured channels work. */
+export declare function sendTestAlert(projectDir: string): Promise<string[]>;