alvin-bot 4.4.1

package/dist/services/embeddings.js

@@ -0,0 +1,386 @@
+/**
+ * Embeddings Service — Vector-based semantic memory search.
+ *
+ * Uses Google's text-embedding-004 model for generating embeddings.
+ * Stores embeddings in a local JSON index file for fast cosine similarity search.
+ *
+ * Architecture:
+ * - Each memory entry (paragraph/section) gets an embedding vector
+ * - Vectors are stored in docs/memory/.embeddings.json
+ * - On query, the search text is embedded and compared via cosine similarity
+ * - Top-K results returned with similarity scores
+ */
+import fs from "fs";
+import path from "path";
+import { resolve } from "path";
+import { config } from "../config.js";
+import os from "os";
+import { MEMORY_DIR, MEMORY_FILE, EMBEDDINGS_IDX as INDEX_FILE } from "../paths.js";
+import { ASSETS_DIR, ASSETS_INDEX_MD } from "../paths.js";
+// Hub memory directory (Claude Hub — read-only, additional context)
+const HUB_MEMORY_DIR = resolve(os.homedir(), ".claude", "hub", "MEMORY");
+// ── Google Embeddings API ───────────────────────────────
+const EMBEDDING_MODEL = "gemini-embedding-001";
+const EMBEDDING_DIMENSION = 3072;
+/**
+ * Get embeddings for one or more texts via Google's API.
+ * Batches up to 100 texts per request.
+ */
+async function getEmbeddings(texts) {
+    const apiKey = config.apiKeys.google;
+    if (!apiKey) {
+        throw new Error("Google API key not configured. Set GOOGLE_API_KEY in .env");
+    }
+    const results = [];
+    const batchSize = 100;
+    for (let i = 0; i < texts.length; i += batchSize) {
+        const batch = texts.slice(i, i + batchSize);
+        const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${EMBEDDING_MODEL}:batchEmbedContents?key=${apiKey}`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({
+                requests: batch.map(text => ({
+                    model: `models/${EMBEDDING_MODEL}`,
+                    content: { parts: [{ text }] },
+                    taskType: "RETRIEVAL_DOCUMENT",
+                })),
+            }),
+        });
+        if (!response.ok) {
+            const err = await response.text();
+            throw new Error(`Embedding API error: ${response.status} — ${err}`);
+        }
+        const data = await response.json();
+        for (const emb of data.embeddings) {
+            results.push(emb.values);
+        }
+    }
+    return results;
+}
+/**
+ * Get embedding for a single query text.
+ */
+async function getQueryEmbedding(text) {
+    const apiKey = config.apiKeys.google;
+    if (!apiKey) {
+        throw new Error("Google API key not configured");
+    }
+    const response = await fetch(`https://generativelanguage.googleapis.com/v1beta/models/${EMBEDDING_MODEL}:embedContent?key=${apiKey}`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({
+            model: `models/${EMBEDDING_MODEL}`,
+            content: { parts: [{ text }] },
+            taskType: "RETRIEVAL_QUERY",
+        }),
+    });
+    if (!response.ok) {
+        const err = await response.text();
+        throw new Error(`Embedding API error: ${response.status} — ${err}`);
+    }
+    const data = await response.json();
+    return data.embedding.values;
+}
+// ── Vector Math ─────────────────────────────────────────
+function cosineSimilarity(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dotProduct += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dotProduct / denom;
+}
+// ── Text Chunking ───────────────────────────────────────
+/**
+ * Split a markdown file into meaningful chunks.
+ * Splits on ## headers, keeping each section as a chunk.
+ * Falls back to paragraph splitting for files without headers.
+ */
+function chunkMarkdown(content, source) {
+    const chunks = [];
+    // Split on ## headers
+    const sections = content.split(/^(?=## )/gm);
+    for (let i = 0; i < sections.length; i++) {
+        const section = sections[i].trim();
+        if (!section || section.length < 20)
+            continue; // Skip tiny sections
+        // If section is too long (>1000 chars), split into paragraphs
+        if (section.length > 1000) {
+            const paragraphs = section.split(/\n\n+/);
+            let currentChunk = "";
+            let chunkIdx = 0;
+            for (const para of paragraphs) {
+                if (currentChunk.length + para.length > 800 && currentChunk.length > 100) {
+                    chunks.push({
+                        id: `${source}:${i}:${chunkIdx}`,
+                        text: currentChunk.trim(),
+                    });
+                    currentChunk = "";
+                    chunkIdx++;
+                }
+                currentChunk += para + "\n\n";
+            }
+            if (currentChunk.trim().length > 20) {
+                chunks.push({
+                    id: `${source}:${i}:${chunkIdx}`,
+                    text: currentChunk.trim(),
+                });
+            }
+        }
+        else {
+            chunks.push({
+                id: `${source}:${i}`,
+                text: section,
+            });
+        }
+    }
+    return chunks;
+}
+// ── Index Management ────────────────────────────────────
+function loadIndex() {
+    try {
+        const raw = fs.readFileSync(INDEX_FILE, "utf-8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return {
+            model: EMBEDDING_MODEL,
+            lastReindex: 0,
+            fileMtimes: {},
+            entries: [],
+        };
+    }
+}
+function saveIndex(index) {
+    fs.writeFileSync(INDEX_FILE, JSON.stringify(index));
+}
+/**
+ * Recursively walk a directory, returning file paths.
+ * Skips INDEX.json and INDEX.md at the directory root.
+ */
+function walkAssetDir(dir) {
+    const results = [];
+    function walk(currentDir) {
+        let entries;
+        try {
+            entries = fs.readdirSync(currentDir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            const fullPath = resolve(currentDir, entry.name);
+            if (entry.isDirectory()) {
+                walk(fullPath);
+            }
+            else if (entry.isFile()) {
+                if (currentDir === dir && (entry.name === "INDEX.json" || entry.name === "INDEX.md"))
+                    continue;
+                results.push({ name: entry.name, path: fullPath });
+            }
+        }
+    }
+    walk(dir);
+    return results;
+}
+const TEXT_EXTENSIONS = new Set([".md", ".html", ".txt", ".css", ".ts"]);
+/**
+ * Get all files that should be indexed — memories + text-based assets.
+ */
+function getIndexableFiles() {
+    const files = [];
+    // ── Memories (existing) ───────────────────────────────
+    // Alvin-Bot MEMORY.md
+    if (fs.existsSync(MEMORY_FILE)) {
+        files.push({ path: MEMORY_FILE, relativePath: "MEMORY.md" });
+    }
+    // Alvin-Bot daily logs
+    if (fs.existsSync(MEMORY_DIR)) {
+        const entries = fs.readdirSync(MEMORY_DIR);
+        for (const entry of entries) {
+            if (entry.endsWith(".md") && !entry.startsWith(".")) {
+                files.push({
+                    path: resolve(MEMORY_DIR, entry),
+                    relativePath: `memory/${entry}`,
+                });
+            }
+        }
+    }
+    // Hub memories (~/.claude/hub/MEMORY/) — Claude Hub knowledge base
+    if (fs.existsSync(HUB_MEMORY_DIR)) {
+        try {
+            const entries = fs.readdirSync(HUB_MEMORY_DIR);
+            for (const entry of entries) {
+                if (entry.endsWith(".md") && !entry.startsWith(".")) {
+                    files.push({
+                        path: resolve(HUB_MEMORY_DIR, entry),
+                        relativePath: `hub/${entry}`,
+                    });
+                }
+            }
+        }
+        catch { /* Hub not available — skip */ }
+    }
+    // ── Assets (new) ──────────────────────────────────────
+    // Asset INDEX.md — compact summary of all assets
+    if (fs.existsSync(ASSETS_INDEX_MD)) {
+        files.push({ path: ASSETS_INDEX_MD, relativePath: "assets/INDEX.md" });
+    }
+    // Text-based asset files (HTML, MD, TXT, CSS, TS)
+    if (fs.existsSync(ASSETS_DIR)) {
+        for (const entry of walkAssetDir(ASSETS_DIR)) {
+            if (TEXT_EXTENSIONS.has(path.extname(entry.name))) {
+                files.push({
+                    path: entry.path,
+                    relativePath: `assets/${path.relative(ASSETS_DIR, entry.path)}`,
+                });
+            }
+        }
+    }
+    return files;
+}
+/**
+ * Check which files need reindexing (new or modified).
+ */
+function getStaleFiles(index) {
+    const allFiles = getIndexableFiles();
+    const stale = [];
+    for (const file of allFiles) {
+        try {
+            const stat = fs.statSync(file.path);
+            const mtime = stat.mtimeMs;
+            if (!index.fileMtimes[file.relativePath] || index.fileMtimes[file.relativePath] < mtime) {
+                stale.push(file);
+            }
+        }
+        catch {
+            // File disappeared — skip
+        }
+    }
+    return stale;
+}
+// ── Public API ──────────────────────────────────────────
+/**
+ * Reindex all memory files (or just stale ones).
+ * Returns number of chunks indexed.
+ */
+export async function reindexMemory(force = false) {
+    const index = loadIndex();
+    const filesToIndex = force ? getIndexableFiles() : getStaleFiles(index);
+    if (filesToIndex.length === 0) {
+        return { indexed: 0, total: index.entries.length };
+    }
+    // Remove old entries for files being reindexed
+    const reindexSources = new Set(filesToIndex.map(f => f.relativePath));
+    index.entries = index.entries.filter(e => !reindexSources.has(e.source));
+    // Chunk all files
+    const allChunks = [];
+    for (const file of filesToIndex) {
+        try {
+            const content = fs.readFileSync(file.path, "utf-8");
+            const chunks = chunkMarkdown(content, file.relativePath);
+            for (const chunk of chunks) {
+                allChunks.push({ ...chunk, source: file.relativePath });
+            }
+            // Update mtime
+            const stat = fs.statSync(file.path);
+            index.fileMtimes[file.relativePath] = stat.mtimeMs;
+        }
+        catch (err) {
+            console.error(`Failed to chunk ${file.relativePath}:`, err);
+        }
+    }
+    if (allChunks.length === 0) {
+        saveIndex(index);
+        return { indexed: 0, total: index.entries.length };
+    }
+    // Get embeddings for all chunks
+    const texts = allChunks.map(c => c.text);
+    const vectors = await getEmbeddings(texts);
+    // Add to index
+    for (let i = 0; i < allChunks.length; i++) {
+        index.entries.push({
+            id: allChunks[i].id,
+            source: allChunks[i].source,
+            text: allChunks[i].text,
+            vector: vectors[i],
+            indexedAt: Date.now(),
+        });
+    }
+    index.lastReindex = Date.now();
+    saveIndex(index);
+    return { indexed: allChunks.length, total: index.entries.length };
+}
+/**
+ * Semantic search across all indexed memory.
+ * Returns top-K results sorted by similarity.
+ */
+export async function searchMemory(query, topK = 5, minScore = 0.3) {
+    const index = loadIndex();
+    if (index.entries.length === 0) {
+        // Auto-index if empty
+        await reindexMemory();
+        // Reload
+        const reloaded = loadIndex();
+        if (reloaded.entries.length === 0)
+            return [];
+    }
+    // Get query embedding
+    const queryVector = await getQueryEmbedding(query);
+    // Calculate similarities
+    const scored = index.entries.map(entry => ({
+        text: entry.text,
+        source: entry.source,
+        score: cosineSimilarity(queryVector, entry.vector),
+    }));
+    // Sort by score descending, filter by minScore, take topK
+    return scored
+        .filter(r => r.score >= minScore)
+        .sort((a, b) => b.score - a.score)
+        .slice(0, topK);
+}
+/**
+ * Get index stats for /status.
+ */
+/**
+ * Auto-reindex on startup. Indexes only stale/new files (incremental).
+ * Runs in background — does not block bot startup.
+ */
+export async function initEmbeddings() {
+    try {
+        const stale = getStaleFiles(loadIndex());
+        if (stale.length === 0) {
+            const idx = loadIndex();
+            if (idx.entries.length > 0)
+                return; // Already indexed, nothing stale
+        }
+        const result = await reindexMemory();
+        if (result.indexed > 0) {
+            console.log(`🔍 Embeddings: indexed ${result.indexed} chunks (${result.total} total)`);
+        }
+    }
+    catch (err) {
+        // Non-fatal — bot works without embeddings
+        console.warn("⚠️ Embeddings init failed:", err instanceof Error ? err.message : err);
+    }
+}
+export function getIndexStats() {
+    const index = loadIndex();
+    let sizeBytes = 0;
+    try {
+        sizeBytes = fs.statSync(INDEX_FILE).size;
+    }
+    catch { /* empty */ }
+    return {
+        entries: index.entries.length,
+        files: Object.keys(index.fileMtimes).length,
+        lastReindex: index.lastReindex,
+        sizeBytes,
+    };
+}

package/dist/services/exec-guard.js

@@ -0,0 +1,46 @@
+import { readFileSync, existsSync } from "fs";
+import { config } from "../config.js";
+import { EXEC_ALLOWLIST_FILE } from "../paths.js";
+const SAFE_BINS = [
+    "ls", "cat", "head", "tail", "grep", "rg", "find", "wc", "sort", "uniq",
+    "echo", "printf", "date", "which", "whoami", "hostname", "uname",
+    "curl", "wget", "git", "node", "npm", "npx", "python3", "pip3",
+    "jq", "ffmpeg", "ffprobe", "ffplay",
+    "mkdir", "touch", "cp", "mv", "ln", "chmod",
+    "tar", "zip", "unzip", "gzip", "gunzip",
+    "ssh", "scp", "rsync",
+    "docker", "docker-compose",
+    "brew", "open", "pbcopy", "pbpaste",
+    "osascript", "defaults", "launchctl",
+];
+function loadUserAllowlist() {
+    if (!existsSync(EXEC_ALLOWLIST_FILE))
+        return [];
+    try {
+        return JSON.parse(readFileSync(EXEC_ALLOWLIST_FILE, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+}
+function extractBinary(command) {
+    // Get first word, strip env vars, handle pipes
+    const cleaned = command.replace(/^(env\s+\w+=\S+\s+)+/, "").trim();
+    const first = cleaned.split(/[\s|;&]/)[0];
+    // Strip path: /usr/bin/curl -> curl
+    return first.split("/").pop() || first;
+}
+export function checkExecAllowed(command) {
+    if (config.execSecurity === "full")
+        return { allowed: true };
+    if (config.execSecurity === "deny")
+        return { allowed: false, reason: "Shell execution is disabled" };
+    // allowlist mode
+    const binary = extractBinary(command);
+    if (SAFE_BINS.includes(binary))
+        return { allowed: true };
+    const userList = loadUserAllowlist();
+    if (userList.includes(binary))
+        return { allowed: true };
+    return { allowed: false, reason: `Binary "${binary}" not in allowlist. Add to ${EXEC_ALLOWLIST_FILE} or set EXEC_SECURITY=full` };
+}

package/dist/services/fallback-order.js

@@ -0,0 +1,151 @@
+/**
+ * Fallback Order Manager — Persistent, user-configurable provider fallback chain.
+ *
+ * Supports reading/writing the fallback order from:
+ * - Telegram (/fallback command)
+ * - Web UI (API endpoint)
+ * - CLI/Terminal
+ *
+ * Persists to docs/fallback-order.json and syncs with .env FALLBACK_PROVIDERS.
+ */
+import fs from "fs";
+import { FALLBACK_FILE, ENV_FILE, DATA_DIR } from "../paths.js";
+// ── Public API ──────────────────────────────────────────────────────────────
+/**
+ * Get the current fallback order.
+ */
+export function getFallbackOrder() {
+    try {
+        if (fs.existsSync(FALLBACK_FILE)) {
+            return JSON.parse(fs.readFileSync(FALLBACK_FILE, "utf-8"));
+        }
+    }
+    catch { /* ignore */ }
+    // Default from env
+    return {
+        primary: process.env.PRIMARY_PROVIDER || "groq",
+        fallbacks: (process.env.FALLBACK_PROVIDERS || "")
+            .split(",")
+            .map(s => s.trim())
+            .filter(Boolean),
+        updatedAt: new Date().toISOString(),
+        updatedBy: "env",
+    };
+}
+/**
+ * Set the fallback order.
+ * Updates both docs/fallback-order.json and .env file.
+ */
+export function setFallbackOrder(primary, fallbacks, updatedBy = "unknown") {
+    const config = {
+        primary,
+        fallbacks,
+        updatedAt: new Date().toISOString(),
+        updatedBy,
+    };
+    // Ensure data dir exists
+    if (!fs.existsSync(DATA_DIR)) {
+        fs.mkdirSync(DATA_DIR, { recursive: true });
+    }
+    // Write JSON
+    fs.writeFileSync(FALLBACK_FILE, JSON.stringify(config, null, 2));
+    // Sync to .env
+    syncToEnv(primary, fallbacks);
+    return config;
+}
+/**
+ * Move a provider up in the fallback order.
+ */
+export function moveUp(providerKey, updatedBy = "unknown") {
+    const current = getFallbackOrder();
+    const idx = current.fallbacks.indexOf(providerKey);
+    if (idx > 0) {
+        // Swap with previous
+        [current.fallbacks[idx - 1], current.fallbacks[idx]] =
+            [current.fallbacks[idx], current.fallbacks[idx - 1]];
+    }
+    else if (idx === 0) {
+        // Move to primary, old primary becomes first fallback
+        const oldPrimary = current.primary;
+        current.primary = providerKey;
+        current.fallbacks[0] = oldPrimary;
+    }
+    return setFallbackOrder(current.primary, current.fallbacks, updatedBy);
+}
+/**
+ * Move a provider down in the fallback order.
+ */
+export function moveDown(providerKey, updatedBy = "unknown") {
+    const current = getFallbackOrder();
+    if (providerKey === current.primary && current.fallbacks.length > 0) {
+        // Move primary to first fallback, first fallback becomes primary
+        const newPrimary = current.fallbacks[0];
+        current.fallbacks[0] = providerKey;
+        current.primary = newPrimary;
+    }
+    else {
+        const idx = current.fallbacks.indexOf(providerKey);
+        if (idx >= 0 && idx < current.fallbacks.length - 1) {
+            [current.fallbacks[idx], current.fallbacks[idx + 1]] =
+                [current.fallbacks[idx + 1], current.fallbacks[idx]];
+        }
+    }
+    return setFallbackOrder(current.primary, current.fallbacks, updatedBy);
+}
+/**
+ * Add a provider to the fallback chain (at the end).
+ */
+export function addFallback(providerKey, updatedBy = "unknown") {
+    const current = getFallbackOrder();
+    if (!current.fallbacks.includes(providerKey) && providerKey !== current.primary) {
+        current.fallbacks.push(providerKey);
+    }
+    return setFallbackOrder(current.primary, current.fallbacks, updatedBy);
+}
+/**
+ * Remove a provider from the fallback chain.
+ */
+export function removeFallback(providerKey, updatedBy = "unknown") {
+    const current = getFallbackOrder();
+    current.fallbacks = current.fallbacks.filter(k => k !== providerKey);
+    return setFallbackOrder(current.primary, current.fallbacks, updatedBy);
+}
+/**
+ * Format the current order as a human-readable string.
+ */
+export function formatOrder() {
+    const config = getFallbackOrder();
+    const lines = [];
+    lines.push(`1. 🥇 ${config.primary} (Primary)`);
+    config.fallbacks.forEach((fb, i) => {
+        lines.push(`${i + 2}. ${i === 0 ? "🥈" : i === 1 ? "🥉" : "  "} ${fb}`);
+    });
+    return lines.join("\n");
+}
+// ── Internal ────────────────────────────────────────────────────────────────
+function syncToEnv(primary, fallbacks) {
+    try {
+        if (!fs.existsSync(ENV_FILE))
+            return;
+        let env = fs.readFileSync(ENV_FILE, "utf-8");
+        // Update PRIMARY_PROVIDER
+        if (env.match(/^PRIMARY_PROVIDER=.*/m)) {
+            env = env.replace(/^PRIMARY_PROVIDER=.*/m, `PRIMARY_PROVIDER=${primary}`);
+        }
+        else {
+            env += `\nPRIMARY_PROVIDER=${primary}`;
+        }
+        // Update FALLBACK_PROVIDERS
+        const fallbackStr = fallbacks.join(",");
+        if (env.match(/^FALLBACK_PROVIDERS=.*/m)) {
+            env = env.replace(/^FALLBACK_PROVIDERS=.*/m, `FALLBACK_PROVIDERS=${fallbackStr}`);
+        }
+        else {
+            env += `\nFALLBACK_PROVIDERS=${fallbackStr}`;
+        }
+        fs.writeFileSync(ENV_FILE, env);
+    }
+    catch (err) {
+        console.error("Failed to sync fallback order to .env:", err);
+    }
+}