agen-vektor 0.3.30 → 0.3.31

package/dist/agent/agent.js

@@ -5,6 +5,7 @@ const factory_1 = require("../providers/factory");
 const registry_1 = require("../tools/registry");
 const filesystem_1 = require("../tools/filesystem");
 const search_1 = require("../tools/search");
+const semantic_search_1 = require("../tools/semantic-search");
 const shell_1 = require("../tools/shell");
 const git_1 = require("../tools/git");
 const web_1 = require("../tools/web");
@@ -59,6 +60,9 @@ class Agent {
         this.tools.registerMany([
             ...(0, filesystem_1.createFilesystemTools)(),
             ...(0, search_1.createSearchTools)(),
+            // Semantic code search (via gateway embeddings relay): semantic_index /
+            // semantic_search — cari kode by MEANING, zero-config (key di gateway).
+            ...(0, semantic_search_1.createSemanticSearchTools)(),
             (0, shell_1.createShellTool)(),
             (0, git_1.createGitTool)(),
             ...(0, web_1.createWebTools)(),

package/dist/tools/semantic-search.js

@@ -0,0 +1,276 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSemanticSearchTools = createSemanticSearchTools;
+/**
+ * Semantic code search — "agent yang hafal codebase" (2026-09-14).
+ *
+ * Alur: chunk file proyek → embedding via GATEWAY relay (/v1/embeddings,
+ * zero-config & zero-key — key provider sk- hidup di gateway, pola sama
+ * dengan e2b.ts/web.ts) → simpan vektor di file lokal (~/.vector/index.json)
+ * → cosine similarity saat query → top-k potongan paling relevan.
+ *
+ * Zero dependency: embedding = HTTP call, similarity = dot product manual,
+ * storage = satu file JSON. Index dibangun manual via tool semantic_index
+ * (atau otomatis saat semantic_search pertama tanpa index).
+ *
+ * Batas wajar (anti-abuse & anti-bloat):
+ * - max file 400 (arg hingga 2000), skip >200KB & file binary-ish
+ * - max chunk 4000; batch embedding 16 input/request; jeda antar-batch
+ *   1.1s (rate limit gateway 60/menit — env VECTOR_EMBED_DELAY_MS utk test)
+ * - skip nama file sensitif (credential/secret/.env/key/pem) — index tidak
+ *   boleh jadi salinan rahasia
+ */
+const fs = __importStar(require("node:fs"));
+const path = __importStar(require("node:path"));
+const free_tier_1 = require("../config/free-tier");
+const paths_1 = require("../utils/paths");
+const GW_BASE = free_tier_1.FREE_GATEWAY_URL.replace(/\/v1$/, '');
+const EMBED_URL = GW_BASE + '/v1/embeddings';
+const DEFAULT_EMBED_MODEL = 'text-embedding-v4';
+const IGNORE = new Set(['.git', 'node_modules', 'vendor', 'dist', 'build', '.cache', '.next', 'target', 'coverage', 'sessions', 'logs', 'memory']);
+const TEXT_EXT = new Set(['.ts', '.tsx', '.js', '.jsx', '.mjs', '.cjs', '.json', '.md', '.py', '.go', '.rs', '.java', '.rb', '.php', '.sh', '.bash', '.yml', '.yaml', '.toml', '.html', '.css', '.scss', '.sql', '.c', '.h', '.cpp', '.hpp', '.cs', '.swift', '.kt', '.txt', '.dockerfile', '.ini', '.cfg']);
+const SENSITIVE = /credential|secret|\.env|\.pem|\.key|id_rsa|password/i;
+const MAX_FILE_BYTES = 200_000;
+const MAX_FILES_DEFAULT = 400;
+const MAX_FILES_HARD = 2000;
+const MAX_CHUNKS = 4000;
+const CHUNK_LINES = 64;
+const BATCH = 16;
+const DELAY_MS = Math.max(0, Number(process.env.VECTOR_EMBED_DELAY_MS ?? 1100));
+function indexPath() {
+    return path.join((0, paths_1.getVectorDir)(), 'index.json');
+}
+function loadIndex() {
+    try {
+        const raw = fs.readFileSync(indexPath(), 'utf8');
+        const j = JSON.parse(raw);
+        return j && j.version === 1 && Array.isArray(j.chunks) ? j : null;
+    }
+    catch {
+        return null;
+    }
+}
+function saveIndex(idx) {
+    fs.mkdirSync((0, paths_1.getVectorDir)(), { recursive: true });
+    fs.writeFileSync(indexPath(), JSON.stringify(idx), { mode: 0o600 });
+}
+function walkFiles(dir, out, max) {
+    let entries;
+    try {
+        entries = fs.readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return;
+    }
+    for (const e of entries) {
+        if (out.length >= max)
+            return;
+        if (IGNORE.has(e.name) || e.name.startsWith('.'))
+            continue;
+        if (SENSITIVE.test(e.name))
+            continue;
+        const full = path.join(dir, e.name);
+        if (e.isDirectory())
+            walkFiles(full, out, max);
+        else if (TEXT_EXT.has(path.extname(e.name).toLowerCase()) || e.name === 'Dockerfile')
+            out.push(full);
+    }
+}
+function chunkText(content) {
+    const lines = content.split('\n');
+    const chunks = [];
+    for (let i = 0; i < lines.length && chunks.length < MAX_CHUNKS; i += CHUNK_LINES) {
+        const slice = lines.slice(i, i + CHUNK_LINES);
+        const text = slice.join('\n').trim();
+        if (text)
+            chunks.push({ start: i + 1, end: i + slice.length, text });
+    }
+    return chunks;
+}
+async function embed(inputs, model) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), 30_000);
+    try {
+        const res = await fetch(EMBED_URL, {
+            method: 'POST',
+            signal: controller.signal,
+            headers: { 'Content-Type': 'application/json', 'User-Agent': 'VectorHead-AI-Agent/0.1' },
+            body: JSON.stringify({ model, input: inputs }),
+        });
+        const data = await res.json().catch(() => null);
+        if (!res.ok || !data || !Array.isArray(data.data)) {
+            const snippet = data && data.error ? JSON.stringify(data.error).slice(0, 200) : `HTTP ${res.status}`;
+            throw new Error(`gateway embeddings gagal: ${snippet}`);
+        }
+        // OpenAI format: data[i].embedding — urut by index utk aman.
+        const sorted = [...data.data].sort((a, b) => (a.index ?? 0) - (b.index ?? 0));
+        return sorted.map((d) => d.embedding);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+function cosine(a, b) {
+    let dot = 0, na = 0, nb = 0;
+    const n = Math.min(a.length, b.length);
+    for (let i = 0; i < n; i++) {
+        dot += a[i] * b[i];
+        na += a[i] * a[i];
+        nb += b[i] * b[i];
+    }
+    return na && nb ? dot / (Math.sqrt(na) * Math.sqrt(nb)) : 0;
+}
+async function buildIndex(root, maxFiles) {
+    const files = [];
+    walkFiles(root, files, maxFiles);
+    if (!files.length)
+        return `ERROR: tidak ada file teks yang bisa di-index di ${root}`;
+    const model = DEFAULT_EMBED_MODEL;
+    const chunks = [];
+    for (const full of files) {
+        let content;
+        try {
+            if (fs.statSync(full).size > MAX_FILE_BYTES)
+                continue;
+            content = fs.readFileSync(full, 'utf8');
+        }
+        catch {
+            continue;
+        }
+        if (content.includes('\u0000'))
+            continue; // binary-ish
+        const rel = path.relative(root, full).split(path.sep).join('/');
+        for (const c of chunkText(content))
+            chunks.push({ file: rel, ...c, embedding: [] });
+    }
+    if (!chunks.length)
+        return 'ERROR: tidak ada chunk yang bisa di-index';
+    for (let i = 0; i < chunks.length; i += BATCH) {
+        const vecs = await embed(chunks.slice(i, i + BATCH).map((c) => c.text), model);
+        if (vecs.length !== Math.min(BATCH, chunks.length - i)) {
+            return `ERROR: jumlah vektor upstream tidak cocok (${vecs.length})`;
+        }
+        vecs.forEach((v, j) => { chunks[i + j].embedding = v; });
+        if (i + BATCH < chunks.length && DELAY_MS)
+            await sleep(DELAY_MS);
+    }
+    saveIndex({ version: 1, model, builtAt: Date.now(), chunks });
+    const preview = [...new Set(chunks.map((c) => c.file))].slice(0, 5).join(', ');
+    return `OK: index dibangun — ${chunks.length} chunk dari ${files.length} file (model ${model}) → ${indexPath()}${preview ? '\nContoh: ' + preview : ''}`;
+}
+function searchIndex(idx, queryVec, k) {
+    const scored = idx.chunks
+        .map((c) => ({ c, score: cosine(queryVec, c.embedding) }))
+        .sort((a, b) => b.score - a.score)
+        .slice(0, Math.max(1, k));
+    if (!scored.length || scored[0].score <= 0) {
+        return 'Tidak ada hasil relevan (index kosong / skor 0) — coba semantic_index ulang.';
+    }
+    const lines = scored.map((s, i) => {
+        const head = s.c.text.replace(/\s+/g, ' ').slice(0, 180);
+        return `${i + 1}. ${s.c.file}:${s.c.start}-${s.c.end} (skor ${s.score.toFixed(3)})\n   ${head}`;
+    });
+    return `Hasil semantic search (top ${scored.length} dari ${idx.chunks.length} chunk):\n${lines.join('\n')}`;
+}
+function createSemanticSearchTools() {
+    return [
+        {
+            definition: {
+                name: 'semantic_index',
+                description: 'Build/refresh the local semantic index: embed project files (via the VectorHead gateway, no API key needed) and store vectors in ~/.vector/index.json. Run once per project (or after big changes) — then semantic_search can find code by MEANING, not just keywords.',
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        path: { type: 'string', description: 'Directory to index, relative to project root (default ".")' },
+                        max_files: { type: 'number', description: 'Max files to index (default 400, max 2000)' },
+                    },
+                    required: [],
+                },
+            },
+            async execute(args, ctx) {
+                const base = path.resolve(ctx.cwd, String(args.path || '.'));
+                const max = Math.min(MAX_FILES_HARD, Math.max(1, Number(args.max_files) || MAX_FILES_DEFAULT));
+                try {
+                    return { output: await buildIndex(base, max) };
+                }
+                catch (e) {
+                    return { output: 'ERROR: ' + String(e.message || e) };
+                }
+            },
+        },
+        {
+            definition: {
+                name: 'semantic_search',
+                description: 'Search the codebase by MEANING (semantic similarity over embedded chunks). Automatically builds the index first if none exists. Use for "where is the retry logic?" style questions where grep/keywords fail.',
+                parameters: {
+                    type: 'object',
+                    properties: {
+                        query: { type: 'string', description: 'Natural-language query, e.g. "where do we validate the device id"' },
+                        k: { type: 'number', description: 'Max results (default 5, max 20)' },
+                        path: { type: 'string', description: 'Directory that was indexed (default ".")' },
+                    },
+                    required: ['query'],
+                },
+            },
+            async execute(args, ctx) {
+                const query = String(args.query || '').trim();
+                if (!query)
+                    return { output: 'ERROR: query wajib' };
+                const k = Math.min(20, Math.max(1, Number(args.k) || 5));
+                try {
+                    let idx = loadIndex();
+                    if (!idx) {
+                        const base = path.resolve(ctx.cwd, String(args.path || '.'));
+                        const built = await buildIndex(base, MAX_FILES_DEFAULT);
+                        if (built.startsWith('ERROR'))
+                            return { output: built };
+                        idx = loadIndex();
+                        if (!idx)
+                            return { output: 'ERROR: index gagal dibaca setelah build' };
+                    }
+                    const [qv] = await embed([query], idx.model || DEFAULT_EMBED_MODEL);
+                    if (!qv)
+                        return { output: 'ERROR: embedding query kosong' };
+                    return { output: searchIndex(idx, qv, k) };
+                }
+                catch (e) {
+                    return { output: 'ERROR: ' + String(e.message || e) };
+                }
+            },
+        },
+    ];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agen-vektor",
-  "version": "0.3.30",
+  "version": "0.3.31",
   "description": "VectorHead (agen-vektor) — AI Coding Agent CLI/TUI for Linux & Termux. Multi-provider, tool calling, session, permission system.",
   "type": "commonjs",
   "bin": {