jstar-reviewer 2.0.0

package/scripts/detective.ts

@@ -0,0 +1,137 @@
+import * as fs from 'fs';
+import * as path from 'path';
+import chalk from 'chalk';
+
+interface Violation {
+    file: string;
+    line: number;
+    message: string;
+    severity: 'high' | 'medium' | 'low';
+    code: string;
+}
+
+interface Rule {
+    id: string;
+    severity: 'high' | 'medium' | 'low';
+    message: string;
+    pattern: RegExp;
+    filePattern?: RegExp; // Only check files matching this pattern
+}
+
+const RULES: Rule[] = [
+    {
+        id: 'SEC-001',
+        severity: 'high',
+        message: 'Possible Hardcoded Secret detected',
+        pattern: /(api_key|secret|password|token)\s*[:=]\s*['"`][a-zA-Z0-9_\-\.]{10,}['"`]/i
+    },
+    {
+        id: 'ARCH-001',
+        severity: 'medium',
+        message: 'Avoid using console.log in production code',
+        pattern: /console\.log\(/
+    },
+];
+
+// File-level rules that check the whole content
+const FILE_RULES: Rule[] = [
+    {
+        id: 'ARCH-002',
+        severity: 'high',
+        message: 'Next.js "use client" must be at the very top of the file',
+        pattern: /^(?!['"]use client['"]).*['"]use client['"]/s,
+        filePattern: /\.tsx?$/
+    }
+];
+
+export class Detective {
+    violations: Violation[] = [];
+
+    constructor(private directory: string) { }
+
+    async scan(): Promise<Violation[]> {
+        this.walk(this.directory);
+        return this.violations;
+    }
+
+    private walk(dir: string) {
+        if (!fs.existsSync(dir)) return;
+        const files = fs.readdirSync(dir);
+        for (const file of files) {
+            const filePath = path.join(dir, file);
+            const stat = fs.statSync(filePath);
+
+            if (stat.isDirectory()) {
+                if (file !== 'node_modules' && file !== '.git' && file !== '.jstar') {
+                    this.walk(filePath);
+                }
+            } else {
+                this.checkFile(filePath);
+            }
+        }
+    }
+
+    private checkFile(filePath: string) {
+        if (!filePath.match(/\.(ts|tsx|js|jsx)$/)) return;
+
+        const content = fs.readFileSync(filePath, 'utf-8');
+        const lines = content.split('\n');
+
+        // Line-based rules
+        for (const rule of RULES) {
+            if (rule.filePattern && !filePath.match(rule.filePattern)) continue;
+
+            lines.forEach((line, index) => {
+                if (rule.pattern.test(line)) {
+                    this.addViolation(filePath, index + 1, rule);
+                }
+            });
+        }
+
+        // File-based rules
+        for (const rule of FILE_RULES) {
+            if (rule.filePattern && !filePath.match(rule.filePattern)) continue;
+            if (rule.pattern.test(content)) {
+                this.addViolation(filePath, 1, rule);
+            }
+        }
+    }
+
+    private addViolation(filePath: string, line: number, rule: Rule) {
+        this.violations.push({
+            file: path.relative(process.cwd(), filePath),
+            line,
+            message: rule.message,
+            severity: rule.severity,
+            code: rule.id
+        });
+    }
+
+    report() {
+        if (this.violations.length === 0) {
+            console.log(chalk.green("✅ Detective Engine: No violations found."));
+            return;
+        }
+
+        console.log(chalk.red(`🚨 Detective Engine found ${this.violations.length} violations:`));
+        // Only show first 10 to avoid wall of text
+        const total = this.violations.length;
+        const toShow = this.violations.slice(0, 10);
+
+        toShow.forEach(v => {
+            const color = v.severity === 'high' ? chalk.red : chalk.yellow;
+            console.log(color(`[${v.code}] ${v.file}:${v.line} - ${v.message}`));
+        });
+
+        if (total > 10) {
+            console.log(chalk.dim(`... and ${total - 10} more.`));
+        }
+    }
+}
+
+// CLI Integration
+if (require.main === module) {
+    const detective = new Detective(path.join(process.cwd(), 'src'));
+    detective.scan();
+    detective.report();
+}

package/scripts/gemini-embedding.ts

@@ -0,0 +1,97 @@
+import { GoogleGenerativeAI } from "@google/generative-ai";
+
+export class GeminiEmbedding {
+    private genAI: GoogleGenerativeAI;
+    private model: any;
+
+    constructor() {
+        const apiKey = process.env.GOOGLE_API_KEY;
+        if (!apiKey) {
+            throw new Error("GOOGLE_API_KEY is missing from environment variables.");
+        }
+        this.genAI = new GoogleGenerativeAI(apiKey);
+        // User requested 'text-embedding-004', which has better rate limits
+        this.model = this.genAI.getGenerativeModel({ model: "text-embedding-004" });
+    }
+
+    async getTextEmbedding(text: string): Promise<number[]> {
+        // Retry logic for transient network errors
+        let retries = 0;
+        const maxRetries = 3;
+        while (retries < maxRetries) {
+            try {
+                const result = await this.model.embedContent(text);
+                return result.embedding.values;
+            } catch (e: any) {
+                if (e.message.includes("fetch failed") || e.message.includes("network")) {
+                    retries++;
+                    const waitTime = Math.pow(2, retries) * 1000;
+                    console.warn(`⚠️ Network error. Retrying in ${waitTime / 1000}s... (${retries}/${maxRetries})`);
+                    await new Promise(resolve => setTimeout(resolve, waitTime));
+                } else {
+                    throw e;
+                }
+            }
+        }
+        throw new Error("Max retries exceeded for embedding request.");
+    }
+
+    async getQueryEmbedding(query: string): Promise<number[]> {
+        return this.getTextEmbedding(query);
+    }
+
+    async getTextEmbeddings(texts: string[]): Promise<number[][]> {
+        const embeddings: number[][] = [];
+        console.log(`Creating embeddings for ${texts.length} chunks (Batching to avoid rate limits)...`);
+
+        // Process in smaller batches with delay
+        const BATCH_SIZE = 1; // Strict serial for safety on free tier
+        const DELAY_MS = 1000; // 1s delay between calls
+
+        for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+            const batch = texts.slice(i, i + BATCH_SIZE);
+            for (const text of batch) {
+                let retries = 0;
+                let success = false;
+                while (!success && retries < 5) {
+                    try {
+                        const embedding = await this.getTextEmbedding(text);
+                        embeddings.push(embedding);
+                        success = true;
+                        // Standard delay between calls
+                        await new Promise(resolve => setTimeout(resolve, DELAY_MS));
+                        process.stdout.write("."); // Progress indicator
+                    } catch (e: any) {
+                        if (e.message.includes("429") || e.message.includes("quota")) {
+                            retries++;
+                            const waitTime = Math.pow(2, retries) * 2000; // 2s, 4s, 8s, 16s...
+                            console.warn(`\n⚠️  Rate limit hit. Retrying in ${waitTime / 1000}s...`);
+                            await new Promise(resolve => setTimeout(resolve, waitTime));
+                        } else {
+                            console.error("\n❌ Embedding failed irreversibly:", e.message);
+                            throw e;
+                        }
+                    }
+                }
+                if (!success) {
+                    throw new Error("Max retries exceeded for rate limits.");
+                }
+            }
+        }
+        console.log("\n✅ Done embedding.");
+        return embeddings;
+    }
+
+    // Stubs for BaseEmbedding compliance
+    embedBatchSize = 10;
+    similarity(embedding1: number[], embedding2: number[]): number {
+        return embedding1.reduce((sum, val, i) => sum + val * embedding2[i], 0);
+    }
+    async transform(nodes: any[], _options?: any): Promise<any[]> {
+        for (const node of nodes) {
+            node.embedding = await this.getTextEmbedding(node.getContent("text"));
+        }
+        return nodes;
+    }
+    async getTextEmbeddingsBatch(texts: string[]): Promise<number[][]> { return this.getTextEmbeddings(texts); }
+}

package/scripts/indexer.ts

@@ -0,0 +1,103 @@
+import {
+    VectorStoreIndex,
+    storageContextFromDefaults,
+    SimpleDirectoryReader,
+    serviceContextFromDefaults
+} from "llamaindex";
+import { GeminiEmbedding } from "./gemini-embedding";
+import { MockLLM } from "./mock-llm";
+import * as path from "path";
+import * as fs from "fs";
+import chalk from "chalk";
+import dotenv from "dotenv";
+
+// Load .env.local first, then .env
+dotenv.config({ path: ".env.local" });
+dotenv.config();
+
+// Configuration
+const STORAGE_DIR = path.join(process.cwd(), ".jstar", "storage");
+const SOURCE_DIR = path.join(process.cwd(), "scripts"); // Changed from src/ to scripts/
+
+// Ensure OpenAI Key exists (LlamaIndex default) or fallback if we configured something else
+// (We are using local now, so this check is less critical, but good to have if we revert)
+// if (!process.env.OPENAI_API_KEY) {
+//     console.warn(chalk.yellow("⚠️  OPENAI_API_KEY not found. Embeddings may fail unless you have configured a local model."));
+// }
+
+async function main() {
+    const args = process.argv.slice(2);
+    const isWatch = args.includes("--watch");
+
+    console.log(chalk.blue("🧠 J-Star Indexer: Scanning codebase..."));
+
+    // 1. Load documents (Your Code)
+    if (!fs.existsSync(SOURCE_DIR)) {
+        console.error(chalk.red(`❌ Source directory not found: ${SOURCE_DIR}`));
+        process.exit(1);
+    }
+
+    const reader = new SimpleDirectoryReader();
+    const documents = await reader.loadData({ directoryPath: SOURCE_DIR });
+
+    console.log(chalk.yellow(`📄 Found ${documents.length} files to index.`));
+
+    const isInit = args.includes("--init");
+
+    try {
+        // 2. Setup Service Context with Google Gemini Embeddings
+        // using 'models/text-embedding-004' which is a strong, recent model
+        const embedModel = new GeminiEmbedding();
+        const llm = new MockLLM();
+        const serviceContext = serviceContextFromDefaults({
+            embedModel,
+            llm: llm as any
+        });
+
+        // 3. Create the Storage Context
+        let storageContext;
+        if (isInit) {
+            console.log(chalk.blue("✨ Initializing fresh Local Brain..."));
+            storageContext = await storageContextFromDefaults({});
+        } else {
+            // Try to load
+            if (!fs.existsSync(STORAGE_DIR)) {
+                console.log(chalk.yellow("⚠️  Storage not found. Running fresh init..."));
+                storageContext = await storageContextFromDefaults({});
+            } else {
+                storageContext = await storageContextFromDefaults({
+                    persistDir: STORAGE_DIR,
+                });
+            }
+        }
+
+        // 4. Generate the Index
+        const index = await VectorStoreIndex.fromDocuments(documents, {
+            storageContext,
+            serviceContext,
+        });
+
+        // 4. Persist (Save the Brain)
+        // Manual persistence for LlamaIndex TS compatibility
+        const ctxToPersist: any = index.storageContext;
+        if (ctxToPersist.docStore) await ctxToPersist.docStore.persist(path.join(STORAGE_DIR, "doc_store.json"));
+        if (ctxToPersist.vectorStore) await ctxToPersist.vectorStore.persist(path.join(STORAGE_DIR, "vector_store.json"));
+        if (ctxToPersist.indexStore) await ctxToPersist.indexStore.persist(path.join(STORAGE_DIR, "index_store.json"));
+        if (ctxToPersist.propStore) await ctxToPersist.propStore.persist(path.join(STORAGE_DIR, "property_store.json"));
+
+        console.log(chalk.green("✅ Indexing Complete. Brain is updated."));
+
+        if (isWatch) {
+            console.log(chalk.blue("👀 Watch mode enabled."));
+        }
+
+    } catch (e: any) {
+        console.error(chalk.red("❌ Indexing Failed:"), e.message);
+        if (e.message.includes("OpenAI")) {
+            console.log(chalk.yellow("👉 Tip: Make sure you have OPENAI_API_KEY in your .env file (or configure a local embedding model)."));
+        }
+        process.exit(1);
+    }
+}
+
+main().catch(console.error);

package/scripts/local-embedding.ts

@@ -0,0 +1,55 @@
+import { pipeline, env } from "@xenova/transformers";
+
+// Skip local model checks if needed, or let it download
+env.allowLocalModels = false;
+env.useBrowserCache = false;
+
+export class LocalEmbedding {
+    private pipe: any;
+    private modelName: string;
+
+    constructor() {
+        this.modelName = "Xenova/bge-small-en-v1.5";
+    }
+
+    async init() {
+        if (!this.pipe) {
+            console.log("📥 Loading local embedding model (Xenova/bge-small-en-v1.5)...");
+            this.pipe = await pipeline("feature-extraction", this.modelName);
+        }
+    }
+
+    async getTextEmbedding(text: string): Promise<number[]> {
+        await this.init();
+        const result = await this.pipe(text, { pooling: "mean", normalize: true });
+        return Array.from(result.data);
+    }
+
+    async getQueryEmbedding(query: string): Promise<number[]> {
+        return this.getTextEmbedding(query);
+    }
+
+    // Batch method (Required by LlamaIndex)
+    async getTextEmbeddings(texts: string[]): Promise<number[][]> {
+        await this.init();
+        const embeddings: number[][] = [];
+        for (const text of texts) {
+            embeddings.push(await this.getTextEmbedding(text));
+        }
+        return embeddings;
+    }
+
+    // Stubs for BaseEmbedding interface compliance
+    embedBatchSize = 10;
+    similarity(embedding1: number[], embedding2: number[]): number {
+        // Simple dot product for normalized vectors
+        return embedding1.reduce((sum, val, i) => sum + val * embedding2[i], 0);
+    }
+    async transform(nodes: any[], _options?: any): Promise<any[]> {
+        for (const node of nodes) {
+            node.embedding = await this.getTextEmbedding(node.getContent("text"));
+        }
+        return nodes;
+    }
+    async getTextEmbeddingsBatch(texts: string[]): Promise<number[][]> { return this.getTextEmbeddings(texts); }
+}

package/scripts/mock-llm.ts

@@ -0,0 +1,18 @@
+export class MockLLM {
+    hasStreaming = false;
+    metadata = {
+        model: "mock",
+        temperature: 0,
+        topP: 1,
+        contextWindow: 1024,
+        tokenizer: undefined,
+    };
+
+    async chat(messages: any[], parentEvent?: any): Promise<any> {
+        return { message: { content: "Mock response" } };
+    }
+
+    async complete(prompt: string, parentEvent?: any): Promise<any> {
+        return { text: "Mock response" };
+    }
+}