inbed 1.0.1 → 1.0.3

package/dist/Inbed.js

@@ -7,10 +7,8 @@ import ts from 'typescript';
 export class Inbed {
     constructor(embedder, options) {
         this.files = new Map();
-        // concurrency
         this.embeddingQueue = [];
         this.maxConcurrentEmbeds = 4;
-        // watcher debounce
         this.debounceMap = new Map();
         this.options = {
             recursive: true,
@@ -129,7 +127,6 @@ export class Inbed {
         if (depth > (this.options.maxDepth || 10))
             return;
         const imports = [];
-        // captura import e require
         const importRegex = /import\s+(?:[^'"\n]+)\s+from\s+['"](.+)['"]|require\(\s*['"](.+)['"]\s*\)/g;
         let match;
         while ((match = importRegex.exec(file.content)) !== null) {
@@ -141,13 +138,11 @@ export class Inbed {
                 const candidate = path.normalize(base + ext);
                 if (this.files.has(candidate)) {
                     imports.push({ source: sourceImport, resolved: candidate });
-                    // recursão
                     this.resolveImports(this.files.get(candidate), depth + 1);
                     break;
                 }
             }
         }
-        // deduplicação
         file.imports = imports.filter((v, i, arr) => arr.findIndex(x => x.resolved === v.resolved) === i);
     }
     async upsertFile(filePath, content) {
@@ -175,16 +170,8 @@ export class Inbed {
                 }
             }
             if (bestScore > 0) {
-                const importsText = Array.isArray(file.imports)
-                    ? file.imports
-                        .map((imp) => typeof imp === 'string' ? imp : `${imp.source} -> ${imp.resolved}`)
-                        .join('\n')
-                    : '';
-                const context = `FILE: ${file.path}
-IMPORTS:
-${importsText || '(none)'}
----
-${bestChunk}`;
+                const importsText = file.imports.map(imp => `${imp.source} -> ${imp.resolved}`).join('\n');
+                const context = `FILE: ${file.path}\nIMPORTS:\n${importsText || '(none)'}\n---\n${bestChunk}`;
                 results.push({ path: file.path, snippet: context.slice(0, 500), score: bestScore });
             }
         }
@@ -209,27 +196,31 @@ ${bestChunk}`;
             this.debounceMap.set(fullPath, setTimeout(handler, 200));
         };
         this.watcher
-            .on('add', fullPath => schedule(fullPath, async () => {
+            .on('add', (fullPath) => schedule(fullPath, async () => {
             const relPath = path.relative(this.options.rootDir, fullPath);
             if (this.options.fileExtensions?.some(ext => fullPath.endsWith(ext))) {
                 try {
                     const content = fs.readFileSync(fullPath, 'utf-8');
                     await this.enqueueAdd(relPath, content);
+                    if (this.options.recursive)
+                        this.resolveImports(this.files.get(relPath), 0);
                 }
                 catch { }
             }
         }))
-            .on('change', fullPath => schedule(fullPath, async () => {
+            .on('change', (fullPath) => schedule(fullPath, async () => {
             const relPath = path.relative(this.options.rootDir, fullPath);
             if (this.options.fileExtensions?.some(ext => fullPath.endsWith(ext))) {
                 try {
                     const content = fs.readFileSync(fullPath, 'utf-8');
                     await this.enqueueAdd(relPath, content);
+                    if (this.options.recursive)
+                        this.resolveImports(this.files.get(relPath), 0);
                 }
                 catch { }
             }
         }))
-            .on('unlink', fullPath => {
+            .on('unlink', (fullPath) => {
             const relPath = path.relative(this.options.rootDir, fullPath);
             this.files.delete(relPath);
         });

package/dist/index.js

@@ -3,3 +3,4 @@ export * from './types.js';
 export * from './embeddings/Embedder.js';
 export * from './embeddings/Ollama.js';
 export * from './embeddings/OpenRouter.js';
+export * from './embeddings/OpenAI.js';

package/package.json

@@ -1,9 +1,10 @@
 {
   "name": "inbed",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Semantic indexing and search over source code using vector embeddings",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "files": ["dist"],
   "scripts": {
     "build": "tsc",
     "start": "node dist/index.js",

package/demo.js

@@ -1,29 +0,0 @@
-import { OpenRouterEmbedder } from './dist/embeddings/OpenRouter.js';
-import { Inbed } from './dist/Inbed.js';
-
-import 'dotenv/config';
-
-async function demo() {
-  try {
-    // const embedder = new OllamaEmbedder('https://ollama.johan.chat', 'mxbai-embed-large');
-    const embedder = new OpenRouterEmbedder(process.env.OPENAI_API_KEY, 'mistralai/codestral-embed-2505');
-
-    const project = new Inbed(embedder, { rootDir: './' });
-    await project.load();
-
-    console.log('Loaded files:', project.listFiles().map(f => f.path));
-
-    const results = await project.semanticSearch('demo');
-    console.log('Semantic search results:');
-    results.forEach(result => {
-      console.log(result)
-      console.log(`- ${result.path}: ${result.score.toFixed(4)}`);
-    });
-
-  } catch (error) {
-    console.error('Demo failed:', error);
-    process.exit(1);
-  }
-}
-
-demo();

package/johankit.yml

@@ -1,6 +0,0 @@
-ignore:
-  - dist
-  - .inbed
-  - node_modules
-  - package-lock.json
-  - yarn.lock

package/src/Inbed.ts

@@ -1,268 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-import crypto from 'crypto';
-import chokidar from 'chokidar';
-
-import { glob } from 'glob';
-import ts from 'typescript';
-import { InbedFile, InbedOptions, SearchResult, InbedImport } from './types.js';
-
-import { Embedder } from './embeddings/Embedder.js';
-
-export class Inbed {
-  private files: Map<string, InbedFile> = new Map();
-  private options: InbedOptions;
-  private embedder: Embedder;
-  private watcher: any;
-  private storageDir: string;
-  private chunkLimit: number;
-
-  private embeddingQueue: Promise<void>[] = [];
-  private maxConcurrentEmbeds = 4;
-
-  private debounceMap: Map<string, NodeJS.Timeout> = new Map();
-
-  constructor(embedder: Embedder, options: InbedOptions) {
-    this.options = {
-      recursive: true,
-      fileExtensions: ['.ts', '.js'],
-      ignorePatterns: ['johankit.yaml', 'dist/**', 'node_modules/**', '.git/**', '.inbed/**', ...(options.ignorePatterns || [])],
-      maxDepth: 10,
-      ...options
-    };
-    this.embedder = embedder;
-    this.chunkLimit = (options as any).chunkLimit || 10;
-    this.storageDir = path.join(this.options.rootDir, '.inbed');
-    if (!fs.existsSync(this.storageDir)) fs.mkdirSync(this.storageDir, { recursive: true });
-  }
-
-  async load(): Promise<void> {
-    const patterns = this.options.fileExtensions!.map(ext => `**/*${ext}`);
-    const files = patterns.flatMap(pattern => glob.sync(pattern, { cwd: this.options.rootDir, ignore: this.options.ignorePatterns, nodir: true }));
-
-    for (const filePath of files) {
-      const fullPath = path.join(this.options.rootDir, filePath);
-      try {
-        const content = fs.readFileSync(fullPath, 'utf-8');
-        await this.enqueueAdd(path.normalize(filePath), content);
-      } catch {}
-    }
-
-    await this.flushQueue();
-
-    if (this.options.recursive) {
-      for (const file of Array.from(this.files.values())) {
-        this.resolveImports(file, 0);
-      }
-    }
-
-    this.watchFiles();
-  }
-
-  private hashContent(content: string): string {
-    const version = 'chunker:v2';
-    return crypto.createHash('sha256').update(version + content + this.chunkLimit + this.embedder.model, 'utf-8').digest('hex');
-  }
-
-  private normalize(vec: number[]): number[] {
-    let norm = 0;
-    for (const v of vec) norm += v * v;
-    norm = Math.sqrt(norm);
-    if (norm === 0) return vec;
-    return vec.map(v => v / norm);
-  }
-
-  private async enqueueAdd(filePath: string, content: string) {
-    const task = async () => {
-      await this.addFile(filePath, content);
-    };
-
-    while (this.embeddingQueue.length >= this.maxConcurrentEmbeds) {
-      await Promise.race(this.embeddingQueue);
-    }
-
-    const p = task().finally(() => {
-      this.embeddingQueue = this.embeddingQueue.filter(x => x !== p);
-    });
-
-    this.embeddingQueue.push(p);
-  }
-
-  private async flushQueue() {
-    await Promise.allSettled(this.embeddingQueue);
-  }
-
-  private async addFile(filePath: string, content: string) {
-    const storagePath = path.join(this.storageDir, `${filePath.replace(/\\/g, '_')}_${this.embedder.model}.json`);
-    let embedding: number[][] | undefined;
-    const currentHash = this.hashContent(content);
-
-    if (fs.existsSync(storagePath)) {
-      try {
-        const saved = JSON.parse(fs.readFileSync(storagePath, 'utf-8'));
-        if (saved.hash === currentHash) embedding = saved.embedding;
-      } catch {}
-    }
-
-    let chunks = this.isSmall(content) ? [content] : this.chunkByAST(content);
-    if (chunks.length > this.chunkLimit) chunks = chunks.slice(0, this.chunkLimit);
-
-    if (!embedding) {
-      const raw = await this.embedder.embed({ path: filePath, content, chunks, imports: [] });
-      embedding = raw.map(v => this.normalize(v));
-      try {
-        fs.writeFileSync(storagePath, JSON.stringify({ hash: currentHash, embedding }, null, 2));
-      } catch {}
-    }
-
-    this.files.set(filePath, { path: filePath, content, chunks, imports: [], embedding });
-  }
-
-  private isSmall(content: string): boolean {
-    return content.split('\n').length < 50;
-  }
-
-  private chunkByAST(content: string): string[] {
-    try {
-      const sourceFile = ts.createSourceFile('temp.ts', content, ts.ScriptTarget.Latest);
-      const chunks: string[] = [];
-      let buffer = '';
-      ts.forEachChild(sourceFile, node => {
-        const text = node.getFullText(sourceFile).trim();
-        if (!text) return;
-        if ((buffer + text).length > 800) {
-          chunks.push(buffer);
-          buffer = text;
-        } else {
-          buffer += '\n' + text;
-        }
-      });
-      if (buffer.trim()) chunks.push(buffer.trim());
-      return chunks.length ? chunks : [content];
-    } catch {
-      return [content];
-    }
-  }
-
-  private resolveImports(file: InbedFile, depth: number) {
-    if (depth > (this.options.maxDepth || 10)) return;
-
-    const imports: InbedImport[] = [];
-
-    const importRegex = /import\s+(?:[^'"\n]+)\s+from\s+['"](.+)['"]|require\(\s*['"](.+)['"]\s*\)/g;
-
-    let match;
-    while ((match = importRegex.exec(file.content)) !== null) {
-      const sourceImport = match[1] || match[2];
-      if (!sourceImport) continue;
-
-      const base = path.join(path.dirname(file.path), sourceImport);
-
-      for (const ext of ['', '.ts', '.tsx', '.js', '/index.ts', '/index.js']) {
-        const candidate = path.normalize(base + ext);
-        if (this.files.has(candidate)) {
-          imports.push({ source: sourceImport, resolved: candidate });
-          this.resolveImports(this.files.get(candidate)!, depth + 1);
-          break;
-        }
-      }
-    }
-
-    file.imports = imports.filter((v, i, arr) => arr.findIndex(x => x.resolved === v.resolved) === i);
-  }
-
-  async upsertFile(filePath: string, content: string) {
-    await this.enqueueAdd(path.normalize(filePath), content);
-  }
-
-  getFile(filePath: string): InbedFile | undefined {
-    return this.files.get(path.normalize(filePath));
-  }
-
-  listFiles(): InbedFile[] {
-    return Array.from(this.files.values());
-  }
-
-  async semanticSearch(query: string, topK: number = 5): Promise<SearchResult[]> {
-    const queryEmbedding = this.normalize(await this.embedder.embedQuery(query));
-    const results: SearchResult[] = [];
-
-    for (const file of this.files.values()) {
-      if (!file.embedding) continue;
-      let bestScore = 0;
-      let bestChunk = '';
-      for (let i = 0; i < file.embedding.length; i++) {
-        const score = this.cosineSimilarity(queryEmbedding, file.embedding[i]);
-        if (score > bestScore) {
-          bestScore = score;
-          bestChunk = file.chunks[i] || '';
-        }
-      }
-      if (bestScore > 0) {
-        const importsText = file.imports.map(imp => `${imp.source} -> ${imp.resolved}`).join('\n');
-        const context = `FILE: ${file.path}\nIMPORTS:\n${importsText || '(none)'}\n---\n${bestChunk}`;
-        results.push({ path: file.path, snippet: context.slice(0, 500), score: bestScore });
-      }
-    }
-
-    return results.sort((a, b) => b.score - a.score).slice(0, topK);
-  }
-
-  private cosineSimilarity(a: number[], b: number[]): number {
-    let dot = 0;
-    const len = Math.min(a.length, b.length);
-    for (let i = 0; i < len; i++) dot += a[i] * b[i];
-    return dot;
-  }
-
-  private watchFiles() {
-    this.watcher = chokidar.watch(this.options.rootDir, {
-      ignored: this.options.ignorePatterns,
-      persistent: true,
-      ignoreInitial: true
-    });
-
-    const schedule = (fullPath: string, handler: () => void) => {
-      if (this.debounceMap.has(fullPath)) clearTimeout(this.debounceMap.get(fullPath)!);
-      this.debounceMap.set(fullPath, setTimeout(handler, 200));
-    };
-
-    this.watcher
-      .on('add', fullPath => schedule(fullPath, async () => {
-        const relPath = path.relative(this.options.rootDir, fullPath);
-        if (this.options.fileExtensions?.some(ext => fullPath.endsWith(ext))) {
-          try {
-            const content = fs.readFileSync(fullPath, 'utf-8');
-            await this.enqueueAdd(relPath, content);
-            if (this.options.recursive) this.resolveImports(this.files.get(relPath)!, 0);
-          } catch {}
-        }
-      }))
-      .on('change', fullPath => schedule(fullPath, async () => {
-        const relPath = path.relative(this.options.rootDir, fullPath);
-        if (this.options.fileExtensions?.some(ext => fullPath.endsWith(ext))) {
-          try {
-            const content = fs.readFileSync(fullPath, 'utf-8');
-            await this.enqueueAdd(relPath, content);
-            if (this.options.recursive) this.resolveImports(this.files.get(relPath)!, 0);
-          } catch {}
-        }
-      }))
-      .on('unlink', fullPath => {
-        const relPath = path.relative(this.options.rootDir, fullPath);
-        this.files.delete(relPath);
-      });
-  }
-
-  stopWatching() {
-    if (this.watcher) {
-      this.watcher.close();
-      this.watcher = null;
-    }
-  }
-
-  clear(): void {
-    this.stopWatching();
-    this.files.clear();
-  }
-}

package/src/embeddings/Embedder.ts

@@ -1,7 +0,0 @@
-import { InbedFile } from '../types.js';
-
-export interface Embedder {
-  model: string;
-  embed(file: InbedFile): Promise<number[][]>;
-  embedQuery(query: string): Promise<number[]>;
-}

package/src/embeddings/Ollama.ts

@@ -1,69 +0,0 @@
-import axios from 'axios';
-import { InbedFile } from '../types.js';
-import { Embedder } from './Embedder.js';
-
-/**
- * Interface InbedFile
- * export interface InbedFile {
- *   name: string;
- *   chunks: string[];
- * }
- */
-
-/**
- * Interface Embedder
- * export interface Embedder {
- *   embed(file: InbedFile): Promise<number[][]>;
- *   embedQuery(query: string): Promise<number[]>;
- * }
- */
-
-export class OllamaEmbedder implements Embedder {
-  private baseUrl: string;
-  model: string;
-
-  constructor(baseUrl: string = 'http://localhost:11434', model: string = 'mxbai-embed-large') {
-    this.baseUrl = baseUrl;
-    this.model = model;
-  }
-
-  async embed(file: InbedFile): Promise<number[][]> {
-    const embeddings: number[][] = [];
-    for (const chunk of file.chunks) {
-      const emb = await this.embedText(chunk);
-      if (emb && emb.length > 0) embeddings.push(emb);
-      else console.warn(`Skipping chunk due to failed embedding: ${chunk.substring(0, 50)}...`);
-    }
-    return embeddings;
-  }
-
-  async embedQuery(query: string): Promise<number[]> {
-    return this.embedText(query);
-  }
-
-  private async embedText(text: string): Promise<number[]> {
-    try {
-      const response = await axios.post(
-        `${this.baseUrl}/api/embed`,
-        {
-          model: this.model,
-          input: text
-        },
-        { timeout: 30000 }
-      );
-
-      if (response.data?.embeddings && response.data.embeddings.length > 0) {
-        return response.data.embeddings[0];
-      }
-      
-      console.error('Resposta da API Ollama inesperada:', response.data);
-      return [];
-    } catch (error: any) {
-      console.error(
-        'Erro em embedText (Ollama):',
-        error.response?.data ? JSON.stringify(error.response.data) : error.message
-      );
-      return [];
-    }
-  }
-}

package/src/embeddings/OpenAI.ts

@@ -1,58 +0,0 @@
-import { InbedFile } from '../types.js';
-import { Embedder } from './Embedder.js';
-import OpenAI from 'openai';
-
-export class OpenAIEmbedder implements Embedder {
-  private client: OpenAI;
-  model: string;
-
-  constructor(apiKey: string, model: string = 'text-embedding-3-small') {
-    this.client = new OpenAI({ apiKey });
-    this.model = model;
-  }
-
-  async embed(file: InbedFile): Promise<number[][]> {
-    const embeddings: number[][] = [];
-
-    for (const chunk of file.chunks) {
-      try {
-        const emb = await this.embedText(chunk);
-        if (emb && emb.length > 0) embeddings.push(emb);
-        else console.warn(`Skipping chunk due to failed embedding: ${chunk.substring(0, 50)}...`);
-      } catch (error) {
-        console.error('Error embedding chunk:', error);
-      }
-    }
-
-    return embeddings;
-  }
-
-  async embedQuery(query: string): Promise<number[]> {
-    try {
-      return await this.embedText(query);
-    } catch (error) {
-      console.error('Error embedding query:', error);
-      return [];
-    }
-  }
-
-  private async embedText(text: string): Promise<number[]> {
-    try {
-      const response = await this.client.embeddings.create({
-        model: this.model,
-        input: text,
-      });
-
-      // A resposta retorna embeddings no campo data[0].embedding
-      if (response.data && response.data.length > 0) {
-        return response.data[0].embedding;
-      }
-
-      console.error('Resposta da API de Embedding inesperada:', response);
-      return [];
-    } catch (error: any) {
-      console.error('Error in embedText:', error.response?.data ?? error.message);
-      return [];
-    }
-  }
-}

package/src/embeddings/OpenRouter.ts

@@ -1,100 +0,0 @@
-import axios from 'axios';
-import { InbedFile } from '../types.js'; // Assumindo que este import está correto
-import { Embedder } from './Embedder.js'; // Assumindo que este import está correto
-
-/**
- * Interface InbedFile (exemplo)
- * export interface InbedFile {
- * name: string;
- * chunks: string[];
- * }
- */
-
-/**
- * Interface Embedder (exemplo)
- * export interface Embedder {
- * embed(file: InbedFile): Promise<number[][]>;
- * embedQuery(query: string): Promise<number[]>;
- * }
- */
-
-export class OpenRouterEmbedder implements Embedder {
-  private apiKey: string;
-  model: string;
-
-  constructor(apiKey: string, model: string) {
-    this.apiKey = apiKey;
-    this.model = model;
-  }
-
-  async embed(file: InbedFile): Promise<number[][]> {
-    try {
-      const embeddings: number[][] = [];
-      // Itera sobre todos os 'chunks' (pedaços) do arquivo
-      for (const chunk of file.chunks) {
-        // Gera o embedding para cada pedaço
-        const emb = await this.embedText(chunk);
-        // Adiciona o embedding resultante ao array de embeddings
-        if (emb && emb.length > 0) {
-          embeddings.push(emb);
-        } else {
-          // Trata o caso em que embedText retorna um array vazio (erro)
-          console.warn(`Skipping chunk due to failed embedding: ${chunk.substring(0, 50)}...`);
-        }
-      }
-      return embeddings;
-    } catch (error) {
-      console.error('Error embedding file:', error);
-      return [];
-    }
-  }
-
-  async embedQuery(query: string): Promise<number[]> {
-    try {
-      // Usa o mesmo método para embutir a query
-      return await this.embedText(query);
-    } catch (error) {
-      console.error('Error embedding query:', error);
-      return [];
-    }
-  }
-
-  private async embedText(text: string): Promise<number[]> {
-    try {
-      const response = await axios.post(
-        'https://openrouter.ai/api/v1/embeddings',
-        { 
-          input: text, 
-          model: this.model 
-        },
-        {
-          headers: {
-            // Utiliza a chave da instância
-            'Authorization': `Bearer ${this.apiKey}`,
-            'Content-Type': 'application/json',
-          },
-          timeout: 30000
-        }
-      );
-
-      // A resposta da OpenRouter/OpenAI retorna um array em 'data', 
-      // e o primeiro objeto contém o 'embedding'
-      // Exemplo de estrutura de resposta: { "data": [{ "embedding": [0.1, 0.2, ...], ... }], ... }
-      if (response.data && response.data.data && response.data.data.length > 0) {
-        return response.data.data[0].embedding;
-      }
-
-      // Se a resposta for inesperada, loga e retorna array vazio
-      console.error('Resposta da API de Embedding inesperada:', response.data);
-      return [];
-    } catch (error: any) {
-      // Loga detalhes do erro (como o corpo da resposta da API, se disponível)
-      console.error(
-        'Error in embedText:', 
-        error.response?.data ? JSON.stringify(error.response.data) : error.message
-      );
-      // Retorna um array vazio para que a chamada externa possa continuar
-      return [];
-    }
-  }
-}

package/src/index.ts

@@ -1,6 +0,0 @@
-export { Inbed } from './Inbed.js';
-export * from './types.js';
-export * from './embeddings/Embedder.js';
-export * from './embeddings/Ollama.js';
-export * from './embeddings/OpenRouter.js';
-export * from './embeddings/OpenAI.js';

package/src/types.ts

@@ -1,28 +0,0 @@
-import { Embedder } from "./embeddings/Embedder.js";
-
-export interface InbedImport {
-  source: string;
-  resolved: string;
-}
-
-export interface InbedFile {
-  path: string;
-  content: string;
-  chunks: string[];
-  imports: InbedImport[];
-  embedding?: number[][];
-}
-
-export interface SearchResult {
-  path: string;
-  snippet: string;
-  score: number;
-}
-
-export interface InbedOptions {
-  rootDir: string;
-  recursive?: boolean;
-  fileExtensions?: string[];
-  ignorePatterns?: string[];
-  maxDepth?: number;
-}

package/src/utils.ts

@@ -1,14 +0,0 @@
-import fs from 'fs';
-import path from 'path';
-
-export function readFileSafe(filePath: string): string | null {
-  try {
-    return fs.readFileSync(filePath, 'utf-8');
-  } catch {
-    return null;
-  }
-}
-
-export function isIgnored(filePath: string, ignorePatterns: string[]): boolean {
-  return ignorePatterns.some(pattern => filePath.includes(pattern));
-}

package/tsconfig.json

@@ -1,15 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2020",
-    "module": "ESNext",
-    "moduleResolution": "Node",
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "skipLibCheck": true,
-    "lib": ["ES2020", "DOM"]
-  },
-  "include": ["src", "demo.js"]
-}