@yesvara/svara 0.1.0

package/src/rag/loader.ts

@@ -0,0 +1,216 @@
+/**
+ * @module rag/loader
+ * SvaraJS — Document loader
+ *
+ * Loads documents from various sources into a normalized format.
+ * Supported formats: TXT, MD, PDF, DOCX, HTML, JSON
+ *
+ * @example
+ * const loader = new DocumentLoader();
+ * const docs = await loader.loadMany(['./docs/*.pdf', './knowledge/*.txt']);
+ */
+
+import fs from 'fs/promises';
+import path from 'path';
+import crypto from 'crypto';
+import type { Document, DocumentType } from '../core/types.js';
+
+// ─── Loader Strategy Interface ────────────────────────────────────────────────
+
+interface FileLoader {
+  extensions: string[];
+  load(filePath: string): Promise<string>;
+}
+
+// ─── Plain Text Loader ────────────────────────────────────────────────────────
+
+class TextFileLoader implements FileLoader {
+  extensions = ['.txt', '.md', '.mdx', '.rst', '.csv', '.log'];
+
+  async load(filePath: string): Promise<string> {
+    return fs.readFile(filePath, 'utf-8');
+  }
+}
+
+// ─── JSON Loader ──────────────────────────────────────────────────────────────
+
+class JsonFileLoader implements FileLoader {
+  extensions = ['.json', '.jsonl'];
+
+  async load(filePath: string): Promise<string> {
+    const raw = await fs.readFile(filePath, 'utf-8');
+
+    if (path.extname(filePath) === '.jsonl') {
+      // JSON Lines — one JSON object per line
+      return raw
+        .split('\n')
+        .filter(Boolean)
+        .map((line) => {
+          const obj = JSON.parse(line) as Record<string, unknown>;
+          return Object.values(obj).join(' ');
+        })
+        .join('\n');
+    }
+
+    const data = JSON.parse(raw);
+    return JSON.stringify(data, null, 2);
+  }
+}
+
+// ─── HTML Loader ──────────────────────────────────────────────────────────────
+
+class HtmlFileLoader implements FileLoader {
+  extensions = ['.html', '.htm'];
+
+  async load(filePath: string): Promise<string> {
+    const raw = await fs.readFile(filePath, 'utf-8');
+    // Strip HTML tags — simple regex, good enough for RAG
+    return raw
+      .replace(/<script[\s\S]*?<\/script>/gi, '')
+      .replace(/<style[\s\S]*?<\/style>/gi, '')
+      .replace(/<[^>]+>/g, ' ')
+      .replace(/\s+/g, ' ')
+      .trim();
+  }
+}
+
+// ─── PDF Loader ───────────────────────────────────────────────────────────────
+
+class PdfFileLoader implements FileLoader {
+  extensions = ['.pdf'];
+
+  async load(filePath: string): Promise<string> {
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const pdfParse = require('pdf-parse') as (buffer: Buffer) => Promise<{ text: string }>;
+      const buffer = await fs.readFile(filePath);
+      const data = await pdfParse(buffer);
+      return data.text;
+    } catch {
+      throw new Error(
+        '[SvaraJS] PDF loading requires the "pdf-parse" package.\n' +
+        'Run: npm install pdf-parse'
+      );
+    }
+  }
+}
+
+// ─── DOCX Loader ──────────────────────────────────────────────────────────────
+
+class DocxFileLoader implements FileLoader {
+  extensions = ['.docx'];
+
+  async load(filePath: string): Promise<string> {
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const mammoth = require('mammoth') as {
+        extractRawText: (opts: { path: string }) => Promise<{ value: string }>;
+      };
+      const result = await mammoth.extractRawText({ path: filePath });
+      return result.value;
+    } catch {
+      throw new Error(
+        '[SvaraJS] DOCX loading requires the "mammoth" package.\n' +
+        'Run: npm install mammoth'
+      );
+    }
+  }
+}
+
+// ─── DocumentLoader ───────────────────────────────────────────────────────────
+
+export class DocumentLoader {
+  private loaders: FileLoader[];
+  private extensionMap: Map<string, FileLoader>;
+
+  constructor() {
+    this.loaders = [
+      new TextFileLoader(),
+      new JsonFileLoader(),
+      new HtmlFileLoader(),
+      new PdfFileLoader(),
+      new DocxFileLoader(),
+    ];
+
+    this.extensionMap = new Map();
+    for (const loader of this.loaders) {
+      for (const ext of loader.extensions) {
+        this.extensionMap.set(ext, loader);
+      }
+    }
+  }
+
+  /**
+   * Load a single file into a Document.
+   */
+  async load(filePath: string): Promise<Document> {
+    const ext = path.extname(filePath).toLowerCase();
+    const loader = this.extensionMap.get(ext);
+
+    if (!loader) {
+      throw new Error(
+        `[SvaraJS] Unsupported file type: "${ext}". ` +
+        `Supported: ${[...this.extensionMap.keys()].join(', ')}`
+      );
+    }
+
+    const content = await loader.load(filePath);
+    const stats = await fs.stat(filePath);
+
+    return {
+      id: this.hashFile(filePath),
+      content,
+      type: this.detectType(ext),
+      source: filePath,
+      metadata: {
+        filename: path.basename(filePath),
+        extension: ext,
+        size: stats.size,
+        lastModified: stats.mtime.toISOString(),
+      },
+    };
+  }
+
+  /**
+   * Load multiple files. Silently skips unreadable files with a warning.
+   */
+  async loadMany(filePaths: string[]): Promise<Document[]> {
+    const results: Document[] = [];
+
+    for (const filePath of filePaths) {
+      try {
+        const doc = await this.load(filePath);
+        results.push(doc);
+      } catch (err) {
+        console.warn(`[SvaraJS:RAG] Skipping "${filePath}": ${(err as Error).message}`);
+      }
+    }
+
+    return results;
+  }
+
+  /** Check if this loader supports a given file extension. */
+  supports(filePath: string): boolean {
+    const ext = path.extname(filePath).toLowerCase();
+    return this.extensionMap.has(ext);
+  }
+
+  private detectType(ext: string): DocumentType {
+    const map: Record<string, DocumentType> = {
+      '.txt': 'text',
+      '.md': 'markdown',
+      '.mdx': 'markdown',
+      '.pdf': 'pdf',
+      '.html': 'html',
+      '.htm': 'html',
+      '.json': 'json',
+      '.jsonl': 'json',
+      '.docx': 'docx',
+    };
+    return map[ext] ?? 'text';
+  }
+
+  private hashFile(filePath: string): string {
+    return crypto.createHash('md5').update(filePath).digest('hex');
+  }
+}

package/src/rag/retriever.ts

@@ -0,0 +1,248 @@
+/**
+ * @module rag/retriever
+ * SvaraJS — Vector retrieval for RAG
+ *
+ * Embeds document chunks and performs similarity search.
+ * Uses in-memory vector store by default (great for dev),
+ * with SQLite persistence for production.
+ *
+ * @example
+ * const retriever = new VectorRetriever();
+ * await retriever.init({ embeddings: { provider: 'openai' } });
+ * await retriever.addDocuments(['./docs/*.pdf']);
+ *
+ * const context = await retriever.retrieve('What is the refund policy?');
+ */
+
+import type { RAGConfig, DocumentChunk, RetrievedContext } from '../core/types.js';
+import type { RAGRetriever } from '../core/agent.js';
+import { DocumentLoader } from './loader.js';
+import { Chunker } from './chunker.js';
+
+// ─── Embedding Interface ──────────────────────────────────────────────────────
+
+interface EmbeddingProvider {
+  embed(texts: string[]): Promise<number[][]>;
+  embedOne(text: string): Promise<number[]>;
+}
+
+// ─── OpenAI Embeddings ────────────────────────────────────────────────────────
+
+class OpenAIEmbeddings implements EmbeddingProvider {
+  private client: unknown;
+  private model: string;
+
+  constructor(apiKey?: string, model = 'text-embedding-3-small') {
+    this.model = model;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const { default: OpenAI } = require('openai');
+      this.client = new OpenAI({ apiKey: apiKey ?? process.env.OPENAI_API_KEY });
+    } catch {
+      throw new Error('[SvaraJS] OpenAI embeddings require the "openai" package.');
+    }
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    const client = this.client as {
+      embeddings: {
+        create: (params: unknown) => Promise<{ data: Array<{ embedding: number[] }> }>;
+      };
+    };
+
+    // Batch to avoid rate limits (max 2048 inputs per request)
+    const BATCH_SIZE = 100;
+    const results: number[][] = [];
+
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+      const batch = texts.slice(i, i + BATCH_SIZE);
+      const response = await client.embeddings.create({
+        model: this.model,
+        input: batch,
+      });
+      results.push(...response.data.map((d) => d.embedding));
+    }
+
+    return results;
+  }
+
+  async embedOne(text: string): Promise<number[]> {
+    const [embedding] = await this.embed([text]);
+    return embedding;
+  }
+}
+
+// ─── Ollama Embeddings (local) ────────────────────────────────────────────────
+
+class OllamaEmbeddings implements EmbeddingProvider {
+  private baseURL: string;
+  private model: string;
+
+  constructor(model = 'nomic-embed-text', baseURL = 'http://localhost:11434') {
+    this.model = model;
+    this.baseURL = baseURL;
+  }
+
+  async embed(texts: string[]): Promise<number[][]> {
+    return Promise.all(texts.map((t) => this.embedOne(t)));
+  }
+
+  async embedOne(text: string): Promise<number[]> {
+    const response = await fetch(`${this.baseURL}/api/embeddings`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model: this.model, prompt: text }),
+    });
+
+    if (!response.ok) {
+      throw new Error(`[SvaraJS] Ollama embeddings failed: ${response.statusText}`);
+    }
+
+    const data = await response.json() as { embedding: number[] };
+    return data.embedding;
+  }
+}
+
+// ─── In-Memory Vector Store ───────────────────────────────────────────────────
+
+interface VectorEntry {
+  chunk: DocumentChunk;
+  embedding: number[];
+}
+
+class InMemoryVectorStore {
+  private entries: VectorEntry[] = [];
+
+  add(chunk: DocumentChunk, embedding: number[]): void {
+    // Replace if same chunk id
+    const existing = this.entries.findIndex((e) => e.chunk.id === chunk.id);
+    if (existing >= 0) {
+      this.entries[existing] = { chunk, embedding };
+    } else {
+      this.entries.push({ chunk, embedding });
+    }
+  }
+
+  search(queryEmbedding: number[], topK: number, threshold = 0): DocumentChunk[] {
+    const scored = this.entries.map((entry) => ({
+      chunk: entry.chunk,
+      score: cosineSimilarity(queryEmbedding, entry.embedding),
+    }));
+
+    return scored
+      .filter((s) => s.score >= threshold)
+      .sort((a, b) => b.score - a.score)
+      .slice(0, topK)
+      .map((s) => s.chunk);
+  }
+
+  get size(): number {
+    return this.entries.length;
+  }
+}
+
+// ─── VectorRetriever ─────────────────────────────────────────────────────────
+
+export class VectorRetriever implements RAGRetriever {
+  private embedder!: EmbeddingProvider;
+  private store: InMemoryVectorStore;
+  private loader: DocumentLoader;
+  private chunker: Chunker;
+  private config!: RAGConfig;
+
+  constructor() {
+    this.store = new InMemoryVectorStore();
+    this.loader = new DocumentLoader();
+    this.chunker = new Chunker();
+  }
+
+  async init(config: RAGConfig): Promise<void> {
+    this.config = config;
+
+    // Init chunker with config
+    if (config.chunking) {
+      this.chunker = new Chunker({
+        strategy: config.chunking.strategy ?? 'sentence',
+        size: config.chunking.size ? config.chunking.size * 4 : 2000, // rough token→char
+        overlap: config.chunking.overlap ? config.chunking.overlap * 4 : 200,
+      });
+    }
+
+    // Init embeddings provider
+    const emb = config.embeddings ?? { provider: 'openai' };
+    switch (emb.provider) {
+      case 'openai':
+        this.embedder = new OpenAIEmbeddings(emb.apiKey, emb.model);
+        break;
+      case 'ollama':
+        this.embedder = new OllamaEmbeddings(emb.model);
+        break;
+      default:
+        throw new Error(`[SvaraJS] Unknown embeddings provider: "${emb.provider}"`);
+    }
+  }
+
+  async addDocuments(filePaths: string[]): Promise<void> {
+    const documents = await this.loader.loadMany(filePaths);
+    if (!documents.length) return;
+
+    const chunks = this.chunker.chunkMany(documents);
+    if (!chunks.length) return;
+
+    console.log(`[SvaraJS:RAG] Embedding ${chunks.length} chunk(s)...`);
+    const embeddings = await this.embedder.embed(chunks.map((c) => c.content));
+
+    for (let i = 0; i < chunks.length; i++) {
+      this.store.add(chunks[i], embeddings[i]);
+    }
+
+    console.log(`[SvaraJS:RAG] Vector store now has ${this.store.size} chunk(s).`);
+  }
+
+  async retrieve(query: string, topK = 5): Promise<string> {
+    if (this.store.size === 0) return '';
+
+    const queryEmbedding = await this.embedder.embedOne(query);
+    const threshold = this.config.retrieval?.threshold ?? 0.3;
+
+    const chunks = this.store.search(queryEmbedding, topK, threshold);
+
+    if (!chunks.length) return '';
+
+    // Format chunks into a context string
+    return chunks
+      .map((chunk, i) => `[Context ${i + 1}]\nSource: ${String(chunk.metadata.filename ?? chunk.documentId)}\n${chunk.content}`)
+      .join('\n\n---\n\n');
+  }
+
+  async retrieveChunks(query: string, topK = 5): Promise<RetrievedContext> {
+    const queryEmbedding = await this.embedder.embedOne(query);
+    const threshold = this.config.retrieval?.threshold ?? 0.3;
+    const chunks = this.store.search(queryEmbedding, topK, threshold);
+
+    return {
+      chunks,
+      query,
+      totalFound: chunks.length,
+    };
+  }
+}
+
+// ─── Math Utils ──────────────────────────────────────────────────────────────
+
+function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) return 0;
+
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+  return denominator === 0 ? 0 : dot / denominator;
+}

package/src/tools/executor.ts

@@ -0,0 +1,54 @@
+/**
+ * @internal
+ * Tool executor — runs tool calls with timeout protection and error isolation.
+ */
+
+import type { LLMToolCall, ToolExecution, InternalAgentContext } from '../core/types.js';
+import type { ToolRegistry } from './registry.js';
+
+export class ToolExecutor {
+  constructor(private registry: ToolRegistry) {}
+
+  async execute(call: LLMToolCall, ctx: InternalAgentContext): Promise<ToolExecution> {
+    const start = Date.now();
+    const tool = this.registry.get(call.name);
+
+    if (!tool) {
+      return {
+        toolCallId: call.id,
+        name: call.name,
+        result: null,
+        error: `Tool "${call.name}" is not registered. Available: ${this.registry.getAll().map((t) => t.name).join(', ')}`,
+        duration: Date.now() - start,
+      };
+    }
+
+    try {
+      const result = await Promise.race([
+        tool.run(call.arguments, ctx),
+        this.timeout(tool.timeout ?? 30_000, tool.name),
+      ]);
+      return { toolCallId: call.id, name: call.name, result, duration: Date.now() - start };
+    } catch (err) {
+      const message = (err as Error).message;
+      console.error(`[@yesvara/svara] Tool "${call.name}" failed: ${message}`);
+      return {
+        toolCallId: call.id,
+        name: call.name,
+        result: null,
+        error: message,
+        duration: Date.now() - start,
+      };
+    }
+  }
+
+  async executeAll(calls: LLMToolCall[], ctx: InternalAgentContext): Promise<ToolExecution[]> {
+    return Promise.all(calls.map((c) => this.execute(c, ctx)));
+  }
+
+  private timeout(ms: number, name: string): Promise<never> {
+    return new Promise((_, reject) =>
+      setTimeout(() => reject(new Error(`Tool "${name}" timed out after ${ms}ms`)), ms)
+    );
+  }
+}

package/src/tools/index.ts

@@ -0,0 +1,89 @@
+/**
+ * @module tools
+ *
+ * `createTool` — the elegant way to define tools for your agent.
+ *
+ * @example
+ * ```ts
+ * import { createTool } from '@yesvara/svara';
+ *
+ * const weatherTool = createTool({
+ *   name: 'get_weather',
+ *   description: 'Get the current weather for a location',
+ *   parameters: {
+ *     city: { type: 'string', description: 'City name', required: true },
+ *     units: { type: 'string', description: 'Temperature units', enum: ['celsius', 'fahrenheit'] },
+ *   },
+ *   async run({ city, units = 'celsius' }) {
+ *     const data = await fetchWeather(city as string);
+ *     return { temp: data.temp, condition: data.description, city };
+ *   },
+ * });
+ *
+ * agent.addTool(weatherTool);
+ * ```
+ */
+
+import type { Tool } from '../types.js';
+
+/**
+ * Create a type-safe tool definition with IDE autocomplete.
+ *
+ * This is a convenience wrapper — it validates your tool at definition time
+ * and returns a properly typed `Tool` object.
+ *
+ * @example Basic tool
+ * ```ts
+ * const timeTool = createTool({
+ *   name: 'get_current_time',
+ *   description: 'Get the current date and time',
+ *   parameters: {},
+ *   async run() {
+ *     return { datetime: new Date().toISOString() };
+ *   },
+ * });
+ * ```
+ *
+ * @example Tool with parameters and error handling
+ * ```ts
+ * const searchTool = createTool({
+ *   name: 'search_database',
+ *   description: 'Search for records in the database',
+ *   parameters: {
+ *     query: { type: 'string', description: 'Search query', required: true },
+ *     limit: { type: 'number', description: 'Max results to return' },
+ *   },
+ *   async run({ query, limit = 10 }, ctx) {
+ *     console.log(`[${ctx.agentName}] Searching for: ${query}`);
+ *     const results = await db.search(String(query), Number(limit));
+ *     return { results, count: results.length };
+ *   },
+ *   timeout: 10_000, // 10 seconds
+ * });
+ * ```
+ */
+export function createTool(definition: Tool): Tool {
+  // Validate at definition time — fail fast, not at runtime
+  if (!definition.name?.trim()) {
+    throw new Error('[@yesvara/svara] createTool: "name" is required.');
+  }
+  if (!definition.description?.trim()) {
+    throw new Error(`[@yesvara/svara] createTool "${definition.name}": "description" is required.`);
+  }
+  if (typeof definition.run !== 'function') {
+    throw new Error(`[@yesvara/svara] createTool "${definition.name}": "run" must be a function.`);
+  }
+  if (!/^[a-zA-Z0-9_-]+$/.test(definition.name)) {
+    throw new Error(
+      `[@yesvara/svara] createTool: Invalid tool name "${definition.name}". ` +
+      'Use only letters, numbers, underscores, or hyphens.'
+    );
+  }
+
+  return {
+    parameters: {},
+    ...definition,
+  };
+}
+
+export type { Tool } from '../types.js';

package/src/tools/registry.ts

@@ -0,0 +1,44 @@
+/**
+ * @internal
+ * Tool registry — stores and validates InternalTool definitions.
+ */
+
+import type { InternalTool } from '../core/types.js';
+
+export class ToolRegistry {
+  private tools: Map<string, InternalTool> = new Map();
+
+  register(tool: InternalTool): void {
+    if (this.tools.has(tool.name)) {
+      throw new Error(
+        `[@yesvara/svara] Tool "${tool.name}" is already registered. ` +
+        'Use a different name or call registry.update() to replace it.'
+      );
+    }
+    this.tools.set(tool.name, tool);
+  }
+
+  update(tool: InternalTool): void {
+    this.tools.set(tool.name, tool);
+  }
+
+  unregister(name: string): void {
+    this.tools.delete(name);
+  }
+
+  get(name: string): InternalTool | undefined {
+    return this.tools.get(name);
+  }
+
+  getAll(): InternalTool[] {
+    return [...this.tools.values()];
+  }
+
+  has(name: string): boolean {
+    return this.tools.has(name);
+  }
+
+  get size(): number {
+    return this.tools.size;
+  }
+}