@aiconnect/easy-rag 0.3.0

package/dist/vector-store/chromadb.js

@@ -0,0 +1,98 @@
+import { ChromaClient } from 'chromadb';
+import { ensureChromaDataDirectory } from './utils.js';
+const client = new ChromaClient({
+    path: process.env.CHROMA_URL || 'http://localhost:8000',
+});
+export async function getOrCreateCollection(baseName) {
+    const sanitizedBaseName = baseName
+        .replace(/[^a-zA-Z0-9_-]/g, '_')
+        .toLowerCase();
+    let collectionName = sanitizedBaseName;
+    let suffix = 1;
+    while (true) {
+        try {
+            const collection = await client.getOrCreateCollection({
+                name: collectionName,
+                metadata: { 'hnsw:space': 'cosine' },
+            });
+            if (suffix === 1 || collectionName !== sanitizedBaseName) {
+                console.log(`Using collection: ${collectionName}`);
+            }
+            return collection;
+        }
+        catch (error) {
+            if (error instanceof Error && error.message.includes('already exists')) {
+                collectionName = `${sanitizedBaseName}_${suffix}`;
+                suffix++;
+            }
+            else {
+                throw error;
+            }
+        }
+    }
+}
+export async function storeEmbeddings(collection, chunks) {
+    if (chunks.length === 0) {
+        return;
+    }
+    const ids = chunks.map((c) => c.id);
+    const embeddings = chunks.map((c) => c.embedding);
+    const documents = chunks.map((c) => c.content);
+    const metadatas = chunks.map((c) => {
+        const metadata = {
+            sourcePath: c.metadata.sourcePath,
+            contentType: c.metadata.contentType,
+            chunkIndex: c.metadata.chunkIndex,
+        };
+        if (c.metadata.sectionTitle !== undefined) {
+            metadata.sectionTitle = c.metadata.sectionTitle;
+        }
+        if (c.metadata.startPosition !== undefined) {
+            metadata.startPosition = c.metadata.startPosition;
+        }
+        if (c.metadata.endPosition !== undefined) {
+            metadata.endPosition = c.metadata.endPosition;
+        }
+        if (c.metadata.rowNumber !== undefined) {
+            metadata.rowNumber = c.metadata.rowNumber;
+        }
+        return metadata;
+    });
+    await collection.add({
+        ids,
+        embeddings,
+        documents,
+        metadatas,
+    });
+}
+export async function initializeChromaDB() {
+    await ensureChromaDataDirectory();
+}
+export async function listCollections() {
+    const collectionNames = await client.listCollections();
+    const collectionInfos = [];
+    for (const name of collectionNames) {
+        try {
+            const collection = await client.getCollection({ name });
+            const count = await collection.count();
+            collectionInfos.push({
+                name,
+                count,
+            });
+        }
+        catch {
+            collectionInfos.push({
+                name,
+                count: 0,
+            });
+        }
+    }
+    return collectionInfos;
+}
+export async function deleteCollection(collectionName) {
+    const sanitized = collectionName
+        .replace(/[^a-zA-Z0-9_-]/g, '_')
+        .toLowerCase();
+    await client.deleteCollection({ name: sanitized });
+}
+//# sourceMappingURL=chromadb.js.map

package/dist/vector-store/chromadb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chromadb.js","sourceRoot":"","sources":["../../src/vector-store/chromadb.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAmB,MAAM,UAAU,CAAC;AAEzD,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAEvD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;IAC9B,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,uBAAuB;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,QAAgB;IAC1D,MAAM,iBAAiB,GAAG,QAAQ;SAC/B,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,WAAW,EAAE,CAAC;IAEjB,IAAI,cAAc,GAAG,iBAAiB,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC;IAEf,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,qBAAqB,CAAC;gBACpD,IAAI,EAAE,cAAc;gBACpB,QAAQ,EAAE,EAAE,YAAY,EAAE,QAAQ,EAAE;aACrC,CAAC,CAAC;YAEH,IAAI,MAAM,KAAK,CAAC,IAAI,cAAc,KAAK,iBAAiB,EAAE,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,qBAAqB,cAAc,EAAE,CAAC,CAAC;YACrD,CAAC;YAED,OAAO,UAAU,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;gBACvE,cAAc,GAAG,GAAG,iBAAiB,IAAI,MAAM,EAAE,CAAC;gBAClD,MAAM,EAAE,CAAC;YACX,CAAC;iBAAM,CAAC;gBACN,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,UAAsB,EACtB,MAAqB;IAErB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO;IACT,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpC,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,QAAQ,GAA8C;YAC1D,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;YACjC,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,WAAW;YACnC,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;SAClC,CAAC;QAEF,IAAI,CAAC,CAAC,QAAQ,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;YAC1C,QAAQ,CAAC,YAAY,GAAG,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC;QAClD,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;YAC3C,QAAQ,CAAC,aAAa,GAAG,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;QACpD,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACzC,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChD,CAAC;QACD,IAAI,CAAC,CAAC,QAAQ,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACvC,QAAQ,CAAC,SAAS,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC5C,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,CAAC,GAAG,CAAC;QACnB,GAAG;QACH,UAAU;QACV,SAAS;QACT,SAAS;KACV,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,yBAAyB,EAAE,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACvD,MAAM,eAAe,GAAqB,EAAE,CAAC;IAE7C,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;YACxD,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;YACvC,eAAe,CAAC,IAAI,CAAC;gBACnB,IAAI;gBACJ,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,eAAe,CAAC,IAAI,CAAC;gBACnB,IAAI;gBACJ,KAAK,EAAE,CAAC;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,cAAsB;IAC3D,MAAM,SAAS,GAAG,cAAc;SAC7B,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,WAAW,EAAE,CAAC;IACjB,MAAM,MAAM,CAAC,gBAAgB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AACrD,CAAC"}

package/dist/vector-store/index.d.ts

@@ -0,0 +1,4 @@
+export { getOrCreateCollection, storeEmbeddings, initializeChromaDB, listCollections, deleteCollection } from './chromadb.js';
+export { ensureChromaDataDirectory } from './utils.js';
+export type { VectorChunk, CollectionInfo } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/vector-store/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,kBAAkB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AACvD,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/vector-store/index.js

@@ -0,0 +1,3 @@
+export { getOrCreateCollection, storeEmbeddings, initializeChromaDB, listCollections, deleteCollection } from './chromadb.js';
+export { ensureChromaDataDirectory } from './utils.js';
+//# sourceMappingURL=index.js.map

package/dist/vector-store/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,kBAAkB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC"}

package/dist/vector-store/types.d.ts

@@ -0,0 +1,12 @@
+import type { ChunkMetadata } from '../chunker/types.js';
+export interface VectorChunk {
+    id: string;
+    embedding: number[];
+    metadata: ChunkMetadata;
+    content: string;
+}
+export interface CollectionInfo {
+    name: string;
+    count: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/vector-store/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vector-store/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,QAAQ,EAAE,aAAa,CAAC;IACxB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf"}

package/dist/vector-store/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/vector-store/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/vector-store/types.ts"],"names":[],"mappings":""}

package/dist/vector-store/utils.d.ts

@@ -0,0 +1,2 @@
+export declare function ensureChromaDataDirectory(): Promise<string>;
+//# sourceMappingURL=utils.d.ts.map

package/dist/vector-store/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/vector-store/utils.ts"],"names":[],"mappings":"AAMA,wBAAsB,yBAAyB,IAAI,OAAO,CAAC,MAAM,CAAC,CAYjE"}

package/dist/vector-store/utils.js

@@ -0,0 +1,17 @@
+import os from 'os';
+import path from 'path';
+import fs from 'fs/promises';
+const CHROMA_DATA_DIR = path.join(os.homedir(), '.easy-rag', 'chromadb');
+export async function ensureChromaDataDirectory() {
+    try {
+        await fs.mkdir(CHROMA_DATA_DIR, { recursive: true });
+        return CHROMA_DATA_DIR;
+    }
+    catch (error) {
+        if (error instanceof Error && 'code' in error && error.code === 'EACCES') {
+            throw new Error(`Cannot write to ${CHROMA_DATA_DIR}. Please check directory permissions.`);
+        }
+        throw error;
+    }
+}
+//# sourceMappingURL=utils.js.map

package/dist/vector-store/utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/vector-store/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,aAAa,CAAC;AAE7B,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;AAEzE,MAAM,CAAC,KAAK,UAAU,yBAAyB;IAC7C,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACrD,OAAO,eAAe,CAAC;IACzB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzE,MAAM,IAAI,KAAK,CACb,mBAAmB,eAAe,uCAAuC,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}

package/package.json

@@ -0,0 +1,57 @@
+{
+  "name": "@aiconnect/easy-rag",
+  "version": "0.3.0",
+  "description": "A TypeScript CLI tool for local RAG indexing and querying",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "easy-rag": "./dist/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsx src/index.ts",
+    "start": "node dist/index.js",
+    "test": "node --experimental-vm-modules node_modules/jest/bin/jest.js",
+    "test:watch": "node --experimental-vm-modules node_modules/jest/bin/jest.js --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "dist",
+    "skills",
+    "README.md"
+  ],
+  "keywords": [
+    "rag",
+    "cli",
+    "vector-search",
+    "embeddings",
+    "chromadb",
+    "openai",
+    "retrieval-augmented-generation"
+  ],
+  "author": "AI Connect",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/johnjohn-aic/easy-rag.git"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.19.11",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.4.6",
+    "typescript": "^5.9.3"
+  },
+  "dependencies": {
+    "commander": "^13.1.0",
+    "pdf-parse": "^1.1.1",
+    "openai": "^4.89.0",
+    "chromadb": "^2.1.5"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/skills/easy-rag/SKILL.md

@@ -0,0 +1,198 @@
+---
+name: easy-rag
+description: >
+  Index documents and query them using local RAG (Retrieval-Augmented Generation).
+  Use when you need to build a knowledge base from files (PDF, Markdown, CSV) and
+  search it with natural language questions. Runs locally with ChromaDB embedded.
+---
+
+# Easy RAG Skill
+
+## When to Use
+
+- You need to **index documents** (PDF, Markdown, CSV) into a searchable knowledge base
+- You need to **query** indexed documents with natural language questions
+- You need to **manage collections** (list, delete)
+
+## Requirements
+
+- **OpenAI API key** must be configured (via `easy-rag init` or `OPENAI_API_KEY` env var)
+- No external services needed — ChromaDB runs embedded locally
+
+### Setting Up API Key
+
+**Option 1: Interactive init (recommended)**
+```bash
+easy-rag init
+```
+Prompts for API key and embedding model selection.
+
+**Option 2: Environment variable**
+```bash
+export OPENAI_API_KEY="sk-..."
+```
+
+Environment variables override config file settings.
+
+## Installation
+
+```bash
+# From npm (when published)
+npx easy-rag <command>
+
+# From source
+cd /path/to/easy-rag
+npm install
+npm run build
+node dist/index.js <command>
+```
+
+## Commands
+
+### Initialize Configuration
+
+Run interactive setup to create global configuration.
+
+```bash
+easy-rag init
+```
+
+Prompts for:
+- OpenAI API key (masked input)
+- Embedding model selection (default: `text-embedding-3-large`)
+
+Creates `~/.easy-rag/config.json` with your settings.
+
+**Always run this first** before indexing if you haven't configured EasyRAG yet.
+
+### Index Documents
+
+Recursively scans a folder for supported files and indexes them into ChromaDB.
+
+```bash
+easy-rag index <folder>
+```
+
+- **Supported formats:** `.pdf`, `.md`, `.csv`
+- **Chunking:** Automatic — Markdown by heading/section, PDF by paragraph, CSV by row
+- **Collection:** Named after the folder (e.g., `my-docs/` → collection `my-docs`)
+- **Idempotent:** Re-indexing the same folder updates the existing collection
+
+**Example:**
+```bash
+easy-rag index ./knowledge-base
+# Indexes all PDF, MD, CSV files in ./knowledge-base/
+```
+
+### Query Documents
+
+Search indexed documents using natural language. Returns the most relevant chunks.
+
+```bash
+easy-rag query [options] "<question>"
+```
+
+| Option | Description | Default |
+|--------|-------------|---------|
+| `--top <n>` | Number of results to return | 5 |
+| `--metadata` | Include source file, score, chunk index | off |
+| `--collection <name>` | Search a specific collection | all |
+
+**Examples:**
+```bash
+# Simple query
+easy-rag query "What is the refund policy?"
+
+# Top 3 results with metadata
+easy-rag query --top 3 --metadata "How do I configure the API?"
+
+# Search specific collection
+easy-rag query --collection my-docs "quarterly revenue"
+```
+
+**Output with `--metadata`:**
+```
+[Score: 0.92 | Source: docs/api-guide.md | Chunk: 3]
+To configure the API, set the following environment variables...
+
+[Score: 0.87 | Source: docs/setup.pdf | Chunk: 12]
+The API configuration requires...
+```
+
+**Output without `--metadata`:**
+```
+To configure the API, set the following environment variables...
+
+The API configuration requires...
+```
+
+### List Collections
+
+Show all indexed collections and their document counts.
+
+```bash
+easy-rag collections
+```
+
+### Delete a Collection
+
+Remove an indexed collection. Asks for confirmation unless `--force` is used.
+
+```bash
+easy-rag delete <collection-name>
+easy-rag delete <collection-name> --force
+```
+
+**Always use `--force` in automated workflows** to avoid interactive prompts.
+
+## Workflow for AI Agents
+
+### Building a Knowledge Base
+
+1. Place documents in a folder
+2. Index: `easy-rag index ./my-folder`
+3. Query: `easy-rag query "your question"`
+
+### Recommended Patterns
+
+- **One collection per topic/project** — index separate folders for different knowledge domains
+- **Always use `--metadata`** — helps trace answers back to source documents
+- **Use `--top 3`** for focused answers, `--top 10` for broad research
+- **Re-index after adding files** — run `easy-rag index` again on the same folder
+
+### Error Handling
+
+| Error | Cause | Fix |
+|-------|-------|-----|
+| `OpenAI API key is required` | Missing configuration | Run `easy-rag init` or export `OPENAI_API_KEY` |
+| `No supported files found` | Folder has no PDF/MD/CSV | Check folder path and file extensions |
+| `Collection not found` | Querying non-existent collection | Run `easy-rag collections` to check available ones |
+
+## Environment Variables
+
+| Variable | Required | Description |
+|----------|----------|-------------|
+| `OPENAI_API_KEY` | Yes (if not in config) | OpenAI API key for generating embeddings |
+| `EMBEDDING_MODEL` | No | Override embedding model (default: `text-embedding-3-large`) |
+
+**Environment variables override config file settings.**
+
+### Configuration File
+
+Config is stored at `~/.easy-rag/config.json`:
+```json
+{
+  "openai_api_key": "sk-...",
+  "embedding_model": "text-embedding-3-large"
+}
+```
+
+**Priority (highest first):**
+1. Environment variables
+2. Config file
+3. Hardcoded defaults
+
+**Available models:**
+- `text-embedding-3-large` (default, best performance)
+- `text-embedding-3-small` (faster, cheaper)
+- `text-embedding-ada-002` (legacy)