@ooneex/rag 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Ooneex
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,337 @@
+# @ooneex/rag
+
+Retrieval-Augmented Generation toolkit with vector database integration, document embedding, and semantic search for AI-powered knowledge retrieval.
+
+![Bun](https://img.shields.io/badge/Bun-Compatible-orange?style=flat-square&logo=bun)
+![TypeScript](https://img.shields.io/badge/TypeScript-Ready-blue?style=flat-square&logo=typescript)
+![MIT License](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)
+
+## Features
+
+✅ **Vector Database** - Abstract base class for building custom vector databases with LanceDB
+
+✅ **PDF Conversion** - Convert PDF documents into structured chunks with heading and page metadata
+
+✅ **Hybrid Search** - Full-text and vector-based hybrid search with RRF reranking
+
+✅ **OpenAI Embeddings** - Built-in support for OpenAI embedding models (ada-002, 3-small, 3-large)
+
+✅ **Schema Definition** - Typed schema definitions using Apache Arrow data types
+
+✅ **Index Management** - Create scalar (btree, bitmap, labelList) and IVF-PQ vector indexes
+
+✅ **Query Filtering** - Composable filter conditions with AND, OR, NOT logical operators
+
+✅ **Query Analysis** - Explain and analyze query plans for performance tuning
+
+✅ **Container Integration** - Decorator-based registration with the DI container
+
+## Installation
+
+```bash
+bun add @ooneex/rag
+```
+
+## Usage
+
+### Defining a Vector Database
+
+```typescript
+import { AbstractVectorDatabase } from '@ooneex/rag';
+import { Utf8 } from 'apache-arrow';
+import type { EmbeddingProviderType, EmbeddingModelType, FieldValueType } from '@ooneex/rag';
+
+type ArticleData = {
+  title: string;
+  category: string;
+};
+
+class ArticleVectorDatabase extends AbstractVectorDatabase<ArticleData> {
+  public getDatabaseUri(): string {
+    return './data/articles.lance';
+  }
+
+  public getEmbeddingModel(): { provider: EmbeddingProviderType; model: EmbeddingModelType['model'] } {
+    return { provider: 'openai', model: 'text-embedding-3-small' };
+  }
+
+  public getSchema(): { [K in keyof ArticleData]: FieldValueType } {
+    return {
+      title: new Utf8(),
+      category: new Utf8(),
+    };
+  }
+}
+```
+
+### Connecting and Adding Data
+
+```typescript
+const db = new ArticleVectorDatabase();
+await db.connect();
+
+const table = await db.open('articles');
+
+await table.add([
+  { id: '1', text: 'Introduction to RAG systems', title: 'RAG Intro', category: 'AI' },
+  { id: '2', text: 'Vector databases explained', title: 'Vector DBs', category: 'Database' },
+]);
+```
+
+### Searching
+
+```typescript
+const results = await table.search('retrieval augmented generation', {
+  limit: 5,
+  select: ['title', 'category'],
+  filter: { field: 'category', op: '=', value: 'AI' },
+});
+
+console.log(results);
+```
+
+### Converting PDFs to Chunks
+
+```typescript
+import { Convertor } from '@ooneex/rag';
+
+const convertor = new Convertor('/path/to/document.pdf');
+
+for await (const chunk of convertor.convert({ outputDir: './output' })) {
+  console.log(chunk.text);
+  console.log(chunk.metadata.heading);
+  console.log(chunk.metadata.pages);
+}
+```
+
+### Composable Filters
+
+```typescript
+const results = await table.search('machine learning', {
+  limit: 10,
+  filter: {
+    AND: [
+      { field: 'category', op: '=', value: 'AI' },
+      { NOT: { field: 'title', op: 'LIKE', value: '%draft%' } },
+    ],
+  },
+});
+```
+
+### Query Plan Analysis
+
+```typescript
+// Explain the query plan
+const plan = await table.explainPlan('search query', {
+  limit: 10,
+  verbose: true,
+});
+console.log(plan);
+
+// Analyze with runtime metrics
+const analysis = await table.analyzePlan('search query', {
+  limit: 10,
+});
+console.log(analysis);
+```
+
+## API Reference
+
+### Classes
+
+#### `AbstractVectorDatabase<DataType>` (Abstract)
+
+Abstract base class for creating vector database implementations.
+
+**Type Parameter:**
+- `DataType` - Record type for additional schema fields
+
+**Abstract Methods:**
+
+##### `getDatabaseUri(): string`
+
+Returns the URI for the LanceDB database storage.
+
+##### `getEmbeddingModel(): { provider: EmbeddingProviderType; model: EmbeddingModelType['model'] }`
+
+Returns the embedding provider and model configuration.
+
+##### `getSchema(): { [K in keyof DataType]: FieldValueType }`
+
+Returns the schema definition using Apache Arrow types.
+
+**Concrete Methods:**
+
+##### `connect(): Promise<void>`
+
+Connect to the LanceDB database.
+
+##### `getDatabase(): Connection`
+
+Get the underlying LanceDB connection. Throws `VectorDatabaseException` if not connected.
+
+##### `open(name: string, options?): Promise<VectorTable<DataType>>`
+
+Open or create a vector table. Automatically creates btree, full-text search, and IVF-PQ indexes on new tables.
+
+**Parameters:**
+- `name` - Table name
+- `options.mode` - `"create"` or `"overwrite"` (default: `"overwrite"`)
+
+---
+
+#### `VectorTable<DataType>`
+
+Provides search, indexing, and data operations on a vector table.
+
+**Methods:**
+
+##### `add(data): Promise<this>`
+
+Add records to the table.
+
+##### `search(query, options?): Promise<DataType[]>`
+
+Perform hybrid (vector + full-text) search with RRF reranking.
+
+**Parameters:**
+- `query` - Search query string
+- `options.limit` - Maximum results (default: 10)
+- `options.select` - Fields to return
+- `options.filter` - Filter conditions
+- `options.nprobes` - IVF partitions to search
+- `options.refineFactor` - Refine step multiplier
+- `options.fastSearch` - Skip un-indexed data (default: true)
+
+##### `createIndex(column, options?): Promise<this>`
+
+Create a scalar index (btree, bitmap, or labelList).
+
+##### `createVectorIndex(column?, options?): Promise<this>`
+
+Create an IVF-PQ vector index.
+
+##### `explainPlan(query, options?): Promise<string>`
+
+Print the resolved query plan.
+
+##### `analyzePlan(query, options?): Promise<string>`
+
+Execute and return a physical plan with runtime metrics.
+
+---
+
+#### `Convertor`
+
+Converts PDF documents into structured text chunks.
+
+**Constructor:**
+```typescript
+new Convertor(source: string)
+```
+
+**Methods:**
+
+##### `convert(options?): AsyncGenerator<ChunkType, { json: ConvertorFileType; markdown: ConvertorFileType }>`
+
+Convert a PDF to chunks, yielding each chunk as it is processed.
+
+**Parameters:**
+- `options.outputDir` - Output directory
+- `options.password` - PDF password
+- `options.imageFormat` - `"png"` or `"jpeg"`
+- `options.pages` - Page range
+- `options.quiet` - Suppress output
+
+### Types
+
+#### `ChunkType`
+
+```typescript
+type ChunkType = {
+  text: string;
+  metadata: {
+    heading: string | null;
+    page: number | null;
+    pages: number[];
+    source: string | null;
+  };
+};
+```
+
+#### `Filter<T>`
+
+Composable filter type supporting field conditions and logical operators.
+
+```typescript
+type Filter<T> =
+  | FilterCondition<T>
+  | { AND: Filter<T>[] }
+  | { OR: Filter<T>[] }
+  | { NOT: Filter<T> };
+```
+
+#### `FilterCondition<T>`
+
+Individual filter conditions with typed operators.
+
+```typescript
+type FilterCondition<T> =
+  | { field: FilterField<T>; op: '>' | '>=' | '<' | '<=' | '='; value: string | number }
+  | { field: FilterField<T>; op: 'IN'; value: (string | number)[] }
+  | { field: FilterField<T>; op: 'LIKE' | 'NOT LIKE'; value: string }
+  | { field: FilterField<T>; op: 'IS NULL' | 'IS NOT NULL' }
+  | { field: FilterField<T>; op: 'IS TRUE' | 'IS NOT TRUE' | 'IS FALSE' | 'IS NOT FALSE' };
+```
+
+#### `EmbeddingProviderType`
+
+```typescript
+type EmbeddingProviderType = 'openai';
+```
+
+#### `FieldValueType`
+
+Apache Arrow types supported for schema fields: `Null`, `Bool`, `Int8`-`Int64`, `Uint8`-`Uint64`, `Float16`-`Float64`, `Utf8`, `LargeUtf8`, `Binary`, `LargeBinary`, `Decimal`, `DateDay`, `DateMillisecond`, and `EmbeddingFunction`.
+
+### Exceptions
+
+#### `VectorDatabaseException`
+
+Thrown when vector database operations fail (e.g., not connected).
+
+#### `ConvertorException`
+
+Thrown when PDF conversion fails.
+
+### Decorators
+
+#### `@decorator.rag()`
+
+Decorator to register RAG classes with the DI container.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](./LICENSE) file for details.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request. For major changes, please open an issue first to discuss what you would like to change.
+
+### Development Setup
+
+1. Clone the repository
+2. Install dependencies: `bun install`
+3. Run tests: `bun run test`
+4. Build the project: `bun run build`
+
+### Guidelines
+
+- Write tests for new features
+- Follow the existing code style
+- Update documentation for API changes
+- Ensure all tests pass before submitting PR
+
+---
+
+Made with ❤️ by the Ooneex team

package/dist/index.d.ts

@@ -0,0 +1,165 @@
+import * as lancedb2 from "@lancedb/lancedb";
+import { ScalarType } from "@ooneex/types";
+import { Connection as Connection2 } from "@lancedb/lancedb";
+import { EmbeddingFunction } from "@lancedb/lancedb/embedding";
+import { Binary, Bool, DateDay, DateMillisecond, Decimal, Float16, Float32, Float64, Int8, Int16, Int32, Int64, LargeBinary, LargeUtf8, Null, Uint8, Uint16, Uint32, Uint64, Utf8 } from "apache-arrow";
+import * as lancedb from "@lancedb/lancedb";
+declare class VectorTable<DataType extends Record<string, unknown>> {
+	private table;
+	private reranker;
+	constructor(table: lancedb.Table);
+	add(data: ({
+		id: string;
+		text: string;
+	} & DataType)[]): Promise<this>;
+	createIndex(column: string, options?: {
+		config?: ReturnType<typeof lancedb.Index.btree | typeof lancedb.Index.bitmap | typeof lancedb.Index.labelList>;
+	}): Promise<this>;
+	createVectorIndex(column?: string, options?: Partial<Parameters<lancedb.Table["createIndex"]>[1] & object>): Promise<this>;
+	search(query: string, options?: {
+		limit?: number;
+		select?: (keyof DataType | "id" | "text")[];
+		filter?: Filter<DataType>;
+		nprobes?: number;
+		refineFactor?: number;
+		fastSearch?: boolean;
+	}): Promise<DataType[]>;
+	explainPlan(query: string, options?: {
+		limit?: number;
+		filter?: Filter<DataType>;
+		verbose?: boolean;
+	}): Promise<string>;
+	analyzePlan(query: string, options?: {
+		limit?: number;
+		filter?: Filter<DataType>;
+	}): Promise<string>;
+}
+type ConvertorOptionsType = {
+	outputDir?: string;
+	password?: string;
+	imageFormat?: "png" | "jpeg";
+	pages?: string;
+	quiet?: boolean;
+};
+type ChunkType = {
+	text: string;
+	metadata: {
+		heading: string | null;
+		page: number | null;
+		pages: number[];
+		source: string | null;
+	};
+};
+type ConvertorFileType = {
+	name: string;
+	path: string;
+};
+type IConvertor = {
+	convert: (options?: ConvertorOptionsType) => AsyncGenerator<ChunkType, {
+		json: ConvertorFileType;
+		markdown: ConvertorFileType;
+	}>;
+};
+type IVectorDatabase<DataType extends Record<string, unknown>> = {
+	getDatabaseUri: () => string;
+	connect: () => Promise<void>;
+	getDatabase: () => Connection2;
+	getEmbeddingModel: () => {
+		provider: EmbeddingProviderType;
+		model: EmbeddingModelType["model"];
+	};
+	getSchema: () => { [K in keyof DataType] : FieldValueType };
+	open: (name: string, options?: {
+		mode?: "create" | "overwrite";
+	}) => Promise<VectorTable<DataType>>;
+};
+type OpenAIModelType = "text-embedding-ada-002" | "text-embedding-3-small" | "text-embedding-3-large";
+type EmbeddingProviderType = "openai";
+type EmbeddingModelMapType = {
+	openai: OpenAIModelType;
+};
+type EmbeddingModelType<P extends EmbeddingProviderType = EmbeddingProviderType> = {
+	provider: P;
+	model: EmbeddingModelMapType[P];
+};
+type FieldValueType = Null | Bool | Int8 | Int16 | Int32 | Int64 | Uint8 | Uint16 | Uint32 | Uint64 | Float16 | Float32 | Float64 | Utf8 | LargeUtf8 | Binary | LargeBinary | Decimal | DateDay | DateMillisecond | EmbeddingFunction;
+type VectorDatabaseClassType = new (...args: any[]) => IVectorDatabase<any>;
+type FilterField<T> = keyof T | "id" | "text";
+type FilterCondition<T> = {
+	field: FilterField<T>;
+	op: ">" | ">=" | "<" | "<=" | "=";
+	value: string | number;
+} | {
+	field: FilterField<T>;
+	op: "IN";
+	value: (string | number)[];
+} | {
+	field: FilterField<T>;
+	op: "LIKE" | "NOT LIKE";
+	value: string;
+} | {
+	field: FilterField<T>;
+	op: "IS NULL" | "IS NOT NULL";
+	value?: never;
+} | {
+	field: FilterField<T>;
+	op: "IS TRUE" | "IS NOT TRUE" | "IS FALSE" | "IS NOT FALSE";
+	value?: never;
+};
+type Filter<T> = FilterCondition<T> | {
+	AND: Filter<T>[];
+} | {
+	OR: Filter<T>[];
+} | {
+	NOT: Filter<T>;
+};
+declare abstract class AbstractVectorDatabase<DataType extends Record<string, ScalarType>> implements IVectorDatabase<DataType> {
+	private db;
+	private embedding;
+	constructor();
+	abstract getEmbeddingModel(): {
+		provider: EmbeddingProviderType;
+		model: EmbeddingModelType["model"];
+	};
+	abstract getSchema(): { [K in keyof DataType] : FieldValueType };
+	connect(): Promise<void>;
+	getDatabase(): lancedb2.Connection;
+	open(name: string, options?: {
+		mode?: "create" | "overwrite";
+	}): Promise<VectorTable<DataType>>;
+}
+declare class Convertor implements IConvertor {
+	private readonly source;
+	constructor(source: string);
+	convert(options?: ConvertorOptionsType): AsyncGenerator<ChunkType, {
+		json: ConvertorFileType;
+		markdown: ConvertorFileType;
+	}>;
+	private generateChunks;
+	private extractContent;
+	private extractTexts;
+}
+import { Exception } from "@ooneex/exception";
+declare class ConvertorException extends Exception {
+	constructor(message: string, data?: Record<string, unknown>);
+}
+import { EContainerScope } from "@ooneex/container";
+declare const decorator: {
+	vectorDatabase: (scope?: EContainerScope) => (target: VectorDatabaseClassType) => void;
+};
+type DataType2 = {
+	name: string;
+};
+declare class VectorDatabase extends AbstractVectorDatabase<DataType2> {
+	getDatabaseUri(): string;
+	getEmbeddingModel(): {
+		provider: EmbeddingProviderType;
+		model: EmbeddingModelType["model"];
+	};
+	getSchema(): { [K in keyof DataType2] : FieldValueType };
+}
+import { Exception as Exception2 } from "@ooneex/exception";
+declare class VectorDatabaseException extends Exception2 {
+	constructor(message: string, data?: Record<string, unknown>);
+}
+export { decorator, VectorTable, VectorDatabaseException, VectorDatabaseClassType, VectorDatabase, OpenAIModelType, IVectorDatabase, IConvertor, FilterField, FilterCondition, Filter, FieldValueType, EmbeddingProviderType, EmbeddingModelType, EmbeddingModelMapType, ConvertorOptionsType, ConvertorFileType, ConvertorException, Convertor, ChunkType, AbstractVectorDatabase };

package/dist/index.js

@@ -0,0 +1,6 @@
+// @bun
+import*as f from"@lancedb/lancedb";import"@lancedb/lancedb/embedding/openai";import{getRegistry as P,LanceSchema as R}from"@lancedb/lancedb/embedding";import{Utf8 as S}from"apache-arrow";import{Exception as O}from"@ooneex/exception";import{HttpStatus as A}from"@ooneex/http-status";class T extends O{constructor(n,o={}){super(n,{status:A.Code.InternalServerError,data:o});this.name="VectorDatabaseException"}}import*as g from"@lancedb/lancedb";var m=(n)=>{if("AND"in n)return`(${n.AND.map(m).join(" AND ")})`;if("OR"in n)return`(${n.OR.map(m).join(" OR ")})`;if("NOT"in n)return`NOT (${m(n.NOT)})`;let o=String(n.field);if(n.op==="IS NULL"||n.op==="IS NOT NULL"||n.op==="IS TRUE"||n.op==="IS NOT TRUE"||n.op==="IS FALSE"||n.op==="IS NOT FALSE")return`${o} ${n.op}`;if(n.op==="IN"){let r=n.value.map((e)=>typeof e==="string"?`'${e}'`:e);return`${o} IN (${r.join(", ")})`}if(n.op==="LIKE"||n.op==="NOT LIKE")return`${o} ${n.op} '${n.value}'`;if(n.op===">"||n.op===">="||n.op==="<"||n.op==="<="||n.op==="=")return`${o} ${n.op} ${typeof n.value==="string"?`'${n.value}'`:n.value}`;return`${o} ${n.op} ${typeof n.value==="string"?`'${n.value}'`:n.value}`};class h{table;reranker=null;constructor(n){this.table=n}async add(n){return await this.table.add(n),this}async createIndex(n,o){return await this.table.createIndex(n,o),this}async createVectorIndex(n="vector",o){return await this.table.createIndex(n,{config:g.Index.ivfPq(),...o}),this}async search(n,o){let{limit:r=10,select:e,filter:s,nprobes:a,refineFactor:i,fastSearch:p=!0}=o??{};if(!this.reranker)this.reranker=await g.rerankers.RRFReranker.create();let t=this.table.search(n,"hybrid","text").rerank(this.reranker).limit(r);if(a)t=t.nprobes(a);if(i)t=t.refineFactor(i);if(p)t=t.fastSearch();if(e)t=t.select(e);if(s)t=t.where(m(s));return t.toArray()}async explainPlan(n,o){let{limit:r=10,filter:e,verbose:s=!0}=o??{};if(!this.reranker)this.reranker=await g.rerankers.RRFReranker.create();let a=this.table.search(n,"hybrid","text").rerank(this.reranker).limit(r);if(e)a=a.where(m(e));return a.explainPlan(s)}async analyzePlan(n,o){let{limit:r=10,filter:e}=o??{};if(!this.reranker)this.reranker=await g.rerankers.RRFReranker.create();let s=this.table.search(n,"hybrid","text").rerank(this.reranker).limit(r);if(e)s=s.where(m(e));return s.analyzePlan()}}class w{db=null;embedding;constructor(){let{provider:n,model:o}=this.getEmbeddingModel();this.embedding=P().get(n)?.create({model:o})}async connect(){this.db=await f.connect(this.getDatabaseUri())}getDatabase(){if(!this.db)throw new T("Database not connected. Call connect() first.");return this.db}async open(n,o){if((await this.getDatabase().tableNames()).includes(n)){let a=await this.getDatabase().openTable(n);return new h(a)}let e=R({id:new S,text:this.embedding.sourceField(new S),vector:this.embedding.vectorField(),...this.getSchema()}),s=await this.getDatabase().createEmptyTable(n,e,{mode:"overwrite",...o});return await s.createIndex("id",{config:f.Index.btree()}),await s.createIndex("text",{config:f.Index.fts()}),await s.createIndex("vector",{config:f.Index.ivfPq()}),new h(s)}}import u from"path";import{random as l}from"@ooneex/utils";import{convert as B}from"@opendataloader/pdf";import{Exception as b}from"@ooneex/exception";import{HttpStatus as F}from"@ooneex/http-status";class c extends b{constructor(n,o={}){super(n,{status:F.Code.InternalServerError,data:o});this.name="ConvertorException"}}class v{source;constructor(n){this.source=u.join(...n.split(/[/\\]/))}async*convert(n={}){try{let o=l.nanoid(15),r=u.join(n.outputDir??"",o),{password:e,imageFormat:s,quiet:a,pages:i}=n;await B([this.source],{outputDir:r,format:"json,markdown",imageDir:u.join(r,"images"),imageOutput:"external",...e!==void 0&&{password:e},...s!==void 0&&{imageFormat:s},...a!==void 0&&{quiet:a},...i!==void 0&&{pages:i}});let p=new Bun.Glob("*"),t,d;for await(let x of p.scan(r)){if(!t&&x.endsWith(".json"))t=x;if(!d&&x.endsWith(".md"))d=x;if(t&&d)break}if(!t)throw new c("No JSON output file found after conversion",{source:this.source});if(!d)throw new c("No Markdown output file found after conversion",{source:this.source});let C=u.join(r,t),k=await Bun.file(C).json(),I=k["file name"]??u.basename(this.source);yield*this.generateChunks(k.kids??[],I);let j=`${l.nanoid(15)}.json`,E=`${l.nanoid(15)}.md`,$=u.join(r,d),D=u.join(r,j),y=u.join(r,E);return await Promise.all([Bun.write(D,Bun.file(C)),Bun.write(y,Bun.file($))]),await Promise.all([Bun.file(C).delete(),Bun.file($).delete()]),{json:{name:j,path:D},markdown:{name:E,path:y}}}catch(o){if(o instanceof c)throw o;throw new c(o instanceof Error?o.message:"PDF conversion with chunking failed",{source:this.source})}}*generateChunks(n,o){let r=null,e=[],s=null,a=new Set;for(let i of n){let p=i.type;if(!p)continue;if(p==="heading"){if(e.length>0)yield{text:e.join(`
+`),metadata:{heading:r,page:s,pages:Array.from(a),source:o}};let t=this.extractContent(i);r=t,e=t?[t]:[],s=i["page number"]??null,a=new Set(s!==null?[s]:[]);continue}if(p==="paragraph"||p==="list"){let t=this.extractContent(i);if(t){e.push(t);let d=i["page number"];if(d!==void 0)a.add(d)}}}if(e.length>0)yield{text:e.join(`
+`),metadata:{heading:r,page:s,pages:Array.from(a),source:o}}}extractContent(n){let o=Array.from(this.extractTexts(n));return o.length>0?o.join(" "):null}*extractTexts(n){let o=n.content;if(o){yield o;return}let r=n.kids;if(r)for(let e of r)yield*this.extractTexts(e)}}import{container as L,EContainerScope as V}from"@ooneex/container";var M={vectorDatabase:(n=V.Singleton)=>{return(o)=>{L.add(o,n)}}};import{Utf8 as G}from"apache-arrow";class N extends w{getDatabaseUri(){return""}getEmbeddingModel(){return{provider:"openai",model:"text-embedding-ada-002"}}getSchema(){return{name:new G}}}export{M as decorator,h as VectorTable,T as VectorDatabaseException,N as VectorDatabase,c as ConvertorException,v as Convertor,w as AbstractVectorDatabase};
+
+//# debugId=FB64C08DA10152BA64756E2164756E21

package/dist/index.js.map

@@ -0,0 +1,17 @@
+{
+  "version": 3,
+  "sources": ["src/AbstractVectorDatabase.ts", "src/VectorDatabaseException.ts", "src/VectorTable.ts", "src/utils.ts", "src/Convertor.ts", "src/ConvertorException.ts", "src/decorators.ts", "src/VectorDatabase.ts"],
+  "sourcesContent": [
+    "import * as lancedb from \"@lancedb/lancedb\";\nimport \"@lancedb/lancedb/embedding/openai\";\nimport type { EmbeddingFunction } from \"@lancedb/lancedb/embedding\";\nimport { getRegistry, LanceSchema } from \"@lancedb/lancedb/embedding\";\nimport type { ScalarType } from \"@ooneex/types\";\nimport { Utf8 } from \"apache-arrow\";\nimport type { EmbeddingModelType, EmbeddingProviderType, FieldValueType, IVectorDatabase } from \"./types.ts\";\nimport { VectorDatabaseException } from \"./VectorDatabaseException.ts\";\nimport { VectorTable } from \"./VectorTable.ts\";\n\nexport abstract class AbstractVectorDatabase<DataType extends Record<string, ScalarType>>\n  implements IVectorDatabase<DataType>\n{\n  private db: lancedb.Connection | null = null;\n  private embedding: EmbeddingFunction;\n\n  constructor() {\n    const { provider, model } = this.getEmbeddingModel();\n    this.embedding = getRegistry().get(provider)?.create({ model }) as EmbeddingFunction;\n  }\n\n  public abstract getDatabaseUri(): string;\n  public abstract getEmbeddingModel(): { provider: EmbeddingProviderType; model: EmbeddingModelType[\"model\"] };\n  public abstract getSchema(): { [K in keyof DataType]: FieldValueType };\n\n  public async connect(): Promise<void> {\n    this.db = await lancedb.connect(this.getDatabaseUri());\n  }\n\n  public getDatabase(): lancedb.Connection {\n    if (!this.db) {\n      throw new VectorDatabaseException(\"Database not connected. Call connect() first.\");\n    }\n\n    return this.db;\n  }\n\n  public async open(name: string, options?: { mode?: \"create\" | \"overwrite\" }): Promise<VectorTable<DataType>> {\n    const tableNames = await this.getDatabase().tableNames();\n    if (tableNames.includes(name)) {\n      const table = await this.getDatabase().openTable(name);\n      return new VectorTable<DataType>(table);\n    }\n\n    const schema = LanceSchema({\n      id: new Utf8(),\n      text: this.embedding.sourceField(new Utf8()),\n      vector: this.embedding.vectorField(),\n      ...this.getSchema(),\n    });\n    const table = await this.getDatabase().createEmptyTable(name, schema, { mode: \"overwrite\", ...options });\n    await table.createIndex(\"id\", { config: lancedb.Index.btree() });\n    await table.createIndex(\"text\", { config: lancedb.Index.fts() });\n    await table.createIndex(\"vector\", { config: lancedb.Index.ivfPq() });\n\n    return new VectorTable<DataType>(table);\n  }\n}\n",
+    "import { Exception } from \"@ooneex/exception\";\nimport { HttpStatus } from \"@ooneex/http-status\";\n\nexport class VectorDatabaseException extends Exception {\n  constructor(message: string, data: Record<string, unknown> = {}) {\n    super(message, {\n      status: HttpStatus.Code.InternalServerError,\n      data,\n    });\n    this.name = \"VectorDatabaseException\";\n  }\n}\n",
+    "import * as lancedb from \"@lancedb/lancedb\";\nimport type { Filter } from \"./types.ts\";\nimport { buildFilter } from \"./utils.ts\";\n\nexport class VectorTable<DataType extends Record<string, unknown>> {\n  private table: lancedb.Table;\n  private reranker: Awaited<ReturnType<typeof lancedb.rerankers.RRFReranker.create>> | null = null;\n\n  constructor(table: lancedb.Table) {\n    this.table = table;\n  }\n\n  public async add(data: ({ id: string; text: string } & DataType)[]): Promise<this> {\n    await this.table.add(data);\n\n    return this;\n  }\n\n  // Create a scalar index (btree, bitmap, or labelList) on a column used in filters.\n  public async createIndex(\n    column: string,\n    options?: {\n      config?: ReturnType<typeof lancedb.Index.btree | typeof lancedb.Index.bitmap | typeof lancedb.Index.labelList>;\n    },\n  ): Promise<this> {\n    await this.table.createIndex(column, options);\n\n    return this;\n  }\n\n  // Create an IVF PQ vector index for approximate nearest neighbor search.\n  public async createVectorIndex(\n    column = \"vector\",\n    options?: Partial<Parameters<lancedb.Table[\"createIndex\"]>[1] & object>,\n  ): Promise<this> {\n    await this.table.createIndex(column, {\n      config: lancedb.Index.ivfPq(),\n      ...options,\n    });\n\n    return this;\n  }\n\n  public async search(\n    query: string,\n    options?: {\n      limit?: number;\n      select?: (keyof DataType | \"id\" | \"text\")[];\n      filter?: Filter<DataType>;\n      // Number of IVF partitions to search. Higher values improve recall but reduce speed.\n      nprobes?: number;\n      // Multiplier for additional candidate rows during IVF PQ refine step to improve recall accuracy.\n      refineFactor?: number;\n      // Skip un-indexed data for faster queries when indices are up to date.\n      fastSearch?: boolean;\n    },\n  ): Promise<DataType[]> {\n    const { limit = 10, select, filter, nprobes, refineFactor, fastSearch = true } = options ?? {};\n\n    if (!this.reranker) {\n      this.reranker = await lancedb.rerankers.RRFReranker.create();\n    }\n\n    let vectorQuery = (this.table.search(query, \"hybrid\", \"text\") as lancedb.VectorQuery)\n      .rerank(this.reranker)\n      .limit(limit);\n\n    if (nprobes) {\n      vectorQuery = vectorQuery.nprobes(nprobes);\n    }\n\n    if (refineFactor) {\n      vectorQuery = vectorQuery.refineFactor(refineFactor);\n    }\n\n    if (fastSearch) {\n      vectorQuery = vectorQuery.fastSearch();\n    }\n\n    if (select) {\n      vectorQuery = vectorQuery.select(select as string[]);\n    }\n\n    if (filter) {\n      vectorQuery = vectorQuery.where(buildFilter(filter));\n    }\n\n    return vectorQuery.toArray();\n  }\n\n  // Print the resolved query plan to identify slow queries and missing indices.\n  public async explainPlan(\n    query: string,\n    options?: {\n      limit?: number;\n      filter?: Filter<DataType>;\n      verbose?: boolean;\n    },\n  ): Promise<string> {\n    const { limit = 10, filter, verbose = true } = options ?? {};\n\n    if (!this.reranker) {\n      this.reranker = await lancedb.rerankers.RRFReranker.create();\n    }\n\n    let vectorQuery = (this.table.search(query, \"hybrid\", \"text\") as lancedb.VectorQuery)\n      .rerank(this.reranker)\n      .limit(limit);\n\n    if (filter) {\n      vectorQuery = vectorQuery.where(buildFilter(filter));\n    }\n\n    return vectorQuery.explainPlan(verbose);\n  }\n\n  // Execute the query and return a physical plan annotated with runtime metrics.\n  public async analyzePlan(\n    query: string,\n    options?: {\n      limit?: number;\n      filter?: Filter<DataType>;\n    },\n  ): Promise<string> {\n    const { limit = 10, filter } = options ?? {};\n\n    if (!this.reranker) {\n      this.reranker = await lancedb.rerankers.RRFReranker.create();\n    }\n\n    let vectorQuery = (this.table.search(query, \"hybrid\", \"text\") as lancedb.VectorQuery)\n      .rerank(this.reranker)\n      .limit(limit);\n\n    if (filter) {\n      vectorQuery = vectorQuery.where(buildFilter(filter));\n    }\n\n    return vectorQuery.analyzePlan();\n  }\n}\n",
+    "import type { Filter } from \"./types.ts\";\n\nexport const buildFilter = <T>(filter: Filter<T>): string => {\n  if (\"AND\" in filter) {\n    return `(${filter.AND.map(buildFilter).join(\" AND \")})`;\n  }\n  if (\"OR\" in filter) {\n    return `(${filter.OR.map(buildFilter).join(\" OR \")})`;\n  }\n  if (\"NOT\" in filter) {\n    return `NOT (${buildFilter(filter.NOT)})`;\n  }\n\n  const col = String(filter.field);\n\n  if (\n    filter.op === \"IS NULL\" ||\n    filter.op === \"IS NOT NULL\" ||\n    filter.op === \"IS TRUE\" ||\n    filter.op === \"IS NOT TRUE\" ||\n    filter.op === \"IS FALSE\" ||\n    filter.op === \"IS NOT FALSE\"\n  ) {\n    return `${col} ${filter.op}`;\n  }\n\n  if (filter.op === \"IN\") {\n    const values = filter.value.map((v) => (typeof v === \"string\" ? `'${v}'` : v));\n    return `${col} IN (${values.join(\", \")})`;\n  }\n\n  if (filter.op === \"LIKE\" || filter.op === \"NOT LIKE\") {\n    return `${col} ${filter.op} '${filter.value}'`;\n  }\n\n  if (filter.op === \">\" || filter.op === \">=\" || filter.op === \"<\" || filter.op === \"<=\" || filter.op === \"=\") {\n    return `${col} ${filter.op} ${typeof filter.value === \"string\" ? `'${filter.value}'` : filter.value}`;\n  }\n\n  return `${col} ${filter.op} ${typeof filter.value === \"string\" ? `'${filter.value}'` : filter.value}`;\n};\n",
+    "import path from \"node:path\";\nimport { random } from \"@ooneex/utils\";\nimport { convert } from \"@opendataloader/pdf\";\nimport { ConvertorException } from \"./ConvertorException\";\nimport type { ChunkType, ConvertorFileType, ConvertorOptionsType, IConvertor } from \"./types\";\n\nexport class Convertor implements IConvertor {\n  private readonly source: string;\n\n  constructor(source: string) {\n    this.source = path.join(...source.split(/[/\\\\]/));\n  }\n\n  public async *convert(\n    options: ConvertorOptionsType = {},\n  ): AsyncGenerator<ChunkType, { json: ConvertorFileType; markdown: ConvertorFileType }> {\n    try {\n      const subDir = random.nanoid(15);\n      const outputDir = path.join(options.outputDir ?? \"\", subDir);\n      const { password, imageFormat, quiet, pages } = options;\n\n      await convert([this.source], {\n        outputDir,\n        format: \"json,markdown\",\n        imageDir: path.join(outputDir, \"images\"),\n        imageOutput: \"external\",\n        ...(password !== undefined && { password }),\n        ...(imageFormat !== undefined && { imageFormat }),\n        ...(quiet !== undefined && { quiet }),\n        ...(pages !== undefined && { pages }),\n      });\n\n      const glob = new Bun.Glob(\"*\");\n      let jsonFile: string | undefined;\n      let mdFile: string | undefined;\n      for await (const file of glob.scan(outputDir)) {\n        if (!jsonFile && file.endsWith(\".json\")) jsonFile = file;\n        if (!mdFile && file.endsWith(\".md\")) mdFile = file;\n        if (jsonFile && mdFile) break;\n      }\n\n      if (!jsonFile) {\n        throw new ConvertorException(\"No JSON output file found after conversion\", { source: this.source });\n      }\n\n      if (!mdFile) {\n        throw new ConvertorException(\"No Markdown output file found after conversion\", { source: this.source });\n      }\n\n      const jsonPath = path.join(outputDir, jsonFile);\n      const doc = await Bun.file(jsonPath).json();\n      const fileName = doc[\"file name\"] ?? path.basename(this.source);\n\n      yield* this.generateChunks(doc.kids ?? [], fileName);\n\n      const renamedJson = `${random.nanoid(15)}.json`;\n      const renamedMd = `${random.nanoid(15)}.md`;\n\n      const mdPath = path.join(outputDir, mdFile);\n      const renamedJsonPath = path.join(outputDir, renamedJson);\n      const renamedMdPath = path.join(outputDir, renamedMd);\n\n      await Promise.all([Bun.write(renamedJsonPath, Bun.file(jsonPath)), Bun.write(renamedMdPath, Bun.file(mdPath))]);\n\n      await Promise.all([Bun.file(jsonPath).delete(), Bun.file(mdPath).delete()]);\n\n      return {\n        json: { name: renamedJson, path: renamedJsonPath },\n        markdown: { name: renamedMd, path: renamedMdPath },\n      };\n    } catch (error) {\n      if (error instanceof ConvertorException) throw error;\n      throw new ConvertorException(error instanceof Error ? error.message : \"PDF conversion with chunking failed\", {\n        source: this.source,\n      });\n    }\n  }\n\n  private *generateChunks(kids: Record<string, unknown>[], source: string): Generator<ChunkType> {\n    let currentHeading: string | null = null;\n    let currentContent: string[] = [];\n    let startPage: number | null = null;\n    let pageSet = new Set<number>();\n\n    for (const element of kids) {\n      const type = element.type as string | undefined;\n      if (!type) continue;\n\n      if (type === \"heading\") {\n        if (currentContent.length > 0) {\n          yield {\n            text: currentContent.join(\"\\n\"),\n            metadata: { heading: currentHeading, page: startPage, pages: Array.from(pageSet), source },\n          };\n        }\n        const content = this.extractContent(element);\n        currentHeading = content;\n        currentContent = content ? [content] : [];\n        startPage = (element[\"page number\"] as number) ?? null;\n        pageSet = new Set(startPage !== null ? [startPage] : []);\n        continue;\n      }\n\n      if (type === \"paragraph\" || type === \"list\") {\n        const content = this.extractContent(element);\n        if (content) {\n          currentContent.push(content);\n          const page = element[\"page number\"] as number | undefined;\n          if (page !== undefined) {\n            pageSet.add(page);\n          }\n        }\n      }\n    }\n\n    if (currentContent.length > 0) {\n      yield {\n        text: currentContent.join(\"\\n\"),\n        metadata: { heading: currentHeading, page: startPage, pages: Array.from(pageSet), source },\n      };\n    }\n  }\n\n  private extractContent(element: Record<string, unknown>): string | null {\n    const parts = Array.from(this.extractTexts(element));\n    return parts.length > 0 ? parts.join(\" \") : null;\n  }\n\n  private *extractTexts(element: Record<string, unknown>): Generator<string> {\n    const content = element.content as string | undefined;\n    if (content) {\n      yield content;\n      return;\n    }\n\n    const kids = element.kids as Record<string, unknown>[] | undefined;\n    if (kids) {\n      for (const kid of kids) {\n        yield* this.extractTexts(kid);\n      }\n    }\n  }\n}\n",
+    "import { Exception } from \"@ooneex/exception\";\nimport { HttpStatus } from \"@ooneex/http-status\";\n\nexport class ConvertorException extends Exception {\n  constructor(message: string, data: Record<string, unknown> = {}) {\n    super(message, {\n      status: HttpStatus.Code.InternalServerError,\n      data,\n    });\n    this.name = \"ConvertorException\";\n  }\n}\n",
+    "import { container, EContainerScope } from \"@ooneex/container\";\nimport type { VectorDatabaseClassType } from \"./types\";\n\nexport const decorator = {\n  vectorDatabase: (scope: EContainerScope = EContainerScope.Singleton) => {\n    return (target: VectorDatabaseClassType): void => {\n      container.add(target, scope);\n    };\n  },\n};\n",
+    "import { AbstractVectorDatabase } from \"./AbstractVectorDatabase.ts\";\nimport type { EmbeddingModelType, EmbeddingProviderType, FieldValueType } from \"./types.ts\";\nimport { Utf8 } from \"apache-arrow\";\n\ntype DataType = {\n  name: string;\n}\n\nexport class VectorDatabase extends AbstractVectorDatabase<DataType> {\n  public getDatabaseUri(): string {\n    return \"\";\n  }\n\n  public getEmbeddingModel(): { provider: EmbeddingProviderType; model: EmbeddingModelType[\"model\"] } {\n    return { provider: \"openai\", model: \"text-embedding-ada-002\" };\n  }\n\n  public getSchema(): { [K in keyof DataType]: FieldValueType } {\n    return {\n      name: new Utf8(),\n    };\n  }\n}\n"
+  ],
+  "mappings": ";AAAA,mCACA,0CAEA,sBAAS,iBAAa,mCAEtB,eAAS,qBCLT,oBAAS,0BACT,qBAAS,4BAEF,MAAM,UAAgC,CAAU,CACrD,WAAW,CAAC,EAAiB,EAAgC,CAAC,EAAG,CAC/D,MAAM,EAAS,CACb,OAAQ,EAAW,KAAK,oBACxB,MACF,CAAC,EACD,KAAK,KAAO,0BAEhB,CCXA,mCCEO,IAAM,EAAc,CAAI,IAA8B,CAC3D,GAAI,QAAS,EACX,MAAO,IAAI,EAAO,IAAI,IAAI,CAAW,EAAE,KAAK,OAAO,KAErD,GAAI,OAAQ,EACV,MAAO,IAAI,EAAO,GAAG,IAAI,CAAW,EAAE,KAAK,MAAM,KAEnD,GAAI,QAAS,EACX,MAAO,QAAQ,EAAY,EAAO,GAAG,KAGvC,IAAM,EAAM,OAAO,EAAO,KAAK,EAE/B,GACE,EAAO,KAAO,WACd,EAAO,KAAO,eACd,EAAO,KAAO,WACd,EAAO,KAAO,eACd,EAAO,KAAO,YACd,EAAO,KAAO,eAEd,MAAO,GAAG,KAAO,EAAO,KAG1B,GAAI,EAAO,KAAO,KAAM,CACtB,IAAM,EAAS,EAAO,MAAM,IAAI,CAAC,IAAO,OAAO,IAAM,SAAW,IAAI,KAAO,CAAE,EAC7E,MAAO,GAAG,SAAW,EAAO,KAAK,IAAI,KAGvC,GAAI,EAAO,KAAO,QAAU,EAAO,KAAO,WACxC,MAAO,GAAG,KAAO,EAAO,OAAO,EAAO,SAGxC,GAAI,EAAO,KAAO,KAAO,EAAO,KAAO,MAAQ,EAAO,KAAO,KAAO,EAAO,KAAO,MAAQ,EAAO,KAAO,IACtG,MAAO,GAAG,KAAO,EAAO,MAAM,OAAO,EAAO,QAAU,SAAW,IAAI,EAAO,SAAW,EAAO,QAGhG,MAAO,GAAG,KAAO,EAAO,MAAM,OAAO,EAAO,QAAU,SAAW,IAAI,EAAO,SAAW,EAAO,SDnCzF,MAAM,CAAsD,CACzD,MACA,SAAoF,KAE5F,WAAW,CAAC,EAAsB,CAChC,KAAK,MAAQ,OAGF,IAAG,CAAC,EAAkE,CAGjF,OAFA,MAAM,KAAK,MAAM,IAAI,CAAI,EAElB,UAII,YAAW,CACtB,EACA,EAGe,CAGf,OAFA,MAAM,KAAK,MAAM,YAAY,EAAQ,CAAO,EAErC,UAII,kBAAiB,CAC5B,EAAS,SACT,EACe,CAMf,OALA,MAAM,KAAK,MAAM,YAAY,EAAQ,CACnC,OAAgB,QAAM,MAAM,KACzB,CACL,CAAC,EAEM,UAGI,OAAM,CACjB,EACA,EAWqB,CACrB,IAAQ,QAAQ,GAAI,SAAQ,SAAQ,UAAS,eAAc,aAAa,IAAS,GAAW,CAAC,EAE7F,GAAI,CAAC,KAAK,SACR,KAAK,SAAW,MAAc,YAAU,YAAY,OAAO,EAG7D,IAAI,EAAe,KAAK,MAAM,OAAO,EAAO,SAAU,MAAM,EACzD,OAAO,KAAK,QAAQ,EACpB,MAAM,CAAK,EAEd,GAAI,EACF,EAAc,EAAY,QAAQ,CAAO,EAG3C,GAAI,EACF,EAAc,EAAY,aAAa,CAAY,EAGrD,GAAI,EACF,EAAc,EAAY,WAAW,EAGvC,GAAI,EACF,EAAc,EAAY,OAAO,CAAkB,EAGrD,GAAI,EACF,EAAc,EAAY,MAAM,EAAY,CAAM,CAAC,EAGrD,OAAO,EAAY,QAAQ,OAIhB,YAAW,CACtB,EACA,EAKiB,CACjB,IAAQ,QAAQ,GAAI,SAAQ,UAAU,IAAS,GAAW,CAAC,EAE3D,GAAI,CAAC,KAAK,SACR,KAAK,SAAW,MAAc,YAAU,YAAY,OAAO,EAG7D,IAAI,EAAe,KAAK,MAAM,OAAO,EAAO,SAAU,MAAM,EACzD,OAAO,KAAK,QAAQ,EACpB,MAAM,CAAK,EAEd,GAAI,EACF,EAAc,EAAY,MAAM,EAAY,CAAM,CAAC,EAGrD,OAAO,EAAY,YAAY,CAAO,OAI3B,YAAW,CACtB,EACA,EAIiB,CACjB,IAAQ,QAAQ,GAAI,UAAW,GAAW,CAAC,EAE3C,GAAI,CAAC,KAAK,SACR,KAAK,SAAW,MAAc,YAAU,YAAY,OAAO,EAG7D,IAAI,EAAe,KAAK,MAAM,OAAO,EAAO,SAAU,MAAM,EACzD,OAAO,KAAK,QAAQ,EACpB,MAAM,CAAK,EAEd,GAAI,EACF,EAAc,EAAY,MAAM,EAAY,CAAM,CAAC,EAGrD,OAAO,EAAY,YAAY,EAEnC,CFlIO,MAAe,CAEtB,CACU,GAAgC,KAChC,UAER,WAAW,EAAG,CACZ,IAAQ,WAAU,SAAU,KAAK,kBAAkB,EACnD,KAAK,UAAY,EAAY,EAAE,IAAI,CAAQ,GAAG,OAAO,CAAE,OAAM,CAAC,OAOnD,QAAO,EAAkB,CACpC,KAAK,GAAK,MAAc,UAAQ,KAAK,eAAe,CAAC,EAGhD,WAAW,EAAuB,CACvC,GAAI,CAAC,KAAK,GACR,MAAM,IAAI,EAAwB,+CAA+C,EAGnF,OAAO,KAAK,QAGD,KAAI,CAAC,EAAc,EAA6E,CAE3G,IADmB,MAAM,KAAK,YAAY,EAAE,WAAW,GACxC,SAAS,CAAI,EAAG,CAC7B,IAAM,EAAQ,MAAM,KAAK,YAAY,EAAE,UAAU,CAAI,EACrD,OAAO,IAAI,EAAsB,CAAK,EAGxC,IAAM,EAAS,EAAY,CACzB,GAAI,IAAI,EACR,KAAM,KAAK,UAAU,YAAY,IAAI,CAAM,EAC3C,OAAQ,KAAK,UAAU,YAAY,KAChC,KAAK,UAAU,CACpB,CAAC,EACK,EAAQ,MAAM,KAAK,YAAY,EAAE,iBAAiB,EAAM,EAAQ,CAAE,KAAM,eAAgB,CAAQ,CAAC,EAKvG,OAJA,MAAM,EAAM,YAAY,KAAM,CAAE,OAAgB,QAAM,MAAM,CAAE,CAAC,EAC/D,MAAM,EAAM,YAAY,OAAQ,CAAE,OAAgB,QAAM,IAAI,CAAE,CAAC,EAC/D,MAAM,EAAM,YAAY,SAAU,CAAE,OAAgB,QAAM,MAAM,CAAE,CAAC,EAE5D,IAAI,EAAsB,CAAK,EAE1C,CIzDA,oBACA,iBAAS,sBACT,kBAAS,4BCFT,oBAAS,0BACT,qBAAS,4BAEF,MAAM,UAA2B,CAAU,CAChD,WAAW,CAAC,EAAiB,EAAgC,CAAC,EAAG,CAC/D,MAAM,EAAS,CACb,OAAQ,EAAW,KAAK,oBACxB,MACF,CAAC,EACD,KAAK,KAAO,qBAEhB,CDLO,MAAM,CAAgC,CAC1B,OAEjB,WAAW,CAAC,EAAgB,CAC1B,KAAK,OAAS,EAAK,KAAK,GAAG,EAAO,MAAM,OAAO,CAAC,QAGpC,OAAO,CACnB,EAAgC,CAAC,EACoD,CACrF,GAAI,CACF,IAAM,EAAS,EAAO,OAAO,EAAE,EACzB,EAAY,EAAK,KAAK,EAAQ,WAAa,GAAI,CAAM,GACnD,WAAU,cAAa,QAAO,SAAU,EAEhD,MAAM,EAAQ,CAAC,KAAK,MAAM,EAAG,CAC3B,YACA,OAAQ,gBACR,SAAU,EAAK,KAAK,EAAW,QAAQ,EACvC,YAAa,cACT,IAAa,QAAa,CAAE,UAAS,KACrC,IAAgB,QAAa,CAAE,aAAY,KAC3C,IAAU,QAAa,CAAE,OAAM,KAC/B,IAAU,QAAa,CAAE,OAAM,CACrC,CAAC,EAED,IAAM,EAAO,IAAI,IAAI,KAAK,GAAG,EACzB,EACA,EACJ,cAAiB,KAAQ,EAAK,KAAK,CAAS,EAAG,CAC7C,GAAI,CAAC,GAAY,EAAK,SAAS,OAAO,EAAG,EAAW,EACpD,GAAI,CAAC,GAAU,EAAK,SAAS,KAAK,EAAG,EAAS,EAC9C,GAAI,GAAY,EAAQ,MAG1B,GAAI,CAAC,EACH,MAAM,IAAI,EAAmB,6CAA8C,CAAE,OAAQ,KAAK,MAAO,CAAC,EAGpG,GAAI,CAAC,EACH,MAAM,IAAI,EAAmB,iDAAkD,CAAE,OAAQ,KAAK,MAAO,CAAC,EAGxG,IAAM,EAAW,EAAK,KAAK,EAAW,CAAQ,EACxC,EAAM,MAAM,IAAI,KAAK,CAAQ,EAAE,KAAK,EACpC,EAAW,EAAI,cAAgB,EAAK,SAAS,KAAK,MAAM,EAE9D,MAAO,KAAK,eAAe,EAAI,MAAQ,CAAC,EAAG,CAAQ,EAEnD,IAAM,EAAc,GAAG,EAAO,OAAO,EAAE,SACjC,EAAY,GAAG,EAAO,OAAO,EAAE,OAE/B,EAAS,EAAK,KAAK,EAAW,CAAM,EACpC,EAAkB,EAAK,KAAK,EAAW,CAAW,EAClD,EAAgB,EAAK,KAAK,EAAW,CAAS,EAMpD,OAJA,MAAM,QAAQ,IAAI,CAAC,IAAI,MAAM,EAAiB,IAAI,KAAK,CAAQ,CAAC,EAAG,IAAI,MAAM,EAAe,IAAI,KAAK,CAAM,CAAC,CAAC,CAAC,EAE9G,MAAM,QAAQ,IAAI,CAAC,IAAI,KAAK,CAAQ,EAAE,OAAO,EAAG,IAAI,KAAK,CAAM,EAAE,OAAO,CAAC,CAAC,EAEnE,CACL,KAAM,CAAE,KAAM,EAAa,KAAM,CAAgB,EACjD,SAAU,CAAE,KAAM,EAAW,KAAM,CAAc,CACnD,EACA,MAAO,EAAO,CACd,GAAI,aAAiB,EAAoB,MAAM,EAC/C,MAAM,IAAI,EAAmB,aAAiB,MAAQ,EAAM,QAAU,sCAAuC,CAC3G,OAAQ,KAAK,MACf,CAAC,IAII,cAAc,CAAC,EAAiC,EAAsC,CAC7F,IAAI,EAAgC,KAChC,EAA2B,CAAC,EAC5B,EAA2B,KAC3B,EAAU,IAAI,IAElB,QAAW,KAAW,EAAM,CAC1B,IAAM,EAAO,EAAQ,KACrB,GAAI,CAAC,EAAM,SAEX,GAAI,IAAS,UAAW,CACtB,GAAI,EAAe,OAAS,EAC1B,KAAM,CACJ,KAAM,EAAe,KAAK;AAAA,CAAI,EAC9B,SAAU,CAAE,QAAS,EAAgB,KAAM,EAAW,MAAO,MAAM,KAAK,CAAO,EAAG,QAAO,CAC3F,EAEF,IAAM,EAAU,KAAK,eAAe,CAAO,EAC3C,EAAiB,EACjB,EAAiB,EAAU,CAAC,CAAO,EAAI,CAAC,EACxC,EAAa,EAAQ,gBAA6B,KAClD,EAAU,IAAI,IAAI,IAAc,KAAO,CAAC,CAAS,EAAI,CAAC,CAAC,EACvD,SAGF,GAAI,IAAS,aAAe,IAAS,OAAQ,CAC3C,IAAM,EAAU,KAAK,eAAe,CAAO,EAC3C,GAAI,EAAS,CACX,EAAe,KAAK,CAAO,EAC3B,IAAM,EAAO,EAAQ,eACrB,GAAI,IAAS,OACX,EAAQ,IAAI,CAAI,IAMxB,GAAI,EAAe,OAAS,EAC1B,KAAM,CACJ,KAAM,EAAe,KAAK;AAAA,CAAI,EAC9B,SAAU,CAAE,QAAS,EAAgB,KAAM,EAAW,MAAO,MAAM,KAAK,CAAO,EAAG,QAAO,CAC3F,EAII,cAAc,CAAC,EAAiD,CACtE,IAAM,EAAQ,MAAM,KAAK,KAAK,aAAa,CAAO,CAAC,EACnD,OAAO,EAAM,OAAS,EAAI,EAAM,KAAK,GAAG,EAAI,MAGrC,YAAY,CAAC,EAAqD,CACzE,IAAM,EAAU,EAAQ,QACxB,GAAI,EAAS,CACX,MAAM,EACN,OAGF,IAAM,EAAO,EAAQ,KACrB,GAAI,EACF,QAAW,KAAO,EAChB,MAAO,KAAK,aAAa,CAAG,EAIpC,CE9IA,oBAAS,qBAAW,0BAGb,IAAM,EAAY,CACvB,eAAgB,CAAC,EAAyB,EAAgB,YAAc,CACtE,MAAO,CAAC,IAA0C,CAChD,EAAU,IAAI,EAAQ,CAAK,GAGjC,ECPA,eAAS,qBAMF,MAAM,UAAuB,CAAiC,CAC5D,cAAc,EAAW,CAC9B,MAAO,GAGF,iBAAiB,EAA4E,CAClG,MAAO,CAAE,SAAU,SAAU,MAAO,wBAAyB,EAGxD,SAAS,EAA8C,CAC5D,MAAO,CACL,KAAM,IAAI,CACZ,EAEJ",
+  "debugId": "FB64C08DA10152BA64756E2164756E21",
+  "names": []
+}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@ooneex/rag",
+  "description": "Retrieval-Augmented Generation toolkit with vector database integration, document embedding, and semantic search for AI-powered knowledge retrieval",
+  "version": "1.0.0",
+  "type": "module",
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md",
+    "package.json"
+  ],
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "license": "MIT",
+  "scripts": {
+    "test": "bun test tests",
+    "build": "bunup",
+    "lint": "tsgo --noEmit && bunx biome lint",
+    "npm:publish": "bun publish --tolerate-republish --access public"
+  },
+  "keywords": [],
+  "dependencies": {
+    "@lancedb/lancedb": "^0.27.1",
+    "@ooneex/exception": "0.0.18",
+    "@ooneex/container": "0.0.19",
+    "@ooneex/http-status": "0.0.18",
+    "@ooneex/utils": "0.1.1",
+    "@opendataloader/pdf": "^2.0.2",
+    "apache-arrow": "^21.1.0"
+  },
+  "devDependencies": {
+    "@ooneex/types": "0.0.19"
+  }
+}