@ixo/data-store 1.0.1

package/.eslintrc.js

@@ -0,0 +1,9 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  extends: ['@ixo/eslint-config/nest.js'],
+  parserOptions: {
+    project: 'tsconfig.json',
+    tsconfigRootDir: __dirname,
+    sourceType: 'module',
+  },
+};

package/.prettierignore

@@ -0,0 +1,3 @@
+pnpm-lock.yaml
+
+node_modules

package/.prettierrc.js

@@ -0,0 +1,4 @@
+/** @type {import("prettier").Config} */
+module.exports = {
+  ...require('@ixo/eslint-config/prettier-base'),
+};

package/.turbo/turbo-build.log

@@ -0,0 +1,4 @@
+
+> @ixo/data-store@1.0.1 build /home/runner/actions-runner/_work/ixo-oracles-boilerplate/ixo-oracles-boilerplate/packages/data-store
+> tsc
+

package/CHANGELOG.md

@@ -0,0 +1,7 @@
+# @ixo/data-store
+
+## 1.0.1
+
+### Patch Changes
+
+- [#53](https://github.com/ixoworld/ixo-oracles-boilerplate/pull/53) [`0a4a5a8`](https://github.com/ixoworld/ixo-oracles-boilerplate/commit/0a4a5a84194acb851e3824e0b74eea54f60c8257) Thanks [@youssefhany-ixo](https://github.com/youssefhany-ixo)! - Upgrade packages and publish events package and preformance upgrades

package/README.md

@@ -0,0 +1,276 @@
+# @ixo/data-store
+
+## Overview
+
+The `@ixo/data-store` package provides abstract interfaces and implementations for both vector and structured data storage in the ixo-oracles ecosystem. It offers a type-safe, consistent API for managing data across different storage backends.
+
+### Key Features
+
+- 🔍 Vector database abstraction with ChromaDB implementation
+- 📊 Structured data storage with Airtable implementation
+- 🔐 Type-safe interfaces for data operations
+- 🎯 Flexible query capabilities
+- 🔄 Batch operations support
+- 🏗️ Extensible architecture for custom implementations
+
+## Table of Contents
+
+1. [Getting Started](#getting-started)
+   - [Installation](#installation)
+   - [Configuration](#configuration)
+2. [Vector Database Storage](#vector-database-storage)
+   - [Implementing Vector Storage](#implementing-vector-storage)
+   - [Using ChromaDB Implementation](#using-chromadb-implementation)
+3. [Structured Data Storage](#structured-data-storage)
+   - [Implementing Structured Storage](#implementing-structured-storage)
+   - [Using Airtable Implementation](#using-airtable-implementation)
+4. [Development](#development)
+
+## Getting Started
+
+### Installation
+
+```bash
+# Install using pnpm (recommended)
+pnpm install @ixo/data-store
+
+# Or using npm
+npm install @ixo/data-store
+
+# Or using yarn
+yarn add @ixo/data-store
+```
+
+### Configuration
+
+#### For ChromaDB Vector Storage
+
+```bash
+OPENAI_API_KEY=your_openai_api_key    # Required for embeddings
+```
+
+#### Running Chroma Backend
+
+The ChromaDB implementation requires a running Chroma backend. You can easily run it using Docker:
+
+```bash
+# Pull the Chroma image
+docker pull chromadb/chroma
+
+# Run the Chroma container
+docker run -p 8000:8000 chromadb/chroma
+```
+
+This will start the Chroma backend server on `http://localhost:8000`.
+
+#### For Airtable Structured Storage
+
+```bash
+AIRTABLE_API_KEY=your_airtable_key    # Required for Airtable operations
+AIRTABLE_BASE_ID=your_base_id         # Required for Airtable operations
+AITABLE_BASE_TABLE_LINK=your_link     # Optional, for record links
+```
+
+## Vector Database Storage
+
+The vector database interface provides methods for storing, retrieving, and querying vector embeddings of documents.
+
+### Implementing Vector Storage
+
+To create a custom vector storage implementation, extend the `VectorDBDataStore` abstract class:
+
+```typescript
+import {
+  VectorDBDataStore,
+  IVectorStoreDocument,
+  IVectorStoreOptions,
+} from '@ixo/data-store';
+
+class CustomVectorStore extends VectorDBDataStore {
+  constructor(options: IVectorStoreOptions) {
+    super(options);
+  }
+
+  async init(): Promise<void> {
+    // Initialize your vector store
+  }
+
+  async upsert(documents: IVectorStoreDocument[]): Promise<void> {
+    // Implement document upsertion
+  }
+
+  async delete(ids: string[]): Promise<void> {
+    // Implement document deletion
+  }
+
+  async query(
+    query: string,
+    options?: IVectorStoreQueryOptions,
+  ): Promise<IVectorStoreDocument[]> {
+    // Implement text-based querying
+  }
+
+  async queryByVector(
+    vector: number[],
+    options?: IVectorStoreQueryOptions,
+  ): Promise<IVectorStoreDocument[]> {
+    // Implement vector-based querying
+  }
+
+  async getById(id: string): Promise<IVectorStoreDocument | null> {
+    // Implement document retrieval by ID
+  }
+
+  async queryWithSimilarity(
+    query: string,
+    options?: IVectorStoreQueryOptions & { similarityThreshold: number },
+  ): Promise<IVectorStoreDocument[]> {
+    // Implement similarity-based querying
+  }
+}
+```
+
+### Using ChromaDB Implementation
+
+The package includes a ChromaDB implementation:
+
+```typescript
+import { ChromaDataStore } from '@ixo/data-store';
+
+const store = new ChromaDataStore({
+  collectionName: 'my-collection',
+  url: 'http://localhost:8000',
+});
+
+await store.init();
+
+// Store documents
+await store.upsert([
+  {
+    id: '1',
+    content: 'Document content',
+    metadata: { type: 'article' },
+  },
+]);
+
+// Query documents
+const results = await store.query('search query', {
+  topK: 5,
+  filters: { type: 'article' },
+});
+
+// Query with similarity threshold
+const similarDocs = await store.queryWithSimilarity('query', {
+  similarityThreshold: 0.8,
+});
+```
+
+## Structured Data Storage
+
+The structured data interface provides CRUD operations for structured data storage.
+
+### Implementing Structured Storage
+
+To create a custom structured storage implementation, implement the `IDataStore` interface:
+
+```typescript
+import { IDataStore } from '@ixo/data-store';
+
+class CustomDataStore<T> implements IDataStore<T> {
+  async getAllRecords(
+    tableName: string,
+    selectOptions: IQueryParams<T>,
+  ): Promise<T[]> {
+    // Implement fetching all records
+  }
+
+  async getRecord(tableName: string, recordId: string): Promise<T> {
+    // Implement single record retrieval
+  }
+
+  async createRecord(tableName: string, recordData: T): Promise<T> {
+    // Implement record creation
+  }
+
+  async updateRecord(
+    tableName: string,
+    recordId: string,
+    recordData: T,
+  ): Promise<T> {
+    // Implement record update
+  }
+
+  async batchUpdateRecords(
+    tableName: string,
+    records: { id: string; fields: T }[],
+  ): Promise<T[]> {
+    // Implement batch update
+  }
+
+  async deleteRecord(tableName: string, recordId: string): Promise<T> {
+    // Implement record deletion
+  }
+
+  async getRecordByField(
+    tableName: string,
+    fieldName: string,
+    fieldValue: string,
+  ): Promise<T[]> {
+    // Implement field-based retrieval
+  }
+}
+```
+
+### Using Airtable Implementation
+
+The package includes an Airtable implementation:
+
+```typescript
+import { AirtableDataStore, FieldSet } from '@ixo/data-store';
+
+interface MyRecord extends FieldSet {
+  name: string;
+  description: string;
+}
+
+const store = new AirtableDataStore<MyRecord>();
+
+// Create a record
+const record = await store.createRecord('tableName', {
+  name: 'Test',
+  description: 'Description',
+});
+
+// Get all records
+const records = await store.getAllRecords('tableName', {
+  maxRecords: 100,
+  view: 'Grid view',
+});
+
+// Update records in batch
+const updated = await store.batchUpdateRecords('tableName', [
+  { id: '1', fields: { name: 'Updated' } },
+  { id: '2', fields: { name: 'Also Updated' } },
+]);
+```
+
+## Development
+
+### Testing
+
+```bash
+# Run tests
+pnpm test
+
+```
+
+### Contributing
+
+1. Implement the appropriate interface (`VectorDBDataStore` or `IDataStore`)
+2. Add comprehensive tests
+3. Document your implementation
+4. Submit a pull request
+
+## License
+
+Internal package - All rights reserved.

package/dist/airtable-store/index.d.ts

@@ -0,0 +1,42 @@
+import { type FieldSet } from 'airtable';
+import { type QueryParams } from 'airtable/lib/query_params';
+import { type IDataStore } from '../types';
+/**
+ * AirtableDataStore class to interact with Airtable API implementing IDataStore interface
+ *
+ * Basic CRUD operations
+ */
+export declare class AirtableDataStore<TFields extends FieldSet> implements IDataStore<TFields> {
+    private readonly base;
+    constructor();
+    private static withId;
+    getAllRecords(tableName: string, selectOptions?: QueryParams<TFields>): Promise<(TFields & {
+        id: string;
+    })[]>;
+    getRecord(tableName: string, recordId: string): Promise<TFields & {
+        id: string;
+    }>;
+    createRecord(tableName: string, recordData: TFields): Promise<TFields & {
+        id: string;
+    }>;
+    updateRecord(tableName: string, recordId: string, recordData: Partial<TFields>): Promise<TFields & {
+        id: string;
+    }>;
+    batchUpdateRecords(tableName: string, records: {
+        id: string;
+        fields: Partial<TFields & {
+            batch?: boolean;
+        }>;
+    }[]): Promise<(TFields & {
+        id: string;
+    })[]>;
+    deleteRecord(tableName: string, recordId: string): Promise<TFields & {
+        id: string;
+    }>;
+    getRecordByField(tableName: string, fieldName: string, fieldValue: string): Promise<(TFields & {
+        id: string;
+    })[]>;
+    getLinkToRecord(recordId: string): string;
+}
+export type { FieldSet };
+//# sourceMappingURL=index.d.ts.map

package/dist/airtable-store/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/airtable-store/index.ts"],"names":[],"mappings":"AAAA,OAAqB,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,KAAK,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAC7D,OAAO,EAAE,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AAG3C;;;;GAIG;AACH,qBAAa,iBAAiB,CAAC,OAAO,SAAS,QAAQ,CACrD,YAAW,UAAU,CAAC,OAAO,CAAC;IAE9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAoB;;IAgBzC,OAAO,CAAC,MAAM,CAAC,MAAM;IASf,aAAa,CACjB,SAAS,EAAE,MAAM,EACjB,aAAa,CAAC,EAAE,WAAW,CAAC,OAAO,CAAC,GACnC,OAAO,CAAC,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IAQlC,SAAS,CACb,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAQ9B,YAAY,CAChB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,OAAO,GAClB,OAAO,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAO9B,YAAY,CAChB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,GAC3B,OAAO,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAgB9B,kBAAkB,CACtB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE;QACP,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,OAAO,CACb,OAAO,GAAG;YACR,KAAK,CAAC,EAAE,OAAO,CAAC;SACjB,CACF,CAAC;KACH,EAAE,GACF,OAAO,CAAC,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IAOlC,YAAY,CAChB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAO9B,gBAAgB,CACpB,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,CAAC,OAAO,GAAG;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC,EAAE,CAAC;IAWjC,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;CAMjD;AAED,YAAY,EAAE,QAAQ,EAAE,CAAC"}

package/dist/airtable-store/index.js

@@ -0,0 +1,79 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AirtableDataStore = void 0;
+const airtable_1 = __importDefault(require("airtable"));
+const utils_1 = require("../utils");
+/**
+ * AirtableDataStore class to interact with Airtable API implementing IDataStore interface
+ *
+ * Basic CRUD operations
+ */
+class AirtableDataStore {
+    base;
+    constructor() {
+        if (!process.env.AIRTABLE_API_KEY) {
+            throw new Error('AIRTABLE_API_KEY is required');
+        }
+        if (!process.env.AIRTABLE_BASE_ID) {
+            throw new Error('AIRTABLE_BASE_ID is required');
+        }
+        this.base = new airtable_1.default({ apiKey: process.env.AIRTABLE_API_KEY }).base(process.env.AIRTABLE_BASE_ID);
+    }
+    static withId(record) {
+        return {
+            ...record.fields,
+            id: record.id,
+        };
+    }
+    async getAllRecords(tableName, selectOptions) {
+        const records = await (0, utils_1.withReportError)(this.base(tableName).select(selectOptions).all());
+        return records.map((record) => AirtableDataStore.withId(record));
+    }
+    async getRecord(tableName, recordId) {
+        const record = await (0, utils_1.withReportError)(this.base(tableName).find(recordId));
+        return AirtableDataStore.withId(record);
+    }
+    async createRecord(tableName, recordData) {
+        const record = await (0, utils_1.withReportError)(this.base(tableName).create(recordData));
+        return AirtableDataStore.withId(record);
+    }
+    async updateRecord(tableName, recordId, recordData) {
+        const record = await (0, utils_1.withReportError)(this.base(tableName).update([
+            {
+                id: recordId,
+                fields: recordData,
+            },
+        ]));
+        if (!record[0]) {
+            // this error will never be thrown as the airtable sdk will throw an error but to satisfy typescript
+            throw new Error('Record not found');
+        }
+        return AirtableDataStore.withId(record[0]);
+    }
+    async batchUpdateRecords(tableName, records) {
+        const updatedRecords = await (0, utils_1.withReportError)(this.base(tableName).update(records));
+        return updatedRecords.map((record) => AirtableDataStore.withId(record));
+    }
+    async deleteRecord(tableName, recordId) {
+        const record = await (0, utils_1.withReportError)(this.base(tableName).destroy(recordId));
+        return AirtableDataStore.withId(record);
+    }
+    async getRecordByField(tableName, fieldName, fieldValue) {
+        const records = await (0, utils_1.withReportError)(this.base(tableName)
+            .select({
+            filterByFormula: `{${fieldName}} = "${fieldValue}"`,
+        })
+            .all());
+        return records.map((record) => AirtableDataStore.withId(record));
+    }
+    getLinkToRecord(recordId) {
+        if (!process.env.AITABLE_BASE_TABLE_LINK) {
+            throw new Error('AITABLE_BASE_TABLE_LINK is required');
+        }
+        return `${process.env.AITABLE_BASE_TABLE_LINK}/${recordId}`;
+    }
+}
+exports.AirtableDataStore = AirtableDataStore;

package/dist/chroma/chroma-data-store.d.ts

@@ -0,0 +1,74 @@
+import { type Metadata } from 'chromadb';
+import { VectorDBDataStore, type IVectorStoreDocument, type IVectorStoreDocumentWithEmbeddings, type IVectorStoreOptions, type IVectorStoreQueryOptions } from '../types/vector-db-data-store';
+import { type IChromaMetadataFilter } from './chroma.types';
+/**
+ * ChromaDataStore class for managing vector storage and retrieval using ChromaDB
+ */
+declare class ChromaDataStore extends VectorDBDataStore {
+    private readonly client;
+    private collection;
+    /**
+     * Creates a new ChromaDataStore instance
+     * @param options - Configuration options for the vector store
+     * @throws Error if OPENAI_API_KEY is not set and no embedding function is provided
+     */
+    constructor(options: IVectorStoreOptions);
+    /**
+     * Checks if the ChromaDB collection is initialized
+     * @throws Error if collection is not initialized
+     */
+    private checkIsInitialized;
+    /**
+     * Initializes the ChromaDB collection
+     */
+    init(): Promise<void>;
+    /**
+     * Queries the vector store using text
+     * @param query - Text query to search for
+     * @param options - Query options including filters and top-k results
+     * @returns Array of matching documents
+     */
+    query(query: string, options?: IVectorStoreQueryOptions<IChromaMetadataFilter>): Promise<IVectorStoreDocument[]>;
+    /**
+     * Queries the vector store and filters results by similarity threshold
+     * @param query - Text query to search for
+     * @param options - Query options including similarity threshold
+     * @returns Array of documents meeting the similarity threshold
+     */
+    queryWithSimilarity(query: string, options?: IVectorStoreQueryOptions<IChromaMetadataFilter> & {
+        similarityThreshold: number;
+    }): Promise<IVectorStoreDocument[]>;
+    /**
+     * Upserts (inserts or updates) documents into the vector store
+     * @param documents - Array of documents to upsert
+     * @throws Error if any document is missing an ID
+     */
+    upsert(documents: IVectorStoreDocument[]): Promise<void>;
+    addDocumentsWithEmbeddings(documents: IVectorStoreDocumentWithEmbeddings[]): Promise<void>;
+    /**
+     * Deletes documents from the vector store by their IDs
+     * @param ids - Array of document IDs to delete
+     */
+    delete(ids: string[]): Promise<void>;
+    /**
+     * Retrieves a document by its ID
+     * @param id - Document ID to retrieve
+     * @returns The document if found, null otherwise
+     */
+    getById(id: string): Promise<IVectorStoreDocument | null>;
+    /**
+     * Queries the vector store using a pre-computed embedding vector
+     * @param vector - The embedding vector to search with
+     * @param options - Query options including top-k results
+     * @returns Array of matching documents
+     */
+    queryByVector(vector: number[], options?: IVectorStoreQueryOptions): Promise<IVectorStoreDocument[]>;
+    /**
+     * Updates the metadata for a document by its ID
+     * @param ids - Array of document IDs to update
+     * @param metadata - Metadata to update
+     */
+    updateMetadata(ids: string[], metadatas: Metadata[]): Promise<void>;
+}
+export { ChromaDataStore };
+//# sourceMappingURL=chroma-data-store.d.ts.map

package/dist/chroma/chroma-data-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chroma-data-store.d.ts","sourceRoot":"","sources":["../../src/chroma/chroma-data-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAIL,KAAK,QAAQ,EAEd,MAAM,UAAU,CAAC;AAClB,OAAO,EACL,iBAAiB,EACjB,KAAK,oBAAoB,EACzB,KAAK,kCAAkC,EACvC,KAAK,mBAAmB,EACxB,KAAK,wBAAwB,EAC9B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,KAAK,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAI5D;;GAEG;AACH,cAAM,eAAgB,SAAQ,iBAAiB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,UAAU,CAAa;IAE/B;;;;OAIG;gBACS,OAAO,EAAE,mBAAmB;IA0BxC;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAM1B;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B;;;;;OAKG;IACG,KAAK,CACT,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,wBAAwB,CAAC,qBAAqB,CAAC,GACxD,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAalC;;;;;OAKG;IACG,mBAAmB,CACvB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,wBAAwB,CAAC,qBAAqB,CAAC,GAAG;QAC1D,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GACA,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAUlC;;;;OAIG;IACG,MAAM,CAAC,SAAS,EAAE,oBAAoB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA0BxD,0BAA0B,CAC9B,SAAS,EAAE,kCAAkC,EAAE,GAC9C,OAAO,CAAC,IAAI,CAAC;IAmChB;;;OAGG;IACG,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO1C;;;;OAIG;IACG,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;IAc/D;;;;;OAKG;IACG,aAAa,CACjB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,oBAAoB,EAAE,CAAC;IASlC;;;;OAIG;IACG,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;CAO1E;AAED,OAAO,EAAE,eAAe,EAAE,CAAC"}