@ixo/data-store 1.0.1

package/dist/types/vector-db-data-store.d.ts

@@ -0,0 +1,58 @@
+import { type IEmbeddingFunction, type Metadata } from 'chromadb';
+/**
+ * Represents a document in the vector store.
+ */
+export interface IVectorStoreDocument {
+    /**
+     * Unique identifier for the document.
+     */
+    id: string;
+    /**
+     * The content or data of the document.
+     */
+    content: string;
+    /**
+     * Optional metadata associated with the document.
+     */
+    metadata?: Metadata;
+    /**
+     * The score of the document.
+     */
+    score?: number;
+}
+export interface IVectorStoreDocumentWithEmbeddings extends IVectorStoreDocument {
+    embedding: number[];
+}
+/**
+ * Options for querying the vector store.
+ */
+export interface IVectorStoreQueryOptions<Filters extends Record<string, unknown> = Record<string, unknown>> {
+    /**
+     * The number of top results to return.
+     */
+    topK?: number;
+    /**
+     * Filters to apply based on document metadata.
+     */
+    filters?: Filters;
+}
+export interface IVectorStoreOptions {
+    collectionName: string;
+    url: string;
+    embeddingFunction?: IEmbeddingFunction;
+}
+export declare abstract class VectorDBDataStore {
+    protected readonly options: IVectorStoreOptions;
+    constructor(options: IVectorStoreOptions);
+    abstract upsert(documents: IVectorStoreDocument[]): Promise<void>;
+    abstract delete(ids: string[]): Promise<void>;
+    abstract queryByVector(vector: number[], options?: IVectorStoreQueryOptions): Promise<IVectorStoreDocument[]>;
+    abstract query(query: string, options?: IVectorStoreQueryOptions): Promise<IVectorStoreDocument[]>;
+    abstract getById(id: string): Promise<IVectorStoreDocument | null>;
+    abstract queryWithSimilarity(query: string, options?: IVectorStoreQueryOptions & {
+        similarityThreshold: number;
+    }): Promise<IVectorStoreDocument[]>;
+    abstract addDocumentsWithEmbeddings(documents: IVectorStoreDocumentWithEmbeddings[]): Promise<void>;
+    abstract init(): Promise<void>;
+}
+//# sourceMappingURL=vector-db-data-store.d.ts.map

package/dist/types/vector-db-data-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-db-data-store.d.ts","sourceRoot":"","sources":["../../src/types/vector-db-data-store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAE,KAAK,QAAQ,EAAE,MAAM,UAAU,CAAC;AAElE;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC;;OAEG;IACH,EAAE,EAAE,MAAM,CAAC;IAEX;;OAEG;IACH,OAAO,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,QAAQ,CAAC,EAAE,QAAQ,CAAC;IAEpB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,kCACf,SAAQ,oBAAoB;IAC5B,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,wBAAwB,CACvC,OAAO,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAEjE;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,cAAc,EAAE,MAAM,CAAC;IACvB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,kBAAkB,CAAC;CACxC;AAED,8BAAsB,iBAAiB;IACzB,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,mBAAmB;gBAA5B,OAAO,EAAE,mBAAmB;IAE3D,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,oBAAoB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IACjE,QAAQ,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAC7C,QAAQ,CAAC,aAAa,CACpB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAClC,QAAQ,CAAC,KAAK,CACZ,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;IAClE,QAAQ,CAAC,mBAAmB,CAC1B,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,wBAAwB,GAAG;QAAE,mBAAmB,EAAE,MAAM,CAAA;KAAE,GACnE,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAClC,QAAQ,CAAC,0BAA0B,CACjC,SAAS,EAAE,kCAAkC,EAAE,GAC9C,OAAO,CAAC,IAAI,CAAC;IAChB,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAC/B"}

package/dist/types/vector-db-data-store.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VectorDBDataStore = void 0;
+class VectorDBDataStore {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+}
+exports.VectorDBDataStore = VectorDBDataStore;

package/dist/utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from './with-report-error';
+//# sourceMappingURL=index.d.ts.map

package/dist/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC"}

package/dist/utils/index.js

@@ -0,0 +1,17 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./with-report-error"), exports);

package/dist/utils/with-report-error.d.ts

@@ -0,0 +1,5 @@
+/**
+ * should report using the logger and throw the error if the promise is rejected
+ */
+export declare const withReportError: <T>(promise: Promise<T>) => Promise<T>;
+//# sourceMappingURL=with-report-error.d.ts.map

package/dist/utils/with-report-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"with-report-error.d.ts","sourceRoot":"","sources":["../../src/utils/with-report-error.ts"],"names":[],"mappings":"AAEA;;GAEG;AACH,eAAO,MAAM,eAAe,GAAU,CAAC,EAAE,SAAS,OAAO,CAAC,CAAC,CAAC,KAAG,OAAO,CAAC,CAAC,CAYvE,CAAC"}

package/dist/utils/with-report-error.js

@@ -0,0 +1,21 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.withReportError = void 0;
+const logger_1 = require("@ixo/logger");
+/**
+ * should report using the logger and throw the error if the promise is rejected
+ */
+const withReportError = async (promise) => {
+    try {
+        const res = await promise;
+        return res;
+    }
+    catch (error) {
+        const errorMessage = error instanceof Error
+            ? error.message
+            : 'Something went wrong in AirtableDataStore';
+        logger_1.Logger.error(errorMessage, error);
+        throw error;
+    }
+};
+exports.withReportError = withReportError;

package/jest.config.js

@@ -0,0 +1,6 @@
+const baseConfig = require('@ixo/jest-config/nest');
+
+module.exports = {
+  ...baseConfig,
+  displayName: 'data-store',
+}; 

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@ixo/data-store",
+  "version": "1.0.1",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "module": "./dist/index.js",
+  "exports": {
+    ".": {
+      "require": "./dist/index.js",
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./*": {
+      "require": "./dist/*",
+      "import": "./dist/*",
+      "types": "./dist/*"
+    }
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.14",
+    "@types/node": "^22.7.3",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.3.3",
+    "@ixo/eslint-config": "1.0.0",
+    "@ixo/jest-config": "0.0.0",
+    "@ixo/typescript-config": "1.0.0"
+  },
+  "dependencies": {
+    "@langchain/community": "^1.0.3",
+    "@langchain/core": "^1.0.5",
+    "@langchain/openai": "^1.0.0",
+    "airtable": "^0.12.2",
+    "chromadb": "^1.10.4",
+    "chromadb-default-embed": "^2.13.2",
+    "neo4j-driver": "^5.28.1",
+    "openai": "^6.9.1",
+    "zod": "^4.1.12",
+    "@ixo/logger": "0.0.2"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "jest"
+  }
+}

package/src/airtable-store/index.ts

@@ -0,0 +1,141 @@
+import AirtableBase, { type FieldSet } from 'airtable';
+import { type QueryParams } from 'airtable/lib/query_params';
+import { type IDataStore } from '../types';
+import { withReportError } from '../utils';
+
+/**
+ * AirtableDataStore class to interact with Airtable API implementing IDataStore interface
+ *
+ * Basic CRUD operations
+ */
+export class AirtableDataStore<TFields extends FieldSet>
+  implements IDataStore<TFields>
+{
+  private readonly base: AirtableBase.Base;
+
+  constructor() {
+    if (!process.env.AIRTABLE_API_KEY) {
+      throw new Error('AIRTABLE_API_KEY is required');
+    }
+
+    if (!process.env.AIRTABLE_BASE_ID) {
+      throw new Error('AIRTABLE_BASE_ID is required');
+    }
+
+    this.base = new AirtableBase({ apiKey: process.env.AIRTABLE_API_KEY }).base(
+      process.env.AIRTABLE_BASE_ID,
+    );
+  }
+
+  private static withId<TFields extends FieldSet>(
+    record: AirtableBase.Record<TFields>,
+  ): TFields & { id: string } {
+    return {
+      ...record.fields,
+      id: record.id,
+    };
+  }
+
+  async getAllRecords(
+    tableName: string,
+    selectOptions?: QueryParams<TFields>,
+  ): Promise<(TFields & { id: string })[]> {
+    const records = await withReportError(
+      this.base<TFields>(tableName).select(selectOptions).all(),
+    );
+
+    return records.map((record) => AirtableDataStore.withId(record));
+  }
+
+  async getRecord(
+    tableName: string,
+    recordId: string,
+  ): Promise<TFields & { id: string }> {
+    const record = await withReportError(
+      this.base<TFields>(tableName).find(recordId),
+    );
+
+    return AirtableDataStore.withId(record);
+  }
+
+  async createRecord(
+    tableName: string,
+    recordData: TFields,
+  ): Promise<TFields & { id: string }> {
+    const record = await withReportError(
+      this.base<TFields>(tableName).create(recordData),
+    );
+    return AirtableDataStore.withId(record);
+  }
+
+  async updateRecord(
+    tableName: string,
+    recordId: string,
+    recordData: Partial<TFields>,
+  ): Promise<TFields & { id: string }> {
+    const record = await withReportError(
+      this.base<TFields>(tableName).update([
+        {
+          id: recordId,
+          fields: recordData,
+        },
+      ]),
+    );
+    if (!record[0]) {
+      // this error will never be thrown as the airtable sdk will throw an error but to satisfy typescript
+      throw new Error('Record not found');
+    }
+    return AirtableDataStore.withId(record[0]);
+  }
+
+  async batchUpdateRecords(
+    tableName: string,
+    records: {
+      id: string;
+      fields: Partial<
+        TFields & {
+          batch?: boolean;
+        }
+      >;
+    }[],
+  ): Promise<(TFields & { id: string })[]> {
+    const updatedRecords = await withReportError(
+      this.base<TFields>(tableName).update(records),
+    );
+    return updatedRecords.map((record) => AirtableDataStore.withId(record));
+  }
+
+  async deleteRecord(
+    tableName: string,
+    recordId: string,
+  ): Promise<TFields & { id: string }> {
+    const record = await withReportError(
+      this.base<TFields>(tableName).destroy(recordId),
+    );
+    return AirtableDataStore.withId(record);
+  }
+
+  async getRecordByField(
+    tableName: string,
+    fieldName: string,
+    fieldValue: string,
+  ): Promise<(TFields & { id: string })[]> {
+    const records = await withReportError(
+      this.base<TFields>(tableName)
+        .select({
+          filterByFormula: `{${fieldName}} = "${fieldValue}"`,
+        })
+        .all(),
+    );
+    return records.map((record) => AirtableDataStore.withId(record));
+  }
+
+  public getLinkToRecord(recordId: string): string {
+    if (!process.env.AITABLE_BASE_TABLE_LINK) {
+      throw new Error('AITABLE_BASE_TABLE_LINK is required');
+    }
+    return `${process.env.AITABLE_BASE_TABLE_LINK}/${recordId}`;
+  }
+}
+
+export type { FieldSet };

package/src/chroma/chroma-data-store.test.ts

@@ -0,0 +1,210 @@
+/* eslint-disable jest/unbound-method  -- ignore this for now */
+import { ChromaClient, Collection, type IEmbeddingFunction } from 'chromadb';
+import { ChromaDataStore } from './chroma-data-store';
+
+jest.mock('chromadb');
+const embeddingFunction: IEmbeddingFunction = {
+  generate: jest.fn(),
+};
+jest
+  .mocked(ChromaClient.prototype.getOrCreateCollection)
+  .mockResolvedValue(
+    new Collection(
+      'test',
+      'test',
+      jest.fn() as unknown as ChromaClient,
+      embeddingFunction,
+    ),
+  );
+
+describe('ChromaDataStore', () => {
+  beforeAll(() => {
+    process.env.OPENAI_API_KEY = 'test';
+  });
+
+  afterAll(() => {
+    process.env.OPENAI_API_KEY = undefined;
+  });
+
+  it('should be defined', () => {
+    expect(ChromaDataStore).toBeDefined();
+  });
+
+  describe('Initialization', () => {
+    it('should construct and initialize without errors', async () => {
+      // Should not throw during construction
+      const chromaDataStore = new ChromaDataStore({
+        collectionName: 'test',
+        url: 'http://localhost:8000',
+      });
+      expect(chromaDataStore).toBeInstanceOf(ChromaDataStore);
+
+      // Should not throw during initialization
+      await expect(chromaDataStore.init()).resolves.not.toThrow();
+    });
+
+    it('should throw error when OPENAI_API_KEY is not set and no embedding function is provided', () => {
+      // Temporarily remove API key
+      const originalKey = process.env.OPENAI_API_KEY;
+      delete process.env.OPENAI_API_KEY;
+
+      // Should throw during construction
+      expect(
+        () =>
+          new ChromaDataStore({
+            collectionName: 'test',
+            url: 'http://localhost:8000',
+          }),
+      ).toThrow('OPENAI_API_KEY is not set');
+
+      // Restore API key
+      process.env.OPENAI_API_KEY = originalKey;
+    });
+
+    it('should create collection or use existing collection', async () => {
+      jest.spyOn(ChromaClient.prototype, 'getOrCreateCollection');
+      const chromaDataStore = new ChromaDataStore({
+        collectionName: 'test',
+        url: 'http://localhost:8000',
+        embeddingFunction,
+      });
+      await chromaDataStore.init();
+      expect(ChromaClient.prototype.getOrCreateCollection).toHaveBeenCalledWith(
+        {
+          name: 'test',
+          embeddingFunction,
+        },
+      );
+    });
+  });
+
+  describe('Query', () => {
+    let chromaDataStore: ChromaDataStore;
+    beforeAll(async () => {
+      chromaDataStore = new ChromaDataStore({
+        collectionName: 'test',
+        url: 'http://localhost:8000',
+        embeddingFunction,
+      });
+      await chromaDataStore.init();
+
+      await chromaDataStore.init();
+      jest.mocked(Collection.prototype.query).mockResolvedValue({
+        ids: [['1', '2']],
+        embeddings: null,
+        documents: [['Hello, world!', 'Hello, world! 2']],
+        metadatas: [[{ title: 'Hello, world!' }, { title: 'Hello, world! 2' }]],
+        distances: [[3.3435163194430493e-16, 0.30934086831133445]],
+        included: [],
+      });
+    });
+
+    it('should query documents', async () => {
+      const documents = await chromaDataStore.query('Hello, world!');
+      expect(documents).toHaveLength(2);
+      expect(documents[0]?.id).toBe('1');
+      expect(documents[0]?.content).toBe('Hello, world!');
+    });
+
+    it('should query documents by vector', async () => {
+      const vector = [1, 2, 3];
+      const documents = await chromaDataStore.queryByVector(vector);
+      expect(documents).toHaveLength(2);
+      expect(documents[0]?.id).toBe('1');
+      expect(documents[0]?.content).toBe('Hello, world!');
+    });
+
+    it('should query documents by similarity', async () => {
+      const documents = await chromaDataStore.queryWithSimilarity(
+        'Hello, world!',
+        { similarityThreshold: 0.9 },
+      );
+      expect(documents).toHaveLength(1);
+      expect(documents[0]?.id).toBe('1');
+      expect(documents[0]?.content).toBe('Hello, world!');
+    });
+
+    it('should get document by id', async () => {
+      jest.spyOn(Collection.prototype, 'get');
+      jest.mocked(Collection.prototype.get).mockResolvedValue({
+        ids: ['1'],
+        embeddings: null,
+        documents: ['Hello, world!'],
+        metadatas: [{ title: 'Hello, world!' }],
+        included: [],
+      });
+      const document = await chromaDataStore.getById('1');
+      expect(Collection.prototype.get).toHaveBeenCalledWith({
+        ids: ['1'],
+      });
+      expect(document?.id).toBe('1');
+      expect(document?.content).toBe('Hello, world!');
+    });
+
+    it('should handle empty results', async () => {
+      jest.mocked(Collection.prototype.query).mockResolvedValue({
+        ids: [],
+        embeddings: null,
+        documents: [],
+        metadatas: [],
+        distances: [],
+        included: [],
+      });
+      const documents = await chromaDataStore.query('Hello, world!');
+      expect(documents).toHaveLength(0);
+    });
+  });
+
+  describe('Upsert', () => {
+    let chromaDataStore: ChromaDataStore;
+    beforeAll(async () => {
+      chromaDataStore = new ChromaDataStore({
+        collectionName: 'test',
+        url: 'http://localhost:8000',
+        embeddingFunction,
+      });
+      await chromaDataStore.init();
+    });
+
+    it('should upsert documents', async () => {
+      jest.spyOn(Collection.prototype, 'upsert');
+      await chromaDataStore.upsert([
+        {
+          id: '1',
+          content: 'Hello, world!',
+          metadata: { title: 'Hello, world!' },
+        },
+        {
+          id: '2',
+          content: 'Hello, world! 2',
+          metadata: { title: 'Hello, world! 2' },
+        },
+      ]);
+      expect(Collection.prototype.upsert).toHaveBeenCalledWith({
+        ids: ['1', '2'],
+        documents: ['Hello, world!', 'Hello, world! 2'],
+        metadatas: [{ title: 'Hello, world!' }, { title: 'Hello, world! 2' }],
+      });
+    });
+  });
+
+  describe('Delete', () => {
+    let chromaDataStore: ChromaDataStore;
+    beforeAll(async () => {
+      chromaDataStore = new ChromaDataStore({
+        collectionName: 'test',
+        url: 'http://localhost:8000',
+        embeddingFunction,
+      });
+      await chromaDataStore.init();
+    });
+
+    it('should delete documents', async () => {
+      jest.spyOn(Collection.prototype, 'delete');
+      await chromaDataStore.delete(['1', '2']);
+      expect(Collection.prototype.delete).toHaveBeenCalledWith({
+        ids: ['1', '2'],
+      });
+    });
+  });
+});