@meaningfully/core 0.1.0

package/tests/embeddings.test.ts

@@ -0,0 +1,144 @@
+//@ts-nocheck
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Document, TextNode } from 'llamaindex';
+
+// First, set up the mock before importing the module
+vi.mock(import("../src/services/embeddings.js"), async (importOriginal) => {
+  const actual = await importOriginal()
+  return {
+    ...actual,
+    // your mocked methods
+    estimateCost: vi.fn(),
+    getExistingVectorStoreIndex: vi.fn(),
+    persistNodes: vi.fn(),
+    persistDocuments: vi.fn(),
+    getExistingDocStore: vi.fn(),
+    searchDocuments: vi.fn()
+  }
+})
+
+// Now import the mocked functions
+import { transformDocumentsToNodes, getEmbedModel } from '../src/services/embeddings.js';
+
+describe('transformDocumentsToNodes', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  const mockConfig = {
+    chunkSize: 100,
+    chunkOverlap: 10,
+    combineSentencesIntoChunks: true,
+    sploderMaxSize: 500,
+    modelProvider: 'mock',
+    modelName: 'text-embedding-3-small',
+    vectorStoreType: "simple" as "simple",
+    storagePath: './storage',
+    projectName: 'test_project',
+    splitIntoSentences: true,
+  };
+
+  const mockSettings = {
+    openAIKey: 'mock-api-key',
+    oLlamaBaseURL: 'http://localhost',
+    azureOpenAIKey: null,
+    azureOpenAIEndpoint: null,
+    azureOpenAIApiVersion: null,
+    mistralApiKey: null,
+    geminiApiKey: null,
+  };
+
+  it('should process documents and return nodes', async () => {
+    const mockDocuments = [
+      new Document({ text: 'Document 1', metadata: { key1: 'value1' } }),
+      new Document({ text: 'Document 2', metadata: { key2: 'value2' } }),
+    ];
+    const mockNodes = [
+      new TextNode({ text: 'Document 1' }),
+      new TextNode({ text: 'Document 2' }),
+    ];
+
+    const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
+
+    expect(result.map((node) => node.text)).toEqual(mockNodes.map((node) => node.text));
+  });
+
+  it('should filter out documents with null, undefined, or zero-length text', async () => {
+    const mockDocuments = [
+      new Document({ text: 'Valid Document', metadata: { key1: 'value1' } }),
+      new Document({ text: undefined, metadata: { key3: 'value3' } }),
+      new Document({ text: '', metadata: { key4: 'value4' } }),
+    ];
+    const filteredDocuments = [mockDocuments[0]];
+    const mockNodes = [new TextNode({ text: 'Valid Document' })];
+
+    // (transformDocumentsToNodes as vi.Mock).mockResolvedValue(mockNodes);
+
+    const result = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings);
+    expect(result.map((n) => n.text)).toEqual(mockNodes.map((n) => n.text));
+    
+    // TODO: I can't get these to work. Apparently you can't spyOn a function that is imported from the same file.
+    // all well and good but ... why did CoPilot generate a test that can't work?
+    // expect(transformDocumentsToNodes).toHaveBeenCalledWith(filteredDocuments, expect.any(Array));
+  });
+
+  it('should exclude all metadata keys from embedding', async () => {
+    const mockDocuments = [
+      new Document({ text: 'Document 1', metadata: { key1: 'value1', key2: 'value2' } }),
+    ];
+
+    const nodes = await transformDocumentsToNodes(mockDocuments, mockConfig, mockSettings)
+    expect(nodes[0].excludedEmbedMetadataKeys).toEqual(['key1', 'key2']);
+  });
+});
+
+describe('getEmbedModel', () => {
+  const mockConfig = {
+    chunkSize: 100,
+    chunkOverlap: 10,
+    combineSentencesIntoChunks: true,
+    sploderMaxSize: 500,
+    modelProvider: 'openai',
+    modelName: 'text-embedding-3-small',
+    vectorStoreType: "simple" as "simple",
+    storagePath: './storage',
+    projectName: 'test_project',
+    splitIntoSentences: true,
+  };
+
+  const mockSettings = {
+    openAIKey: 'mock-api-key',
+    oLlamaBaseURL: 'http://localhost',
+    azureOpenAIKey: null,
+    azureOpenAIEndpoint: null,
+    azureOpenAIApiVersion: null,
+    mistralApiKey: null,
+    geminiApiKey: null,
+  };
+
+
+  it('should handle different model providers correctly', () => {
+    // Test with 'ollama' provider
+    const ollamaModel = getEmbedModel(
+      { ...mockConfig, modelProvider: 'ollama' }, 
+      mockSettings
+    );
+    expect(ollamaModel).toBeDefined();
+    
+    // Test with 'mock' provider
+    const mockModel = getEmbedModel(
+      { ...mockConfig, modelProvider: 'mock' }, 
+      mockSettings
+    );
+    expect(mockModel).toBeDefined();
+    
+    // Test with invalid provider
+    expect(() => {
+      getEmbedModel(
+        { ...mockConfig, modelProvider: 'invalid' as any }, 
+        mockSettings
+      );
+    }).toThrow('Unsupported embedding model provider: invalid');
+  });
+});

package/tests/sentenceSplitter.test.d.ts

@@ -0,0 +1 @@
+export {};

package/tests/sentenceSplitter.test.ts

@@ -0,0 +1,81 @@
+//@ts-nocheck
+import { expect, test } from 'vitest'
+import { CustomSentenceSplitter } from '../src/services/sentenceSplitter.js'
+import { SentenceSplitter, IngestionPipeline, Document } from "llamaindex";
+
+// do these tests just to make sure that we can factor out my hacky fixes when llamaindex is fixed.
+// test that original sentenceSplitter splits on abbreviations
+// test that original sentenceSplitter splits on abbreviations even when specified
+
+// test that my modified sentenceSplitter excludes metadata when arg is specified
+// test that my modified sentenceSplitter includes metadata when arg is specified the other way
+
+
+
+let documents = [
+    new Document({ text: "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.  Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.  He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time." }),
+];
+
+let originalSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [  
+        new SentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
+        ],
+    });
+let customSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [
+      new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
+    ],
+  });
+
+test("my modified sentenceSplitter doesn't eliminate spaces", () => {
+    customSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
+        expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(false);
+        expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(false);
+        expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(false);
+    });
+});
+
+// test("original sentenceSplitter does eliminate spaces", () => {
+//     originalSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
+//         expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(true);
+//         expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(true);
+//         expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(true);
+//     });
+// });
+
+let noAbbrevsCustomSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [
+      new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10, abbreviations: []}),
+    ],
+  });
+
+
+  test("my modified sentenceSplitter doesn't split on specified abbreviations", () => {
+    customSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
+        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).not.toContainEqual(true);
+    });
+});
+
+// this is only a problem on branch fix/sentence-splitter-spaces
+// where the chunker is eliminated entirely in favor of just splitting by sentences with natural.
+test("original sentenceSplitter splits in silly places, like Mr", () => {
+    noAbbrevsCustomSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
+        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).toContainEqual(true);
+    });
+});
+
+const testcases = [
+    ["USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41", "USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41"],
+    ["JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.", "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee."],
+    ["Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.", "Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019."],
+    ["He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time.", "He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time."],
+
+];
+testcases.forEach(([testcase_input, testcase_expected_output]) => {
+    test(`my sentenceSplitter correctly handles short sentence ${testcase_input}`, () => {
+        customSentenceSplitterPipeline.run({documents: [new Document({text: testcase_input})]}).then((nodes) => {
+            expect(nodes.length).toEqual(1);
+            expect(nodes[0]["text"]).toEqual(testcase_expected_output);
+        });
+    })
+});

package/tsconfig.json

@@ -0,0 +1,31 @@
+{
+  "compilerOptions": {
+    /* Base Options: */
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "target": "es2022",
+    "allowJs": true,
+    "resolveJsonModule": true,
+    "moduleDetection": "force",
+    "isolatedModules": true,
+    "verbatimModuleSyntax": false,
+
+    /* Strictness */
+    "strict": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": false,
+
+    /* If transpiling with TypeScript: */
+    "module": "NodeNext",
+    "outDir": "dist",
+    "rootDir": "src",
+    "sourceMap": true,
+
+    /* AND if you're building for a library: */
+    "declaration": true,
+
+    /* AND if you're building for a library in a monorepo: */
+    "declarationMap": true
+  },
+  "exclude": ["node_modules/natural", "dist/**"]
+}