@meaningfully/core 0.1.0 → 0.1.2

package/dist/services/sentenceSplitter.test.js

@@ -1,68 +0,0 @@
-//@ts-nocheck
-import { expect, test } from 'vitest';
-import { CustomSentenceSplitter } from './sentenceSplitter';
-import { SentenceSplitter, IngestionPipeline, Document } from "llamaindex";
-// do these tests just to make sure that we can factor out my hacky fixes when llamaindex is fixed.
-// test that original sentenceSplitter splits on abbreviations
-// test that original sentenceSplitter splits on abbreviations even when specified
-// test that my modified sentenceSplitter excludes metadata when arg is specified
-// test that my modified sentenceSplitter includes metadata when arg is specified the other way
-let documents = [
-    new Document({ text: "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.  Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.  He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time." }),
-];
-let originalSentenceSplitterPipeline = new IngestionPipeline({
-    transformations: [
-        new SentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
-    ],
-});
-let customSentenceSplitterPipeline = new IngestionPipeline({
-    transformations: [
-        new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
-    ],
-});
-test("my modified sentenceSplitter doesn't eliminate spaces", () => {
-    customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
-        expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(false);
-        expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(false);
-        expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(false);
-    });
-});
-// test("original sentenceSplitter does eliminate spaces", () => {
-//     originalSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
-//         expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(true);
-//         expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(true);
-//         expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(true);
-//     });
-// });
-let noAbbrevsCustomSentenceSplitterPipeline = new IngestionPipeline({
-    transformations: [
-        new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10, abbreviations: [] }),
-    ],
-});
-test("my modified sentenceSplitter doesn't split on specified abbreviations", () => {
-    customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
-        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).not.toContainEqual(true);
-    });
-});
-// this is only a problem on branch fix/sentence-splitter-spaces
-// where the chunker is eliminated entirely in favor of just splitting by sentences with natural.
-test("original sentenceSplitter splits in silly places, like Mr", () => {
-    noAbbrevsCustomSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
-        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).toContainEqual(true);
-    });
-});
-const testcases = [
-    ["USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41", "USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41"],
-    ["JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.", "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee."],
-    ["Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.", "Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019."],
-    ["He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time.", "He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time."],
-];
-testcases.forEach(([testcase_input, testcase_expected_output]) => {
-    test(`my sentenceSplitter correctly handles short sentence ${testcase_input}`, () => {
-        customSentenceSplitterPipeline.run({ documents: [new Document({ text: testcase_input })] }).then((nodes) => {
-            expect(nodes.length).toEqual(1);
-            expect(nodes[0]["text"]).toEqual(testcase_expected_output);
-        });
-    });
-});
-//# sourceMappingURL=sentenceSplitter.test.js.map

package/dist/services/sentenceSplitter.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"sentenceSplitter.test.js","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3E,mGAAmG;AACnG,8DAA8D;AAC9D,kFAAkF;AAElF,iFAAiF;AACjF,+FAA+F;AAI/F,IAAI,SAAS,GAAG;IACZ,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,oeAAoe,EAAE,CAAC;CAC/f,CAAC;AAEF,IAAI,gCAAgC,GAAG,IAAI,iBAAiB,CAAC;IACzD,eAAe,EAAE;QACb,IAAI,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KACxD;CACJ,CAAC,CAAC;AACP,IAAI,8BAA8B,GAAG,IAAI,iBAAiB,CAAC;IACvD,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KAChE;CACF,CAAC,CAAC;AAEL,IAAI,CAAC,uDAAuD,EAAE,GAAG,EAAE;IAC/D,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,kEAAkE;AAClE,qFAAqF;AACrF,+FAA+F;AAC/F,kGAAkG;AAClG,0FAA0F;AAC1F,UAAU;AACV,MAAM;AAEN,IAAI,uCAAuC,GAAG,IAAI,iBAAiB,CAAC;IAChE,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAC,CAAC;KAClF;CACF,CAAC,CAAC;AAGH,IAAI,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACjF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,gEAAgE;AAChE,iGAAiG;AACjG,IAAI,CAAC,2DAA2D,EAAE,GAAG,EAAE;IACnE,uCAAuC,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/E,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG;IACd,CAAC,gGAAgG,EAAE,gGAAgG,CAAC;IACpM,CAAC,sKAAsK,EAAE,sKAAsK,CAAC;IAChV,CAAC,oGAAoG,EAAE,oGAAoG,CAAC;IAC5M,CAAC,0NAA0N,EAAE,0NAA0N,CAAC;CAE3b,CAAC;AACF,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE,wBAAwB,CAAC,EAAE,EAAE;IAC7D,IAAI,CAAC,wDAAwD,cAAc,EAAE,EAAE,GAAG,EAAE;QAChF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,CAAC,IAAI,QAAQ,CAAC,EAAC,IAAI,EAAE,cAAc,EAAC,CAAC,CAAC,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAA;AACN,CAAC,CAAC,CAAC"}

package/src/api/embedding.d.ts

@@ -1,6 +0,0 @@
-import type { EmbeddingConfig, EmbeddingResult, SearchResult, PreviewResult, Settings, MetadataFilter, Clients } from "../types";
-export declare function createEmbeddings(csvPath: string, textColumnName: string, config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<EmbeddingResult>;
-export declare function previewResults(csvPath: string, textColumnName: string, config: EmbeddingConfig): Promise<PreviewResult>;
-export declare function getDocStore(config: EmbeddingConfig): Promise<import("llamaindex").BaseDocumentStore>;
-export declare function getIndex(config: EmbeddingConfig, settings: Settings, clients: Clients): Promise<import("llamaindex").VectorStoreIndex>;
-export declare function search(index: any, query: string, numResults?: number, filters?: MetadataFilter[]): Promise<SearchResult[]>;

package/tests/csvLoader.test.d.ts

@@ -1 +0,0 @@
-export {};

package/tests/embedding.test.d.ts

@@ -1 +0,0 @@
-export {};

package/tests/embeddings.test.d.ts

@@ -1 +0,0 @@
-export {};

package/tests/sentenceSplitter.test.d.ts

@@ -1 +0,0 @@
-export {};