@meaningfully/core 0.1.0

package/dist/services/mockEmbedding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mockEmbedding.d.ts","sourceRoot":"","sources":["../../src/services/mockEmbedding.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,qBAAa,aAAc,SAAQ,aAAa;;IAItC,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAK1D"}

package/dist/services/mockEmbedding.js

@@ -0,0 +1,14 @@
+//@ts-nocheck
+import { BaseEmbedding } from "llamaindex";
+export class MockEmbedding extends BaseEmbedding {
+    constructor() {
+        super();
+    }
+    async getTextEmbedding(text) {
+        return new Promise((resolve) => {
+            resolve([1, 0, 0, 0, 0, 0]);
+        });
+    }
+}
+;
+//# sourceMappingURL=mockEmbedding.js.map

package/dist/services/mockEmbedding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mockEmbedding.js","sourceRoot":"","sources":["../../src/services/mockEmbedding.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,MAAM,OAAO,aAAc,SAAQ,aAAa;IAC5C;QACI,KAAK,EAAE,CAAC;IACZ,CAAC;IACD,KAAK,CAAC,gBAAgB,CAAC,IAAY;QAC/B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3B,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AAAA,CAAC"}

package/dist/services/progressManager.d.ts

@@ -0,0 +1,21 @@
+/**
+ * A simple manager to track progress of various operations
+ */
+export declare class ProgressManager {
+    private static instance;
+    private progressMap;
+    private currentOperation;
+    private constructor();
+    static getInstance(): ProgressManager;
+    startOperation(operationId: string, total?: number): void;
+    updateProgress(operationId: string, progress: number): void;
+    completeOperation(operationId: string): void;
+    getCurrentProgress(): {
+        progress: number;
+        total: number;
+        elapsedTimeMs: number;
+        estimatedTimeRemainingMs: number | null;
+    };
+    clearOperation(operationId: string): void;
+}
+//# sourceMappingURL=progressManager.d.ts.map

package/dist/services/progressManager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressManager.d.ts","sourceRoot":"","sources":["../../src/services/progressManager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH,qBAAa,eAAe;IAC1B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAkB;IACzC,OAAO,CAAC,WAAW,CAA0G;IAC7H,OAAO,CAAC,gBAAgB,CAAuB;IAG/C,OAAO;WAEO,WAAW,IAAI,eAAe;IAOrC,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,KAAK,GAAE,MAAY,GAAG,IAAI;IAM9D,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,IAAI;IAY3D,iBAAiB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;IAgB5C,kBAAkB,IAAI;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,wBAAwB,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE;IA0BzH,cAAc,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI;CAMjD"}

package/dist/services/progressManager.js

@@ -0,0 +1,76 @@
+/**
+ * A simple manager to track progress of various operations
+ */
+const FUDGE_FACTOR = 1.2; // seat of my pants guess
+export class ProgressManager {
+    static instance;
+    progressMap = new Map();
+    currentOperation = null;
+    constructor() { }
+    static getInstance() {
+        if (!ProgressManager.instance) {
+            ProgressManager.instance = new ProgressManager();
+        }
+        return ProgressManager.instance;
+    }
+    startOperation(operationId, total = 100) {
+        const now = Date.now();
+        this.progressMap.set(operationId, { progress: 0, total, startTime: now, lastUpdateTime: now });
+        this.currentOperation = operationId;
+    }
+    updateProgress(operationId, progress) {
+        const currentProgress = this.progressMap.get(operationId);
+        if (currentProgress) {
+            this.progressMap.set(operationId, {
+                progress,
+                total: currentProgress.total,
+                startTime: currentProgress.startTime,
+                lastUpdateTime: Date.now()
+            });
+        }
+    }
+    completeOperation(operationId) {
+        const currentProgress = this.progressMap.get(operationId);
+        if (currentProgress) {
+            this.progressMap.set(operationId, {
+                progress: currentProgress.total,
+                total: currentProgress.total,
+                startTime: currentProgress.startTime,
+                lastUpdateTime: Date.now()
+            });
+        }
+        if (this.currentOperation === operationId) {
+            this.currentOperation = null;
+        }
+    }
+    getCurrentProgress() {
+        if (this.currentOperation) {
+            const operationData = this.progressMap.get(this.currentOperation);
+            if (operationData) {
+                const now = Date.now();
+                const elapsedTimeMs = now - operationData.startTime;
+                let estimatedTimeRemainingMs = null;
+                // Only estimate if we have meaningful progress (at least 5% complete and some time elapsed)
+                if (operationData.progress > 0.05 * operationData.total && elapsedTimeMs > 1000) {
+                    const progressPercentage = operationData.progress / operationData.total;
+                    const estimatedTotalTime = (elapsedTimeMs / progressPercentage) * FUDGE_FACTOR;
+                    estimatedTimeRemainingMs = Math.max(0, estimatedTotalTime - elapsedTimeMs);
+                }
+                return {
+                    progress: operationData.progress,
+                    total: operationData.total,
+                    elapsedTimeMs,
+                    estimatedTimeRemainingMs
+                };
+            }
+        }
+        return { progress: 0, total: 100, elapsedTimeMs: 0, estimatedTimeRemainingMs: null };
+    }
+    clearOperation(operationId) {
+        this.progressMap.delete(operationId);
+        if (this.currentOperation === operationId) {
+            this.currentOperation = null;
+        }
+    }
+}
+//# sourceMappingURL=progressManager.js.map

package/dist/services/progressManager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressManager.js","sourceRoot":"","sources":["../../src/services/progressManager.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,YAAY,GAAG,GAAG,CAAC,CAAC,yBAAyB;AAEnD,MAAM,OAAO,eAAe;IAClB,MAAM,CAAC,QAAQ,CAAkB;IACjC,WAAW,GAAgG,IAAI,GAAG,EAAE,CAAC;IACrH,gBAAgB,GAAkB,IAAI,CAAC;IAG/C,gBAAuB,CAAC;IAEjB,MAAM,CAAC,WAAW;QACvB,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;YAC9B,eAAe,CAAC,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;QACnD,CAAC;QACD,OAAO,eAAe,CAAC,QAAQ,CAAC;IAClC,CAAC;IAEM,cAAc,CAAC,WAAmB,EAAE,QAAgB,GAAG;QAC5D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,cAAc,EAAE,GAAG,EAAE,CAAC,CAAC;QAC/F,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;IACtC,CAAC;IAEM,cAAc,CAAC,WAAmB,EAAE,QAAgB;QACzD,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC1D,IAAI,eAAe,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE;gBAChC,QAAQ;gBACR,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEM,iBAAiB,CAAC,WAAmB;QAC1C,MAAM,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QAC1D,IAAI,eAAe,EAAE,CAAC;YACpB,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE;gBAChC,QAAQ,EAAE,eAAe,CAAC,KAAK;gBAC/B,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,cAAc,EAAE,IAAI,CAAC,GAAG,EAAE;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,KAAK,WAAW,EAAE,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;IACH,CAAC;IAEM,kBAAkB;QACvB,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;YAClE,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvB,MAAM,aAAa,GAAG,GAAG,GAAG,aAAa,CAAC,SAAS,CAAC;gBACpD,IAAI,wBAAwB,GAAkB,IAAI,CAAC;gBAEnD,4FAA4F;gBAC5F,IAAI,aAAa,CAAC,QAAQ,GAAG,IAAI,GAAG,aAAa,CAAC,KAAK,IAAI,aAAa,GAAG,IAAI,EAAE,CAAC;oBAChF,MAAM,kBAAkB,GAAG,aAAa,CAAC,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC;oBACxE,MAAM,kBAAkB,GAAG,CAAC,aAAa,GAAG,kBAAkB,CAAC,GAAG,YAAY,CAAC;oBAC/E,wBAAwB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,kBAAkB,GAAG,aAAa,CAAC,CAAC;gBAC7E,CAAC;gBAED,OAAO;oBACL,QAAQ,EAAE,aAAa,CAAC,QAAQ;oBAChC,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,aAAa;oBACb,wBAAwB;iBACzB,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,wBAAwB,EAAE,IAAI,EAAE,CAAC;IACvF,CAAC;IAEM,cAAc,CAAC,WAAmB;QACvC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,gBAAgB,KAAK,WAAW,EAAE,CAAC;YAC1C,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC/B,CAAC;IACH,CAAC;CACF"}

package/dist/services/progressVectorStoreIndex.d.ts

@@ -0,0 +1,21 @@
+import { VectorStoreIndex, type VectorIndexOptions as BaseVectorIndexOptions } from "llamaindex";
+import { BaseNode } from "llamaindex";
+export interface VectorIndexOptions extends BaseVectorIndexOptions {
+    progressCallback?: (progress: number, total: number) => void;
+}
+export declare class ProgressVectorStoreIndex extends VectorStoreIndex {
+    static init(options: VectorIndexOptions): Promise<VectorStoreIndex>;
+    buildIndexFromNodes(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<void>;
+    insertNodes(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<void>;
+    getNodeEmbeddingResults(nodes: BaseNode[], options?: {
+        logProgress?: boolean;
+        progressCallback?: (progress: number, total: number) => void;
+    }): Promise<BaseNode[]>;
+}
+//# sourceMappingURL=progressVectorStoreIndex.d.ts.map

package/dist/services/progressVectorStoreIndex.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressVectorStoreIndex.d.ts","sourceRoot":"","sources":["../../src/services/progressVectorStoreIndex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,KAAK,kBAAkB,IAAI,sBAAsB,EAAyC,MAAM,YAAY,CAAC;AACxI,OAAO,EAAE,QAAQ,EAAkC,MAAM,YAAY,CAAC;AAItE,MAAM,WAAW,kBAAmB,SAAQ,sBAAsB;IAChE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAC9D;AAID,qBAAa,wBAAyB,SAAQ,gBAAgB;WACxC,IAAI,CACtB,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC;IAuCtB,mBAAmB,CACvB,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE;IAK7F,WAAW,CACf,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE;IAe7F,uBAAuB,CAC3B,KAAK,EAAE,QAAQ,EAAE,EACjB,OAAO,CAAC,EAAE;QAAE,WAAW,CAAC,EAAE,OAAO,CAAC;QAAC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE,GAChG,OAAO,CAAC,QAAQ,EAAE,CAAC;CAcvB"}

package/dist/services/progressVectorStoreIndex.js

@@ -0,0 +1,60 @@
+import { VectorStoreIndex, storageContextFromDefaults, IndexDict } from "llamaindex";
+import { splitNodesByType } from "llamaindex";
+import { addNodesToVectorStores } from "llamaindex";
+// Subclass VectorStoreIndex to handle progressCallback
+// @ts-ignore
+export class ProgressVectorStoreIndex extends VectorStoreIndex {
+    static async init(options) {
+        const storageContext = options.storageContext ?? (await storageContextFromDefaults({}));
+        const indexStore = storageContext.indexStore;
+        const docStore = storageContext.docStore;
+        // @ts-ignore 
+        let indexStruct = await VectorStoreIndex.setupIndexStructFromStorage(indexStore, options);
+        if (!options.nodes && !indexStruct) {
+            throw new Error("Cannot initialize VectorStoreIndex without nodes or indexStruct");
+        }
+        indexStruct = indexStruct ?? new IndexDict();
+        // @ts-ignore
+        const index = new this({
+            storageContext,
+            docStore,
+            indexStruct,
+            indexStore,
+            vectorStores: options.vectorStores,
+        });
+        if (options.nodes) {
+            // If nodes are passed in, then we need to update the index
+            await index.buildIndexFromNodes(options.nodes, {
+                logProgress: options.logProgress,
+                progressCallback: options.progressCallback,
+            });
+        }
+        return index;
+    }
+    async buildIndexFromNodes(nodes, options) {
+        await this.insertNodes(nodes, options);
+    }
+    async insertNodes(nodes, options) {
+        if (!nodes || nodes.length === 0) {
+            return;
+        }
+        nodes = await this.getNodeEmbeddingResults(nodes, options);
+        await addNodesToVectorStores(nodes, this.vectorStores, this.insertNodesToStore.bind(this));
+        await this.indexStore.addIndexStruct(this.indexStruct);
+    }
+    async getNodeEmbeddingResults(nodes, options) {
+        const nodeMap = splitNodesByType(nodes);
+        for (const type in nodeMap) {
+            const nodes = nodeMap[type];
+            const embedModel = this.vectorStores[type]?.embedModel ?? this.embedModel;
+            if (embedModel && nodes) {
+                await embedModel(nodes, {
+                    logProgress: options?.logProgress,
+                    progressCallback: options?.progressCallback, // Pass progressCallback to embedModel
+                });
+            }
+        }
+        return nodes;
+    }
+}
+//# sourceMappingURL=progressVectorStoreIndex.js.map

package/dist/services/progressVectorStoreIndex.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"progressVectorStoreIndex.js","sourceRoot":"","sources":["../../src/services/progressVectorStoreIndex.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAqD,0BAA0B,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACxI,OAAO,EAA0B,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AAOpD,uDAAuD;AACvD,aAAa;AACb,MAAM,OAAO,wBAAyB,SAAQ,gBAAgB;IACrD,MAAM,CAAC,KAAK,CAAC,IAAI,CACtB,OAA2B;QAE3B,MAAM,cAAc,GAClB,OAAO,CAAC,cAAc,IAAI,CAAC,MAAM,0BAA0B,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,MAAM,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC;QAC7C,MAAM,QAAQ,GAAG,cAAc,CAAC,QAAQ,CAAC;QAEzC,cAAc;QACd,IAAI,WAAW,GAAG,MAAM,gBAAgB,CAAC,2BAA2B,CAClE,UAAU,EACV,OAAO,CACR,CAAC;QAEF,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,iEAAiE,CAClE,CAAC;QACJ,CAAC;QAED,WAAW,GAAG,WAAW,IAAI,IAAI,SAAS,EAAE,CAAC;QAE7C,aAAa;QACb,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC;YACrB,cAAc;YACd,QAAQ;YACR,WAAW;YACX,UAAU;YACV,YAAY,EAAE,OAAO,CAAC,YAAY;SACnC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAClB,2DAA2D;YAC3D,MAAM,KAAK,CAAC,mBAAmB,CAAC,OAAO,CAAC,KAAK,EAAE;gBAC7C,WAAW,EAAE,OAAO,CAAC,WAAW;gBAChC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;aAC3C,CAAC,CAAC;QACL,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,mBAAmB,CACvB,KAAiB,EACjB,OAAiG;QAEjG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,WAAW,CACf,KAAiB,EACjB,OAAiG;QAEjG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO;QACT,CAAC;QAED,KAAK,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC3D,MAAM,sBAAsB,CAC1B,KAAK,EACL,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CACnC,CAAC;QACF,MAAM,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,uBAAuB,CAC3B,KAAiB,EACjB,OAAiG;QAEjG,MAAM,OAAO,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;QACxC,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAoB,CAAC,CAAC;YAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,IAAoB,CAAC,EAAE,UAAU,IAAI,IAAI,CAAC,UAAU,CAAC;YAC1F,IAAI,UAAU,IAAI,KAAK,EAAE,CAAC;gBACxB,MAAM,UAAU,CAAC,KAAK,EAAE;oBACtB,WAAW,EAAE,OAAO,EAAE,WAAW;oBACjC,gBAAgB,EAAE,OAAO,EAAE,gBAAgB,EAAE,sCAAsC;iBACpF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/services/sentenceSplitter.d.ts

@@ -0,0 +1,17 @@
+import { SentenceSplitter } from "llamaindex";
+import natural from "natural";
+type TextSplitterFn = (text: string) => string[];
+export declare class CustomSentenceSplitter extends SentenceSplitter {
+    #private;
+    chunkingTokenizerFn: () => TextSplitterFn;
+    abbreviations: string[];
+    tokenizer: natural.SentenceTokenizer;
+    constructor(params?: {
+        chunkSize?: number;
+        chunkOverlap?: number;
+        abbreviations?: string[];
+    });
+    _splitText(text: string, chunkSize: number): string[];
+}
+export {};
+//# sourceMappingURL=sentenceSplitter.d.ts.map

package/dist/services/sentenceSplitter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentenceSplitter.d.ts","sourceRoot":"","sources":["../../src/services/sentenceSplitter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAmD,MAAM,YAAY,CAAC;AAC/F,OAAO,OAAO,MAAM,SAAS,CAAA;AA2C7B,KAAK,cAAc,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;AAejD,qBAAa,sBAAuB,SAAQ,gBAAgB;;IAG1D,mBAAmB,QAAO,cAAc,CAQtC;IAGF,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC;gBAEzB,MAAM,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;KAAO;IAoChG,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE;CAkItD"}

package/dist/services/sentenceSplitter.js

@@ -0,0 +1,207 @@
+import { SentenceSplitter, splitBySep, splitByRegex, splitByChar, Settings } from "llamaindex";
+import natural from "natural";
+/*
+LlamaIndex's includes the length of the metadata as part of the size of the chunk when splitting by sentences.
+This produces very unintuitive behavior: e.g. when the user specifies a chunk-size of 50 and nodes have metadata of length 40,
+the resulting split sentences are about 10 tokens long -- as opposed to the specified 50.
+
+This modified SentenceSplitter adds a `include_metadata_in_chunksize` flag that disables the above behavior,
+ignoring metadata when calculating chunksize (i.e. only including the size of the text datga when calculating chunksize.)
+
+Additionally, splitTextMetadataAware does some bizarre stuff where it will split sentences at abbreviations -- even if the
+underlying tokenizer knows about the abbreviations, I think due to some weird sub-sentence splitting. It also sews sentence
+chunks back together in a way that eliminates spaces, e.g. `JPMorgan Chase & Co.elected Mark Weinberger` and  `Mr.Weinberger was Global Chairman`.
+
+I also tried making SentenceSplitter just split on sentences (with Natural) but this misbehaved by splitting TOO much. I do need short sentences grouped
+together (whether they are true short sentences, or false-positives like "USA v. one 12 ft. I.B.M. mainframe").
+
+
+*/
+// TODO: make this configurable
+const INCLUDE_METADATA_IN_CHUNKSIZE = false;
+SentenceSplitter.prototype.splitTextMetadataAware = function (text, metadata) {
+    const metadataLength = this.tokenSize(metadata);
+    const effectiveChunkSize = INCLUDE_METADATA_IN_CHUNKSIZE ? this.chunkSize - metadataLength : this.chunkSize;
+    if (effectiveChunkSize <= 0) {
+        throw new Error(`Metadata length (${metadataLength}) is longer than chunk size (${this.chunkSize}). Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`);
+    }
+    else if (effectiveChunkSize < 50) {
+        console.log(`Metadata length (${metadataLength}) is close to chunk size (${this.chunkSize}). Resulting chunks are less than 50 tokens. Consider increasing the chunk size or decreasing the size of your metadata to avoid this.`);
+    }
+    return this._splitText(text, effectiveChunkSize);
+};
+const default_abbreviations = ['dr.', 'vs.', 'mr.', 'ms.', 'mx.', 'mrs.', 'prof.', 'inc.', 'corp.', 'co.', 'llc.', 'ltd.', 'etc.', "i.e.",
+    "etc.",
+    "vs.",
+    "A.S.A.P.",
+];
+// This varies from SentenceSplitter in two ways:
+// 1. it uses abbreviations set here.
+// 2. it uses a custom SentenceTokenizer with a second trimSentences arguemnt that controls
+//    whether or not leading/trailing whitespace is preserved.
+//    We want to preserve it, so that when sentences are merged back again, we don't end up with 
+//    sentences that are not separated by spaces.
+// Because JavaScript is stupid, we have to copy over almost the whole SentenceSplitter just to make those few small changes.
+export class CustomSentenceSplitter extends SentenceSplitter {
+    // this function is new.
+    chunkingTokenizerFn = () => {
+        return (text) => {
+            try {
+                return this.tokenizer.tokenize(text);
+            }
+            catch {
+                return [text];
+            }
+        };
+    };
+    #splitFns = new Set();
+    #subSentenceSplitFns = new Set();
+    abbreviations;
+    tokenizer;
+    constructor(params = {}) {
+        super(params);
+        // Create custom tokenizer with abbreviations
+        this.abbreviations = params.abbreviations || default_abbreviations;
+        // I modified my local node_modules/natural/lib/natural/tokenizers/index.d.ts to add the second argument to the natural.SentenceTokenizer constructor.
+        // once that gets fixed in the next version of the library, remove the ts-ignore.
+        // @ts-ignore
+        this.tokenizer = new natural.SentenceTokenizer(this.abbreviations, false); // false is don't trim sentences
+        // copied from the superclass.
+        this.#splitFns.add(splitBySep(this.paragraphSeparator));
+        this.#splitFns.add(this.chunkingTokenizerFn()); // the ONLY change here in the constructor.
+        // copied from the superclass.
+        this.#subSentenceSplitFns.add(splitByRegex(this.secondaryChunkingRegex));
+        this.#subSentenceSplitFns.add(splitBySep(this.separator));
+        this.#subSentenceSplitFns.add(splitByChar());
+        // left over from a failed attempt to JUST use natural.SentenceTokenizer
+        // but I DO in fact need the merge stuff.
+        // const tokenizer = 
+        // Override the default splitText method
+        // this.splitText = (text: string): string[] => {
+        //   return tokenizer.tokenize(text);
+        // };
+        // /* tslint:disable:no-unused-variable */
+        // this.splitTextMetadataAware = (text: string, metadata: string): string[] => {
+        //   return tokenizer.tokenize(text);
+        // }
+    }
+    //just verbatim copies of the parent class
+    _splitText(text, chunkSize) {
+        if (text === "")
+            return [text];
+        const callbackManager = Settings.callbackManager;
+        callbackManager.dispatchEvent("chunking-start", {
+            text: [text],
+        });
+        const splits = this.#split(text, chunkSize);
+        const chunks = this.#merge(splits, chunkSize);
+        callbackManager.dispatchEvent("chunking-end", {
+            chunks,
+        });
+        return chunks;
+    }
+    #split(text, chunkSize) {
+        const tokenSize = this.tokenSize(text);
+        if (tokenSize <= chunkSize) {
+            return [
+                {
+                    text,
+                    isSentence: true,
+                    tokenSize,
+                },
+            ];
+        }
+        const [textSplitsByFns, isSentence] = this.#getSplitsByFns(text);
+        const textSplits = [];
+        for (const textSplit of textSplitsByFns) {
+            const tokenSize = this.tokenSize(textSplit);
+            if (tokenSize <= chunkSize) {
+                textSplits.push({
+                    text: textSplit,
+                    isSentence,
+                    tokenSize,
+                });
+            }
+            else {
+                const recursiveTextSplits = this.#split(textSplit, chunkSize);
+                textSplits.push(...recursiveTextSplits);
+            }
+        }
+        return textSplits;
+    }
+    #getSplitsByFns(text) {
+        for (const splitFn of this.#splitFns) {
+            const splits = splitFn(text);
+            if (splits.length > 1) {
+                return [splits, true];
+            }
+        }
+        for (const splitFn of this.#subSentenceSplitFns) {
+            const splits = splitFn(text);
+            if (splits.length > 1) {
+                return [splits, false];
+            }
+        }
+        return [[text], true];
+    }
+    #merge(splits, chunkSize) {
+        const chunks = [];
+        let currentChunk = [];
+        let lastChunk = [];
+        let currentChunkLength = 0;
+        let newChunk = true;
+        const closeChunk = () => {
+            chunks.push(currentChunk.map(([text]) => text).join(""));
+            lastChunk = currentChunk;
+            currentChunk = [];
+            currentChunkLength = 0;
+            newChunk = true;
+            let lastIndex = lastChunk.length - 1;
+            while (lastIndex >= 0 &&
+                currentChunkLength + lastChunk[lastIndex][1] <= this.chunkOverlap) {
+                const [text, length] = lastChunk[lastIndex];
+                currentChunkLength += length;
+                currentChunk.unshift([text, length]);
+                lastIndex -= 1;
+            }
+        };
+        while (splits.length > 0) {
+            const curSplit = splits[0];
+            if (curSplit.tokenSize > chunkSize) {
+                throw new Error("Single token exceeded chunk size");
+            }
+            if (currentChunkLength + curSplit.tokenSize > chunkSize && !newChunk) {
+                closeChunk();
+            }
+            else {
+                if (curSplit.isSentence ||
+                    currentChunkLength + curSplit.tokenSize <= chunkSize ||
+                    newChunk) {
+                    currentChunkLength += curSplit.tokenSize;
+                    currentChunk.push([curSplit.text, curSplit.tokenSize]);
+                    splits.shift();
+                    newChunk = false;
+                }
+                else {
+                    closeChunk();
+                }
+            }
+        }
+        // Handle the last chunk
+        if (!newChunk) {
+            chunks.push(currentChunk.map(([text]) => text).join(""));
+        }
+        return this.#postprocessChunks(chunks);
+    }
+    #postprocessChunks(chunks) {
+        const newChunks = [];
+        for (const chunk of chunks) {
+            const trimmedChunk = chunk.trim();
+            if (trimmedChunk !== "") {
+                newChunks.push(trimmedChunk);
+            }
+        }
+        return newChunks;
+    }
+}
+//# sourceMappingURL=sentenceSplitter.js.map

package/dist/services/sentenceSplitter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentenceSplitter.js","sourceRoot":"","sources":["../../src/services/sentenceSplitter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAC/F,OAAO,OAAO,MAAM,SAAS,CAAA;AAE7B;;;;;;;;;;;;;;;;EAgBE;AACF,+BAA+B;AAC/B,MAAM,6BAA6B,GAAG,KAAK,CAAC;AAC5C,gBAAgB,CAAC,SAAS,CAAC,sBAAsB,GAAG,UAAS,IAAY,EAAE,QAAgB;IACzF,MAAM,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,kBAAkB,GAAG,6BAA6B,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;IAC5G,IAAI,kBAAkB,IAAI,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CACb,oBAAoB,cAAc,gCAAgC,IAAI,CAAC,SAAS,8FAA8F,CAC/K,CAAC;IACJ,CAAC;SAAM,IAAI,kBAAkB,GAAG,EAAE,EAAE,CAAC;QACnC,OAAO,CAAC,GAAG,CACT,oBAAoB,cAAc,6BAA6B,IAAI,CAAC,SAAS,wIAAwI,CACtN,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;AACnD,CAAC,CAAA;AAED,MAAM,qBAAqB,GAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IACtI,MAAM;IACN,KAAK;IACL,UAAU;CACX,CAAC;AAWF,iDAAiD;AACjD,qCAAqC;AACrC,2FAA2F;AAC3F,8DAA8D;AAC9D,iGAAiG;AACjG,iDAAiD;AACjD,6HAA6H;AAC7H,MAAM,OAAO,sBAAuB,SAAQ,gBAAgB;IAE1D,wBAAwB;IACxB,mBAAmB,GAAG,GAAmB,EAAE;QACzC,OAAO,CAAC,IAAY,EAAE,EAAE;YACtB,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,IAAI,CAAC,CAAC;YAChB,CAAC;QACH,CAAC,CAAC;IACJ,CAAC,CAAC;IACF,SAAS,GAAwB,IAAI,GAAG,EAAE,CAAC;IAC3C,oBAAoB,GAAwB,IAAI,GAAG,EAAE,CAAC;IACtD,aAAa,CAAW;IACxB,SAAS,CAA4B;IAErC,YAAY,SAAkF,EAAE;QAC9F,KAAK,CAAC,MAAM,CAAC,CAAC;QACd,6CAA6C;QAC7C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,qBAAqB,CAAC;QAEnE,sJAAsJ;QACtJ,iFAAiF;QACjF,aAAa;QACb,IAAI,CAAC,SAAS,GAAG,IAAI,OAAO,CAAC,iBAAiB,CAAC,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,gCAAgC;QAE3G,8BAA8B;QAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAExD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,CAAC,2CAA2C;QAE3F,8BAA8B;QAC9B,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC;QACzE,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;QAE7C,wEAAwE;QACxE,yCAAyC;QACzC,qBAAqB;QACrB,wCAAwC;QACxC,iDAAiD;QACjD,qCAAqC;QACrC,KAAK;QACL,0CAA0C;QAC1C,gFAAgF;QAChF,qCAAqC;QACrC,IAAI;IACN,CAAC;IAGD,0CAA0C;IAE1C,UAAU,CAAC,IAAY,EAAE,SAAiB;QACxC,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAE/B,MAAM,eAAe,GAAG,QAAQ,CAAC,eAAe,CAAC;QAEjD,eAAe,CAAC,aAAa,CAAC,gBAAgB,EAAE;YAC9C,IAAI,EAAE,CAAC,IAAI,CAAC;SACb,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE9C,eAAe,CAAC,aAAa,CAAC,cAAc,EAAE;YAC5C,MAAM;SACP,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,CAAC,IAAY,EAAE,SAAiB;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;YAC3B,OAAO;gBACL;oBACE,IAAI;oBACJ,UAAU,EAAE,IAAI;oBAChB,SAAS;iBACV;aACF,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;QACjE,MAAM,UAAU,GAAa,EAAE,CAAC;QAEhC,KAAK,MAAM,SAAS,IAAI,eAAe,EAAE,CAAC;YACxC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;YAC5C,IAAI,SAAS,IAAI,SAAS,EAAE,CAAC;gBAC3B,UAAU,CAAC,IAAI,CAAC;oBACd,IAAI,EAAE,SAAS;oBACf,UAAU;oBACV,SAAS;iBACV,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,MAAM,mBAAmB,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;gBAC9D,UAAU,CAAC,IAAI,CAAC,GAAG,mBAAmB,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,eAAe,CAAC,IAAY;QAC1B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAChD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;YAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,CAAC,MAAgB,EAAE,SAAiB;QACxC,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,YAAY,GAAuB,EAAE,CAAC;QAC1C,IAAI,SAAS,GAAuB,EAAE,CAAC;QACvC,IAAI,kBAAkB,GAAG,CAAC,CAAC;QAC3B,IAAI,QAAQ,GAAG,IAAI,CAAC;QAEpB,MAAM,UAAU,GAAG,GAAS,EAAE;YAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;YACzD,SAAS,GAAG,YAAY,CAAC;YACzB,YAAY,GAAG,EAAE,CAAC;YAClB,kBAAkB,GAAG,CAAC,CAAC;YACvB,QAAQ,GAAG,IAAI,CAAC;YAEhB,IAAI,SAAS,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,OACE,SAAS,IAAI,CAAC;gBACd,kBAAkB,GAAG,SAAS,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,YAAY,EAClE,CAAC;gBACD,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC,SAAS,CAAE,CAAC;gBAC7C,kBAAkB,IAAI,MAAM,CAAC;gBAC7B,YAAY,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACrC,SAAS,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC,CAAC;QAEF,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;YAC5B,IAAI,QAAQ,CAAC,SAAS,GAAG,SAAS,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACtD,CAAC;YACD,IAAI,kBAAkB,GAAG,QAAQ,CAAC,SAAS,GAAG,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACrE,UAAU,EAAE,CAAC;YACf,CAAC;iBAAM,CAAC;gBACN,IACE,QAAQ,CAAC,UAAU;oBACnB,kBAAkB,GAAG,QAAQ,CAAC,SAAS,IAAI,SAAS;oBACpD,QAAQ,EACR,CAAC;oBACD,kBAAkB,IAAI,QAAQ,CAAC,SAAS,CAAC;oBACzC,YAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;oBACvD,MAAM,CAAC,KAAK,EAAE,CAAC;oBACf,QAAQ,GAAG,KAAK,CAAC;gBACnB,CAAC;qBAAM,CAAC;oBACN,UAAU,EAAE,CAAC;gBACf,CAAC;YACH,CAAC;QACH,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IAED,kBAAkB,CAAC,MAAgB;QACjC,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,YAAY,KAAK,EAAE,EAAE,CAAC;gBACxB,SAAS,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}

package/dist/services/sentenceSplitter.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=sentenceSplitter.test.d.ts.map

package/dist/services/sentenceSplitter.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentenceSplitter.test.d.ts","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":""}

package/dist/services/sentenceSplitter.test.js

@@ -0,0 +1,68 @@
+//@ts-nocheck
+import { expect, test } from 'vitest';
+import { CustomSentenceSplitter } from './sentenceSplitter';
+import { SentenceSplitter, IngestionPipeline, Document } from "llamaindex";
+// do these tests just to make sure that we can factor out my hacky fixes when llamaindex is fixed.
+// test that original sentenceSplitter splits on abbreviations
+// test that original sentenceSplitter splits on abbreviations even when specified
+// test that my modified sentenceSplitter excludes metadata when arg is specified
+// test that my modified sentenceSplitter includes metadata when arg is specified the other way
+let documents = [
+    new Document({ text: "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.  Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.  He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time." }),
+];
+let originalSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [
+        new SentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
+    ],
+});
+let customSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [
+        new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10 }),
+    ],
+});
+test("my modified sentenceSplitter doesn't eliminate spaces", () => {
+    customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
+        expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(false);
+        expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(false);
+        expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(false);
+    });
+});
+// test("original sentenceSplitter does eliminate spaces", () => {
+//     originalSentenceSplitterPipeline.run({documents: documents}).then((nodes) => {
+//         expect(nodes.some((node) => node["text"].indexOf("Co.elected") > -1)).toEqual(true);
+//         expect(nodes.some((node) => node["text"].indexOf("Mr.Weinberger") > -1)).toEqual(true);
+//         expect(nodes.some((node) => node["text"].indexOf("A.and") > -1)).toEqual(true);
+//     });
+// });
+let noAbbrevsCustomSentenceSplitterPipeline = new IngestionPipeline({
+    transformations: [
+        new CustomSentenceSplitter({ chunkSize: 50, chunkOverlap: 10, abbreviations: [] }),
+    ],
+});
+test("my modified sentenceSplitter doesn't split on specified abbreviations", () => {
+    customSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
+        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).not.toContainEqual(true);
+    });
+});
+// this is only a problem on branch fix/sentence-splitter-spaces
+// where the chunker is eliminated entirely in favor of just splitting by sentences with natural.
+test("original sentenceSplitter splits in silly places, like Mr", () => {
+    noAbbrevsCustomSentenceSplitterPipeline.run({ documents: documents }).then((nodes) => {
+        expect(nodes.map((node) => !!node["text"].match(/Mr\.$/))).toContainEqual(true);
+    });
+});
+const testcases = [
+    ["USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41", "USA v. 4227 JENIFER STREET N.W. WASHINGTON, D.C., AND ELECTRONIC DEVICES THEREIN UNDER RULE 41"],
+    ["JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee.", "JPMorgan Chase & Co. elected Mark Weinberger as a director, effective January 16, 2024, and the Board of Directors appointed him as a member of the Audit Committee."],
+    ["Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019.", "Mr. Weinberger was Global Chairman and Chief Executive Officer of Ernst & Young from 2013 to 2019."],
+    ["He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time.", "He was also elected a director of JPMorgan Chase Bank, N.A. and a manager of JPMorgan Chase Holdings LLC, and may be elected a director of such other subsidiary or subsidiaries as may be determined from time to time."],
+];
+testcases.forEach(([testcase_input, testcase_expected_output]) => {
+    test(`my sentenceSplitter correctly handles short sentence ${testcase_input}`, () => {
+        customSentenceSplitterPipeline.run({ documents: [new Document({ text: testcase_input })] }).then((nodes) => {
+            expect(nodes.length).toEqual(1);
+            expect(nodes[0]["text"]).toEqual(testcase_expected_output);
+        });
+    });
+});
+//# sourceMappingURL=sentenceSplitter.test.js.map

package/dist/services/sentenceSplitter.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sentenceSplitter.test.js","sourceRoot":"","sources":["../../src/services/sentenceSplitter.test.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,sBAAsB,EAAE,MAAM,oBAAoB,CAAA;AAC3D,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3E,mGAAmG;AACnG,8DAA8D;AAC9D,kFAAkF;AAElF,iFAAiF;AACjF,+FAA+F;AAI/F,IAAI,SAAS,GAAG;IACZ,IAAI,QAAQ,CAAC,EAAE,IAAI,EAAE,oeAAoe,EAAE,CAAC;CAC/f,CAAC;AAEF,IAAI,gCAAgC,GAAG,IAAI,iBAAiB,CAAC;IACzD,eAAe,EAAE;QACb,IAAI,gBAAgB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KACxD;CACJ,CAAC,CAAC;AACP,IAAI,8BAA8B,GAAG,IAAI,iBAAiB,CAAC;IACvD,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;KAChE;CACF,CAAC,CAAC;AAEL,IAAI,CAAC,uDAAuD,EAAE,GAAG,EAAE;IAC/D,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACrF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACxF,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,kEAAkE;AAClE,qFAAqF;AACrF,+FAA+F;AAC/F,kGAAkG;AAClG,0FAA0F;AAC1F,UAAU;AACV,MAAM;AAEN,IAAI,uCAAuC,GAAG,IAAI,iBAAiB,CAAC;IAChE,eAAe,EAAE;QACf,IAAI,sBAAsB,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAC,CAAC;KAClF;CACF,CAAC,CAAC;AAGH,IAAI,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACjF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACtE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACxF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,gEAAgE;AAChE,iGAAiG;AACjG,IAAI,CAAC,2DAA2D,EAAE,GAAG,EAAE;IACnE,uCAAuC,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,SAAS,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/E,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG;IACd,CAAC,gGAAgG,EAAE,gGAAgG,CAAC;IACpM,CAAC,sKAAsK,EAAE,sKAAsK,CAAC;IAChV,CAAC,oGAAoG,EAAE,oGAAoG,CAAC;IAC5M,CAAC,0NAA0N,EAAE,0NAA0N,CAAC;CAE3b,CAAC;AACF,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,EAAE,wBAAwB,CAAC,EAAE,EAAE;IAC7D,IAAI,CAAC,wDAAwD,cAAc,EAAE,EAAE,GAAG,EAAE;QAChF,8BAA8B,CAAC,GAAG,CAAC,EAAC,SAAS,EAAE,CAAC,IAAI,QAAQ,CAAC,EAAC,IAAI,EAAE,cAAc,EAAC,CAAC,CAAC,EAAC,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACnG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC;QAC/D,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAA;AACN,CAAC,CAAC,CAAC"}

package/dist/services/sploder.d.ts

@@ -0,0 +1,13 @@
+import { TextNode, TransformComponent } from "llamaindex";
+interface SploderConfig {
+    maxStringTokenCount: number;
+}
+export declare class Sploder extends TransformComponent {
+    private maxTokenCount;
+    private tokenizer;
+    constructor(config: SploderConfig);
+    private getTokenCount;
+    transform(nodes: TextNode[]): Promise<TextNode[]>;
+}
+export {};
+//# sourceMappingURL=sploder.d.ts.map

package/dist/services/sploder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sploder.d.ts","sourceRoot":"","sources":["../../src/services/sploder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAY,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAGpE,UAAU,aAAa;IACrB,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,qBAAa,OAAQ,SAAQ,kBAAkB;IAC7C,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,SAAS,CAAM;gBAIX,MAAM,EAAE,aAAa;IAMjC,OAAO,CAAC,aAAa;IAIf,SAAS,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;CAsCxD"}

package/dist/services/sploder.js

@@ -0,0 +1,45 @@
+import { TextNode, TransformComponent } from "llamaindex";
+import { encodingForModel } from "js-tiktoken";
+export class Sploder extends TransformComponent {
+    maxTokenCount;
+    tokenizer; // js-tiktoken encoder
+    // TODO: this is a hack to get the tokenizer for the embedding model
+    // TODO: this should be a singleton
+    constructor(config) {
+        super(async (nodes) => nodes); // no-op, to be replaced later
+        this.maxTokenCount = config.maxStringTokenCount;
+        this.tokenizer = encodingForModel("text-embedding-3-small");
+    }
+    getTokenCount(text) {
+        return this.tokenizer.encode(text).length;
+    }
+    async transform(nodes) {
+        const newNodes = [];
+        nodes.forEach((node, index) => {
+            // Keep original node
+            newNodes.push(node);
+            // Skip if text is too long
+            if (this.getTokenCount(node.text) > this.maxTokenCount) {
+                return;
+            }
+            const prevNode = index > 0 ? nodes[index - 1] : null;
+            const nextNode = index < nodes.length - 1 ? nodes[index + 1] : null;
+            // Create node with current + next if available
+            if (nextNode) {
+                newNodes.push(new TextNode({
+                    text: node.text + " " + nextNode.text,
+                    metadata: { ...node.metadata, isExpanded: true }
+                }));
+            }
+            // Create node with prev + current + next if both available
+            if (prevNode && nextNode) {
+                newNodes.push(new TextNode({
+                    text: prevNode.text + " " + node.text + " " + nextNode.text,
+                    metadata: { ...node.metadata, isExpanded: true }
+                }));
+            }
+        });
+        return newNodes;
+    }
+}
+//# sourceMappingURL=sploder.js.map

package/dist/services/sploder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sploder.js","sourceRoot":"","sources":["../../src/services/sploder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAY,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAM/C,MAAM,OAAO,OAAQ,SAAQ,kBAAkB;IACrC,aAAa,CAAS;IACtB,SAAS,CAAM,CAAC,sBAAsB;IAE9C,oEAAoE;IACpE,mCAAmC;IACnC,YAAY,MAAqB;QAC/B,KAAK,CAAC,KAAK,EAAE,KAAiB,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,8BAA8B;QACzE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,mBAAmB,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,gBAAgB,CAAC,wBAAwB,CAAC,CAAC;IAC9D,CAAC;IAEO,aAAa,CAAC,IAAY;QAChC,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAiB;QAC/B,MAAM,QAAQ,GAAe,EAAE,CAAC;QAEhC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC5B,qBAAqB;YACrB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAEpB,2BAA2B;YAC3B,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvD,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YAEpE,+CAA+C;YAC/C,IAAI,QAAQ,EAAE,CAAC;gBACb,QAAQ,CAAC,IAAI,CACX,IAAI,QAAQ,CAAC;oBACX,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI;oBACrC,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;iBACjD,CAAC,CACH,CAAC;YACJ,CAAC;YAED,2DAA2D;YAC3D,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CACX,IAAI,QAAQ,CAAC;oBACX,IAAI,EAAE,QAAQ,CAAC,IAAI,GAAG,GAAG,GAAG,IAAI,CAAC,IAAI,GAAG,GAAG,GAAG,QAAQ,CAAC,IAAI;oBAC3D,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE;iBACjD,CAAC,CACH,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}