k0ntext 3.6.0 → 3.7.0

package/dist/utils/chunking.js

@@ -0,0 +1,133 @@
+/**
+ * Text Chunking Utility
+ *
+ * Splits large texts into chunks suitable for embedding generation.
+ * Handles token limits, word boundaries, and overlap for context preservation.
+ */
+/**
+ * Estimate token count for text.
+ *
+ * Uses a simple heuristic: ~4 characters per token for English text.
+ * This is approximate but works well for our use case.
+ *
+ * @param text - Text to estimate tokens for
+ * @returns Estimated token count
+ */
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    // Remove whitespace for more accurate estimate
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return 0;
+    // Rough estimate: 1 token per 4 characters for English text
+    // This is a simplification but works well for most cases
+    return Math.ceil(trimmed.length / 4);
+}
+/**
+ * Split text into chunks that fit within max tokens.
+ *
+ * Tries to break at word boundaries when possible.
+ * Adds overlap between chunks to preserve context.
+ *
+ * @param text - Text to chunk
+ * @param maxTokens - Maximum tokens per chunk (default: 8000 for OpenRouter)
+ * @param overlapTokens - Number of tokens to overlap between chunks (default: 0)
+ * @returns Array of text chunks
+ */
+export function chunkText(text, maxTokens = 8000, overlapTokens = 0) {
+    // Handle empty or very short text
+    if (!text || text.trim().length === 0) {
+        return [''];
+    }
+    const trimmedText = text.trim();
+    const estimatedTokens = estimateTokens(trimmedText);
+    // If text is under the limit, return as-is
+    if (estimatedTokens <= maxTokens) {
+        return [trimmedText];
+    }
+    const chunks = [];
+    const maxChars = maxTokens * 4; // Convert tokens to approximate characters
+    const overlapChars = overlapTokens * 4;
+    let startIndex = 0;
+    let previousEndIndex = 0;
+    let loopCount = 0;
+    const maxLoops = 1000; // Safety limit to prevent infinite loops
+    while (startIndex < trimmedText.length && loopCount < maxLoops) {
+        loopCount++;
+        // Calculate end index for this chunk
+        let endIndex = Math.min(startIndex + maxChars, trimmedText.length);
+        // If not the last chunk, try to break at a word boundary
+        if (endIndex < trimmedText.length) {
+            // Look for word boundary near the end
+            const boundaryChars = 200; // Look back up to 200 chars
+            const searchStart = Math.max(startIndex, endIndex - boundaryChars);
+            const substring = trimmedText.slice(searchStart, endIndex);
+            // Try to find line break first, then space, then punctuation
+            let breakIndex = -1;
+            // Look for last newline in the window
+            const lastNewline = substring.lastIndexOf('\n');
+            if (lastNewline !== -1) {
+                breakIndex = searchStart + lastNewline + 1;
+            }
+            else {
+                // Look for last space in the window
+                const lastSpace = substring.lastIndexOf(' ');
+                if (lastSpace !== -1) {
+                    breakIndex = searchStart + lastSpace + 1;
+                }
+                else {
+                    // Look for sentence-ending punctuation
+                    for (let i = substring.length - 1; i >= Math.max(0, substring.length - 100); i--) {
+                        const char = substring[i];
+                        if (char === '.' || char === '!' || char === '?') {
+                            // Make sure it's actually a sentence end (followed by space or end)
+                            const nextChar = substring[i + 1];
+                            if (!nextChar || nextChar === ' ' || nextChar === '\n') {
+                                breakIndex = searchStart + i + 1;
+                                break;
+                            }
+                        }
+                    }
+                }
+            }
+            // Use the break index if found, otherwise use the calculated end
+            if (breakIndex > startIndex) {
+                endIndex = breakIndex;
+            }
+        }
+        // Extract the chunk
+        const chunk = trimmedText.slice(startIndex, endIndex);
+        chunks.push(chunk);
+        // Move to next chunk, accounting for overlap
+        if (overlapChars > 0 && endIndex < trimmedText.length) {
+            // Only apply overlap if not at the end
+            startIndex = Math.max(endIndex - overlapChars, endIndex - maxChars / 2);
+            // Ensure we make progress
+            if (startIndex <= previousEndIndex) {
+                startIndex = endIndex;
+            }
+            // Also ensure we move forward at least a bit
+            if (startIndex >= endIndex) {
+                startIndex = endIndex;
+            }
+        }
+        else {
+            startIndex = endIndex;
+        }
+        previousEndIndex = endIndex;
+    }
+    return chunks;
+}
+/**
+ * Chunk text specifically for embedding generation.
+ *
+ * Uses 8000 token limit (OpenRouter's limit for text-embedding-3-small).
+ *
+ * @param text - Text to chunk
+ * @returns Array of text chunks suitable for embeddings
+ */
+export function chunkForEmbedding(text) {
+    return chunkText(text, 8000, 100); // 100 token overlap for context
+}
+//# sourceMappingURL=chunking.js.map

package/dist/utils/chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../src/utils/chunking.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,+CAA+C;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEnC,4DAA4D;IAC5D,yDAAyD;IACzD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,YAAoB,IAAI,EACxB,gBAAwB,CAAC;IAEzB,kCAAkC;IAClC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,eAAe,GAAG,cAAc,CAAC,WAAW,CAAC,CAAC;IAEpD,2CAA2C;IAC3C,IAAI,eAAe,IAAI,SAAS,EAAE,CAAC;QACjC,OAAO,CAAC,WAAW,CAAC,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,2CAA2C;IAC3E,MAAM,YAAY,GAAG,aAAa,GAAG,CAAC,CAAC;IAEvC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,yCAAyC;IAEhE,OAAO,UAAU,GAAG,WAAW,CAAC,MAAM,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;QAC/D,SAAS,EAAE,CAAC;QAEZ,qCAAqC;QACrC,IAAI,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,QAAQ,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;QAEnE,yDAAyD;QACzD,IAAI,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;YAClC,sCAAsC;YACtC,MAAM,aAAa,GAAG,GAAG,CAAC,CAAC,4BAA4B;YACvD,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,QAAQ,GAAG,aAAa,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YAE3D,6DAA6D;YAC7D,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;YAEpB,sCAAsC;YACtC,MAAM,WAAW,GAAG,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAChD,IAAI,WAAW,KAAK,CAAC,CAAC,EAAE,CAAC;gBACvB,UAAU,GAAG,WAAW,GAAG,WAAW,GAAG,CAAC,CAAC;YAC7C,CAAC;iBAAM,CAAC;gBACN,oCAAoC;gBACpC,MAAM,SAAS,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC7C,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;oBACrB,UAAU,GAAG,WAAW,GAAG,SAAS,GAAG,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,uCAAuC;oBACvC,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;wBACjF,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;wBAC1B,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;4BACjD,oEAAoE;4BACpE,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;4BAClC,IAAI,CAAC,QAAQ,IAAI,QAAQ,KAAK,GAAG,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gCACvD,UAAU,GAAG,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC;gCACjC,MAAM;4BACR,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,iEAAiE;YACjE,IAAI,UAAU,GAAG,UAAU,EAAE,CAAC;gBAC5B,QAAQ,GAAG,UAAU,CAAC;YACxB,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAEnB,6CAA6C;QAC7C,IAAI,YAAY,GAAG,CAAC,IAAI,QAAQ,GAAG,WAAW,CAAC,MAAM,EAAE,CAAC;YACtD,uCAAuC;YACvC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,YAAY,EAAE,QAAQ,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC;YAExE,0BAA0B;YAC1B,IAAI,UAAU,IAAI,gBAAgB,EAAE,CAAC;gBACnC,UAAU,GAAG,QAAQ,CAAC;YACxB,CAAC;YAED,6CAA6C;YAC7C,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;gBAC3B,UAAU,GAAG,QAAQ,CAAC;YACxB,CAAC;QACH,CAAC;aAAM,CAAC;YACN,UAAU,GAAG,QAAQ,CAAC;QACxB,CAAC;QAED,gBAAgB,GAAG,QAAQ,CAAC;IAC9B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,OAAO,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC,gCAAgC;AACrE,CAAC"}

package/dist/utils/encoding.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Encoding Utilities
+ *
+ * Handles text encoding issues across different platforms.
+ */
+/**
+ * Strip UTF-8 BOM (Byte Order Mark) from a string.
+ *
+ * The UTF-8 BOM is the byte sequence EF BB BF (U+FEFF).
+ * Some Windows editors add this to the start of files,
+ * which can break environment variable parsing.
+ *
+ * @param str - String that may contain a BOM
+ * @returns String with BOM removed if present
+ */
+export declare function stripBOM(str: string): string;
+/**
+ * Detect if a string has a UTF-8 BOM.
+ *
+ * @param str - String to check
+ * @returns true if BOM is present
+ */
+export declare function hasBOM(str: string): boolean;
+//# sourceMappingURL=encoding.d.ts.map

package/dist/utils/encoding.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoding.d.ts","sourceRoot":"","sources":["../../src/utils/encoding.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;GASG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM5C;AAED;;;;;GAKG;AACH,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAE3C"}

package/dist/utils/encoding.js

@@ -0,0 +1,32 @@
+/**
+ * Encoding Utilities
+ *
+ * Handles text encoding issues across different platforms.
+ */
+/**
+ * Strip UTF-8 BOM (Byte Order Mark) from a string.
+ *
+ * The UTF-8 BOM is the byte sequence EF BB BF (U+FEFF).
+ * Some Windows editors add this to the start of files,
+ * which can break environment variable parsing.
+ *
+ * @param str - String that may contain a BOM
+ * @returns String with BOM removed if present
+ */
+export function stripBOM(str) {
+    // Check for BOM at position 0
+    if (str.charCodeAt(0) === 0xFEFF) {
+        return str.slice(1);
+    }
+    return str;
+}
+/**
+ * Detect if a string has a UTF-8 BOM.
+ *
+ * @param str - String to check
+ * @returns true if BOM is present
+ */
+export function hasBOM(str) {
+    return str.charCodeAt(0) === 0xFEFF;
+}
+//# sourceMappingURL=encoding.js.map

package/dist/utils/encoding.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoding.js","sourceRoot":"","sources":["../../src/utils/encoding.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH;;;;;;;;;GASG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,8BAA8B;IAC9B,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;QACjC,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,MAAM,CAAC,GAAW;IAChC,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC;AACtC,CAAC"}

package/dist/utils/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Shared Utilities Module
+ *
+ * Exports all shared utility functions.
+ */
+export * from './encoding.js';
+export * from './chunking.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/utils/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}

package/dist/utils/index.js

@@ -0,0 +1,8 @@
+/**
+ * Shared Utilities Module
+ *
+ * Exports all shared utility functions.
+ */
+export * from './encoding.js';
+export * from './chunking.js';
+//# sourceMappingURL=index.js.map

package/dist/utils/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/utils/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}