@chonkiejs/chunk 0.9.1 → 0.9.3

package/index.js

@@ -28,6 +28,7 @@ import initWasm, {
     chunk_offsets_pattern as wasmChunkOffsetsPattern,
     split_offsets as wasmSplitOffsets,
     merge_splits as wasmMergeSplits,
+    initSync as initWasmSync,
 } from './pkg/chonkiejs_chunk.js';
 
 export { default_target_size, default_delimiters };
@@ -220,10 +221,30 @@ let initialized = false;
 
 /**
  * Initialize the WASM module. Must be called before using chunk functions.
+ * Automatically detects Node.js vs browser environment.
  */
 export async function init() {
     if (!initialized) {
-        await initWasm();
+        // Check if we're in Node.js
+        const isNode = typeof process !== 'undefined' &&
+                       process.versions != null &&
+                       process.versions.node != null;
+
+        if (isNode) {
+            // Node.js: read the wasm file and use initSync
+            const { readFileSync } = await import('node:fs');
+            const { fileURLToPath } = await import('node:url');
+            const { dirname, join } = await import('node:path');
+
+            const __filename = fileURLToPath(import.meta.url);
+            const __dirname = dirname(__filename);
+            const wasmPath = join(__dirname, 'pkg', 'chonkiejs_chunk_bg.wasm');
+            const wasmBytes = readFileSync(wasmPath);
+            initWasmSync({ module: wasmBytes });
+        } else {
+            // Browser: use fetch-based init
+            await initWasm();
+        }
         initialized = true;
     }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chonkiejs/chunk",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "description": "The fastest semantic text chunking library",
   "type": "module",
   "main": "index.js",

package/pkg/chonkiejs_chunk.d.ts

@@ -101,6 +101,28 @@ export function default_target_size(): number;
  */
 export function find_merge_indices(token_counts: Uint32Array, chunk_size: number): Uint32Array;
 
+/**
+ * Merge segments based on token counts, respecting chunk size limits.
+ * Returns a flat array [endIndex1, tokenCount1, endIndex2, tokenCount2, ...].
+ *
+ * This is the equivalent of Chonkie's `_merge_splits` function.
+ * Used by RecursiveChunker to merge small segments into larger chunks
+ * that fit within a token budget.
+ *
+ * @param token_counts - Array of token counts for each segment
+ * @param chunk_size - Maximum tokens per merged chunk
+ * @param combine_whitespace - If true, adds +1 token per join for whitespace
+ * @returns Flat array of [endIndex, tokenCount] pairs
+ *
+ * @example
+ * ```javascript
+ * const result = merge_splits([1, 1, 1, 1, 1, 1, 1], 3, false);
+ * // result = [3, 3, 6, 3, 7, 1]
+ * // Meaning: chunk 0-3 has 3 tokens, chunk 3-6 has 3 tokens, chunk 6-7 has 1 token
+ * ```
+ */
+export function merge_splits(token_counts: Uint32Array, chunk_size: number, combine_whitespace?: boolean | null): Uint32Array;
+
 /**
  * Split text at every delimiter occurrence, returning offsets.
  * Unlike chunk_offsets which creates size-based chunks, this splits at
@@ -140,11 +162,12 @@ export interface InitOutput {
   readonly default_delimiters: () => [number, number];
   readonly default_target_size: () => number;
   readonly find_merge_indices: (a: number, b: number, c: number) => [number, number];
+  readonly merge_splits: (a: number, b: number, c: number, d: number) => [number, number];
   readonly split_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
   readonly __wbindgen_externrefs: WebAssembly.Table;
   readonly __wbindgen_malloc: (a: number, b: number) => number;
-  readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
   readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+  readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
   readonly __wbindgen_start: () => void;
 }
 

package/pkg/chonkiejs_chunk.js

@@ -352,6 +352,39 @@ export function find_merge_indices(token_counts, chunk_size) {
     return v2;
 }
 
+/**
+ * Merge segments based on token counts, respecting chunk size limits.
+ * Returns a flat array [endIndex1, tokenCount1, endIndex2, tokenCount2, ...].
+ *
+ * This is the equivalent of Chonkie's `_merge_splits` function.
+ * Used by RecursiveChunker to merge small segments into larger chunks
+ * that fit within a token budget.
+ *
+ * @param token_counts - Array of token counts for each segment
+ * @param chunk_size - Maximum tokens per merged chunk
+ * @param combine_whitespace - If true, adds +1 token per join for whitespace
+ * @returns Flat array of [endIndex, tokenCount] pairs
+ *
+ * @example
+ * ```javascript
+ * const result = merge_splits([1, 1, 1, 1, 1, 1, 1], 3, false);
+ * // result = [3, 3, 6, 3, 7, 1]
+ * // Meaning: chunk 0-3 has 3 tokens, chunk 3-6 has 3 tokens, chunk 6-7 has 1 token
+ * ```
+ * @param {Uint32Array} token_counts
+ * @param {number} chunk_size
+ * @param {boolean | null} [combine_whitespace]
+ * @returns {Uint32Array}
+ */
+export function merge_splits(token_counts, chunk_size, combine_whitespace) {
+    const ptr0 = passArray32ToWasm0(token_counts, wasm.__wbindgen_malloc);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.merge_splits(ptr0, len0, chunk_size, isLikeNone(combine_whitespace) ? 0xFFFFFF : combine_whitespace ? 1 : 0);
+    var v2 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
+    wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
+    return v2;
+}
+
 /**
  * Split text at every delimiter occurrence, returning offsets.
  * Unlike chunk_offsets which creates size-based chunks, this splits at

package/pkg/chonkiejs_chunk_bg.wasm.d.ts

@@ -12,9 +12,10 @@ export const chunker_with_pattern: (a: number, b: number, c: number, d: number,
 export const default_delimiters: () => [number, number];
 export const default_target_size: () => number;
 export const find_merge_indices: (a: number, b: number, c: number) => [number, number];
+export const merge_splits: (a: number, b: number, c: number, d: number) => [number, number];
 export const split_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
 export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_malloc: (a: number, b: number) => number;
-export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_free: (a: number, b: number, c: number) => void;
+export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
 export const __wbindgen_start: () => void;

package/pkg/package.json

@@ -1,7 +1,7 @@
 {
   "name": "chonkiejs-chunk",
   "type": "module",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "license": "MIT OR Apache-2.0",
   "files": [
     "chonkiejs_chunk_bg.wasm",