@chonkiejs/chunk 0.10.0 → 0.10.1

package/index.js

@@ -74,14 +74,14 @@ function toBytes(input) {
 export function* chunk(text, options = {}) {
     const isString = typeof text === 'string';
     const bytes = toBytes(text);
-    const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+    const { size, delimiters, pattern, patterns, prefix, consecutive, forwardFallback } = options;
 
     let flat;
     if (pattern) {
         const patternBytes = toBytes(pattern);
         flat = wasmChunkOffsetsPattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
     } else {
-        flat = wasmChunkOffsets(bytes, size, delimiters, prefix);
+        flat = wasmChunkOffsets(bytes, size, delimiters, prefix, consecutive, forwardFallback, patterns);
     }
 
     for (let i = 0; i < flat.length; i += 2) {
@@ -99,6 +99,7 @@ export function* chunk(text, options = {}) {
  * @param {number} [options.size=4096] - Target chunk size in bytes
  * @param {string} [options.delimiters="\n.?"] - Delimiter characters
  * @param {string | Uint8Array} [options.pattern] - Multi-byte pattern to split on
+ * @param {string[]} [options.patterns] - Multi-byte patterns, composable with delimiters
  * @param {boolean} [options.prefix=false] - Put delimiter/pattern at start of next chunk
  * @param {boolean} [options.consecutive=false] - Split at START of consecutive runs
  * @param {boolean} [options.forwardFallback=false] - Search forward if no pattern in backward window
@@ -106,14 +107,14 @@ export function* chunk(text, options = {}) {
  */
 export function chunk_offsets(text, options = {}) {
     const bytes = toBytes(text);
-    const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+    const { size, delimiters, pattern, patterns, prefix, consecutive, forwardFallback } = options;
 
     let flat;
     if (pattern) {
         const patternBytes = toBytes(pattern);
         flat = wasmChunkOffsetsPattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
     } else {
-        flat = wasmChunkOffsets(bytes, size, delimiters, prefix);
+        flat = wasmChunkOffsets(bytes, size, delimiters, prefix, consecutive, forwardFallback, patterns);
     }
 
     const pairs = [];
@@ -275,6 +276,7 @@ export class Chunker {
      * @param {number} [options.size=4096] - Target chunk size in bytes
      * @param {string} [options.delimiters="\n.?"] - Delimiter characters
      * @param {string | Uint8Array} [options.pattern] - Multi-byte pattern to split on
+     * @param {string[]} [options.patterns] - Multi-byte patterns, composable with delimiters
      * @param {boolean} [options.prefix=false] - Put delimiter/pattern at start of next chunk
      * @param {boolean} [options.consecutive=false] - Split at START of consecutive runs
      * @param {boolean} [options.forwardFallback=false] - Search forward if no pattern in backward window
@@ -282,13 +284,13 @@ export class Chunker {
     constructor(text, options = {}) {
         this._isString = typeof text === 'string';
         const bytes = toBytes(text);
-        const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+        const { size, delimiters, pattern, patterns, prefix, consecutive, forwardFallback } = options;
 
         if (pattern) {
             const patternBytes = toBytes(pattern);
             this._chunker = WasmChunker.with_pattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
         } else {
-            this._chunker = new WasmChunker(bytes, size, delimiters, prefix);
+            this._chunker = new WasmChunker(bytes, size, delimiters, prefix, consecutive, forwardFallback, patterns);
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chonkiejs/chunk",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "description": "The fastest semantic text chunking library",
   "type": "module",
   "main": "index.js",

package/pkg/chonkiejs_chunk.d.ts

@@ -30,7 +30,7 @@ export class Chunker {
    * @param consecutive - Split at START of consecutive runs (default: false)
    * @param forward_fallback - Search forward if no delimiter in backward window (default: false)
    */
-  constructor(text: Uint8Array, size?: number | null, delimiters?: string | null, prefix?: boolean | null, consecutive?: boolean | null, forward_fallback?: boolean | null);
+  constructor(text: Uint8Array, size?: number | null, delimiters?: string | null, prefix?: boolean | null, consecutive?: boolean | null, forward_fallback?: boolean | null, patterns?: Array<any> | null);
   /**
    * Get the next chunk, or undefined if exhausted.
    */
@@ -55,7 +55,7 @@ export class Chunker {
  * }
  * ```
  */
-export function chunk_offsets(text: Uint8Array, size?: number | null, delimiters?: string | null, prefix?: boolean | null, consecutive?: boolean | null, forward_fallback?: boolean | null): Uint32Array;
+export function chunk_offsets(text: Uint8Array, size?: number | null, delimiters?: string | null, prefix?: boolean | null, consecutive?: boolean | null, forward_fallback?: boolean | null, patterns?: Array<any> | null): Uint32Array;
 
 /**
  * Fast chunking function with multi-byte pattern support.
@@ -152,10 +152,10 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl
 export interface InitOutput {
   readonly memory: WebAssembly.Memory;
   readonly __wbg_chunker_free: (a: number, b: number) => void;
-  readonly chunk_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => [number, number];
+  readonly chunk_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
   readonly chunk_offsets_pattern: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => [number, number];
   readonly chunker_collect_offsets: (a: number) => [number, number];
-  readonly chunker_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
+  readonly chunker_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => number;
   readonly chunker_next: (a: number) => [number, number];
   readonly chunker_reset: (a: number) => void;
   readonly chunker_with_pattern: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
@@ -164,10 +164,11 @@ export interface InitOutput {
   readonly find_merge_indices: (a: number, b: number, c: number) => [number, number];
   readonly merge_splits: (a: number, b: number, c: number, d: number) => [number, number];
   readonly split_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
-  readonly __wbindgen_externrefs: WebAssembly.Table;
   readonly __wbindgen_malloc: (a: number, b: number) => number;
   readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+  readonly __wbindgen_externrefs: WebAssembly.Table;
   readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+  readonly __externref_table_alloc: () => number;
   readonly __wbindgen_start: () => void;
 }
 

package/pkg/chonkiejs_chunk.js

@@ -1,5 +1,11 @@
 let wasm;
 
+function addToExternrefTable0(obj) {
+    const idx = wasm.__externref_table_alloc();
+    wasm.__wbindgen_externrefs.set(idx, obj);
+    return idx;
+}
+
 function getArrayU32FromWasm0(ptr, len) {
     ptr = ptr >>> 0;
     return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
@@ -10,6 +16,14 @@ function getArrayU8FromWasm0(ptr, len) {
     return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
 }
 
+let cachedDataViewMemory0 = null;
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
+    }
+    return cachedDataViewMemory0;
+}
+
 function getStringFromWasm0(ptr, len) {
     ptr = ptr >>> 0;
     return decodeText(ptr, len);
@@ -207,13 +221,14 @@ export class Chunker {
      * @param {boolean | null} [prefix]
      * @param {boolean | null} [consecutive]
      * @param {boolean | null} [forward_fallback]
+     * @param {Array<any> | null} [patterns]
      */
-    constructor(text, size, delimiters, prefix, consecutive, forward_fallback) {
+    constructor(text, size, delimiters, prefix, consecutive, forward_fallback, patterns) {
         const ptr0 = passArray8ToWasm0(text, wasm.__wbindgen_malloc);
         const len0 = WASM_VECTOR_LEN;
         var ptr1 = isLikeNone(delimiters) ? 0 : passStringToWasm0(delimiters, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
         var len1 = WASM_VECTOR_LEN;
-        const ret = wasm.chunker_new(ptr0, len0, isLikeNone(size) ? 0x100000001 : (size) >>> 0, ptr1, len1, isLikeNone(prefix) ? 0xFFFFFF : prefix ? 1 : 0, isLikeNone(consecutive) ? 0xFFFFFF : consecutive ? 1 : 0, isLikeNone(forward_fallback) ? 0xFFFFFF : forward_fallback ? 1 : 0);
+        const ret = wasm.chunker_new(ptr0, len0, isLikeNone(size) ? 0x100000001 : (size) >>> 0, ptr1, len1, isLikeNone(prefix) ? 0xFFFFFF : prefix ? 1 : 0, isLikeNone(consecutive) ? 0xFFFFFF : consecutive ? 1 : 0, isLikeNone(forward_fallback) ? 0xFFFFFF : forward_fallback ? 1 : 0, isLikeNone(patterns) ? 0 : addToExternrefTable0(patterns));
         this.__wbg_ptr = ret >>> 0;
         ChunkerFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -259,14 +274,15 @@ if (Symbol.dispose) Chunker.prototype[Symbol.dispose] = Chunker.prototype.free;
  * @param {boolean | null} [prefix]
  * @param {boolean | null} [consecutive]
  * @param {boolean | null} [forward_fallback]
+ * @param {Array<any> | null} [patterns]
  * @returns {Uint32Array}
  */
-export function chunk_offsets(text, size, delimiters, prefix, consecutive, forward_fallback) {
+export function chunk_offsets(text, size, delimiters, prefix, consecutive, forward_fallback, patterns) {
     const ptr0 = passArray8ToWasm0(text, wasm.__wbindgen_malloc);
     const len0 = WASM_VECTOR_LEN;
     var ptr1 = isLikeNone(delimiters) ? 0 : passStringToWasm0(delimiters, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
     var len1 = WASM_VECTOR_LEN;
-    const ret = wasm.chunk_offsets(ptr0, len0, isLikeNone(size) ? 0x100000001 : (size) >>> 0, ptr1, len1, isLikeNone(prefix) ? 0xFFFFFF : prefix ? 1 : 0, isLikeNone(consecutive) ? 0xFFFFFF : consecutive ? 1 : 0, isLikeNone(forward_fallback) ? 0xFFFFFF : forward_fallback ? 1 : 0);
+    const ret = wasm.chunk_offsets(ptr0, len0, isLikeNone(size) ? 0x100000001 : (size) >>> 0, ptr1, len1, isLikeNone(prefix) ? 0xFFFFFF : prefix ? 1 : 0, isLikeNone(consecutive) ? 0xFFFFFF : consecutive ? 1 : 0, isLikeNone(forward_fallback) ? 0xFFFFFF : forward_fallback ? 1 : 0, isLikeNone(patterns) ? 0 : addToExternrefTable0(patterns));
     var v3 = getArrayU32FromWasm0(ret[0], ret[1]).slice();
     wasm.__wbindgen_free(ret[0], ret[1] * 4, 4);
     return v3;
@@ -460,9 +476,25 @@ async function __wbg_load(module, imports) {
 function __wbg_get_imports() {
     const imports = {};
     imports.wbg = {};
+    imports.wbg.__wbg___wbindgen_string_get_a2a31e16edf96e42 = function(arg0, arg1) {
+        const obj = arg1;
+        const ret = typeof(obj) === 'string' ? obj : undefined;
+        var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc);
+        var len1 = WASM_VECTOR_LEN;
+        getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+        getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+    };
     imports.wbg.__wbg___wbindgen_throw_dd24417ed36fc46e = function(arg0, arg1) {
         throw new Error(getStringFromWasm0(arg0, arg1));
     };
+    imports.wbg.__wbg_get_6b7bd52aca3f9671 = function(arg0, arg1) {
+        const ret = arg0[arg1 >>> 0];
+        return ret;
+    };
+    imports.wbg.__wbg_length_d45040a40c570362 = function(arg0) {
+        const ret = arg0.length;
+        return ret;
+    };
     imports.wbg.__wbindgen_init_externref_table = function() {
         const table = wasm.__wbindgen_externrefs;
         const offset = table.grow(4);
@@ -479,6 +511,7 @@ function __wbg_get_imports() {
 function __wbg_finalize_init(instance, module) {
     wasm = instance.exports;
     __wbg_init.__wbindgen_wasm_module = module;
+    cachedDataViewMemory0 = null;
     cachedUint32ArrayMemory0 = null;
     cachedUint8ArrayMemory0 = null;
 

package/pkg/chonkiejs_chunk_bg.wasm.d.ts

@@ -2,10 +2,10 @@
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
 export const __wbg_chunker_free: (a: number, b: number) => void;
-export const chunk_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => [number, number];
+export const chunk_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => [number, number];
 export const chunk_offsets_pattern: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => [number, number];
 export const chunker_collect_offsets: (a: number) => [number, number];
-export const chunker_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
+export const chunker_new: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number, i: number) => number;
 export const chunker_next: (a: number) => [number, number];
 export const chunker_reset: (a: number) => void;
 export const chunker_with_pattern: (a: number, b: number, c: number, d: number, e: number, f: number, g: number, h: number) => number;
@@ -14,8 +14,9 @@ export const default_target_size: () => number;
 export const find_merge_indices: (a: number, b: number, c: number) => [number, number];
 export const merge_splits: (a: number, b: number, c: number, d: number) => [number, number];
 export const split_offsets: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => [number, number];
-export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_malloc: (a: number, b: number) => number;
 export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+export const __wbindgen_externrefs: WebAssembly.Table;
 export const __wbindgen_free: (a: number, b: number, c: number) => void;
+export const __externref_table_alloc: () => number;
 export const __wbindgen_start: () => void;

package/pkg/package.json

@@ -1,7 +1,7 @@
 {
   "name": "chonkiejs-chunk",
   "type": "module",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "license": "MIT OR Apache-2.0",
   "files": [
     "chonkiejs_chunk_bg.wasm",