@chonkiejs/chunk 0.5.0

package/README.md

@@ -0,0 +1,97 @@
+<p align="center">
+  <img src="../../assets/memchunk_wide.png" alt="@chonkiejs/chunk" width="500">
+</p>
+
+<h1 align="center">@chonkiejs/chunk</h1>
+
+<p align="center">
+  <em>the fastest text chunking library — up to 1 TB/s throughput</em>
+</p>
+
+<p align="center">
+  <a href="https://crates.io/crates/chunk"><img src="https://img.shields.io/crates/v/chunk.svg?color=e74c3c" alt="crates.io"></a>
+  <a href="https://pypi.org/project/chonkie-core"><img src="https://img.shields.io/pypi/v/chonkie-core.svg?color=e67e22" alt="PyPI"></a>
+  <a href="https://www.npmjs.com/package/@chonkiejs/chunk"><img src="https://img.shields.io/npm/v/@chonkiejs/chunk.svg?color=2ecc71" alt="npm"></a>
+  <a href="https://github.com/chonkie-inc/chunk"><img src="https://img.shields.io/badge/github-chunk-3498db" alt="GitHub"></a>
+  <a href="LICENSE-MIT"><img src="https://img.shields.io/badge/license-MIT%2FApache--2.0-9b59b6.svg" alt="License"></a>
+</p>
+
+---
+
+you know how every chunking library claims to be fast? yeah, we actually meant it.
+
+**@chonkiejs/chunk** splits text at semantic boundaries (periods, newlines, the usual suspects) and does it stupid fast. we're talking "chunk the entire english wikipedia in 120ms" fast.
+
+want to know how? [read the blog post](https://minha.sh/posts/so,-you-want-to-chunk-really-fast) where we nerd out about SIMD instructions and lookup tables.
+
+## 📦 installation
+
+```bash
+npm install @chonkiejs/chunk
+```
+
+looking for [rust](https://github.com/chonkie-inc/chunk) or [python](https://github.com/chonkie-inc/chunk/tree/main/packages/python)?
+
+## 🚀 usage
+
+```javascript
+import { init, chunk } from '@chonkiejs/chunk';
+
+// initialize wasm (required once)
+await init();
+
+const text = "Hello world. How are you? I'm fine.\nThanks for asking.";
+
+// with defaults (4KB chunks, split at \n . ?)
+for (const slice of chunk(text)) {
+    console.log(slice);
+}
+
+// with custom size
+for (const slice of chunk(text, { size: 1024 })) {
+    console.log(slice);
+}
+
+// with custom delimiters
+for (const slice of chunk(text, { delimiters: ".?!\n" })) {
+    console.log(slice);
+}
+
+// with multi-byte pattern (e.g., metaspace ▁ for SentencePiece tokenizers)
+for (const slice of chunk(text, { pattern: "▁", prefix: true })) {
+    console.log(slice);
+}
+
+// with consecutive pattern handling (split at START of runs, not middle)
+for (const slice of chunk("word   next", { pattern: " ", consecutive: true })) {
+    console.log(slice);
+}
+
+// with forward fallback (search forward if no pattern in backward window)
+for (const slice of chunk(text, { pattern: " ", forwardFallback: true })) {
+    console.log(slice);
+}
+
+// collect all chunks
+const chunks = [...chunk(text)];
+```
+
+pass strings and get strings back. for zero-copy performance with binary data, pass `Uint8Array` and you'll get `Uint8Array` views back.
+
+## 📝 citation
+
+if you use @chonkiejs/chunk in your research, please cite it as follows:
+
+```bibtex
+@software{chunk2025,
+  author = {Minhas, Bhavnick},
+  title = {chunk: The fastest text chunking library},
+  year = {2025},
+  publisher = {GitHub},
+  howpublished = {\url{https://github.com/chonkie-inc/chunk}},
+}
+```
+
+## 📄 license
+
+licensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or [MIT license](LICENSE-MIT) at your option.

package/index.js

@@ -0,0 +1,225 @@
+/**
+ * @chonkiejs/chunk - The fastest semantic text chunking library
+ *
+ * @example
+ * ```javascript
+ * import { init, chunk } from '@chonkiejs/chunk';
+ *
+ * await init();
+ *
+ * // Simple string API - strings in, strings out
+ * for (const slice of chunk("Hello. World. Test.", { size: 10 })) {
+ *     console.log(slice);
+ * }
+ *
+ * // Or use bytes for zero-copy performance
+ * const bytes = new TextEncoder().encode("Hello. World.");
+ * for (const slice of chunk(bytes, { size: 10 })) {
+ *     console.log(slice); // Uint8Array
+ * }
+ * ```
+ */
+
+import initWasm, {
+    Chunker as WasmChunker,
+    default_target_size,
+    default_delimiters,
+    chunk_offsets as wasmChunkOffsets,
+    chunk_offsets_pattern as wasmChunkOffsetsPattern,
+} from './pkg/chonkiejs_chunk.js';
+
+export { default_target_size, default_delimiters };
+
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+/**
+ * Convert input to bytes if it's a string.
+ * @param {string | Uint8Array} input
+ * @returns {Uint8Array}
+ */
+function toBytes(input) {
+    return typeof input === 'string' ? encoder.encode(input) : input;
+}
+
+/**
+ * Split text into chunks at delimiter boundaries.
+ * Accepts strings or Uint8Array. Returns the same type as input.
+ *
+ * @param {string | Uint8Array} text - The text to chunk
+ * @param {Object} [options] - Options
+ * @param {number} [options.size=4096] - Target chunk size in bytes
+ * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+ * @param {string | Uint8Array} [options.pattern] - Multi-byte pattern to split on
+ * @param {boolean} [options.prefix=false] - Put delimiter/pattern at start of next chunk
+ * @param {boolean} [options.consecutive=false] - Split at START of consecutive runs
+ * @param {boolean} [options.forwardFallback=false] - Search forward if no pattern in backward window
+ * @yields {string | Uint8Array} Chunks (same type as input)
+ *
+ * @example
+ * // String input returns strings
+ * for (const slice of chunk("Hello. World.", { size: 10 })) {
+ *     console.log(slice);
+ * }
+ *
+ * @example
+ * // With pattern (e.g., metaspace for SentencePiece)
+ * for (const slice of chunk("Hello▁World▁Test", { pattern: "▁", prefix: true })) {
+ *     console.log(slice);
+ * }
+ */
+export function* chunk(text, options = {}) {
+    const isString = typeof text === 'string';
+    const bytes = toBytes(text);
+    const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+
+    let flat;
+    if (pattern) {
+        const patternBytes = toBytes(pattern);
+        flat = wasmChunkOffsetsPattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
+    } else {
+        flat = wasmChunkOffsets(bytes, size, delimiters, prefix);
+    }
+
+    for (let i = 0; i < flat.length; i += 2) {
+        const slice = bytes.subarray(flat[i], flat[i + 1]);
+        yield isString ? decoder.decode(slice) : slice;
+    }
+}
+
+/**
+ * Get chunk offsets without creating views.
+ * Returns an array of [start, end] offset pairs.
+ *
+ * @param {string | Uint8Array} text - The text to chunk
+ * @param {Object} [options] - Options
+ * @param {number} [options.size=4096] - Target chunk size in bytes
+ * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+ * @param {string | Uint8Array} [options.pattern] - Multi-byte pattern to split on
+ * @param {boolean} [options.prefix=false] - Put delimiter/pattern at start of next chunk
+ * @param {boolean} [options.consecutive=false] - Split at START of consecutive runs
+ * @param {boolean} [options.forwardFallback=false] - Search forward if no pattern in backward window
+ * @returns {Array<[number, number]>} Array of [start, end] byte offset pairs
+ */
+export function chunk_offsets(text, options = {}) {
+    const bytes = toBytes(text);
+    const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+
+    let flat;
+    if (pattern) {
+        const patternBytes = toBytes(pattern);
+        flat = wasmChunkOffsetsPattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
+    } else {
+        flat = wasmChunkOffsets(bytes, size, delimiters, prefix);
+    }
+
+    const pairs = [];
+    for (let i = 0; i < flat.length; i += 2) {
+        pairs.push([flat[i], flat[i + 1]]);
+    }
+    return pairs;
+}
+
+let initialized = false;
+
+/**
+ * Initialize the WASM module. Must be called before using chunk functions.
+ */
+export async function init() {
+    if (!initialized) {
+        await initWasm();
+        initialized = true;
+    }
+}
+
+/**
+ * Chunker splits text at delimiter boundaries.
+ * Implements Symbol.iterator for use in for...of loops.
+ *
+ * @example
+ * // String input
+ * const chunker = new Chunker("Hello. World. Test.", { size: 10 });
+ * for (const slice of chunker) {
+ *     console.log(slice); // strings
+ * }
+ *
+ * @example
+ * // With pattern
+ * const chunker = new Chunker("Hello▁World", { pattern: "▁", prefix: true });
+ * for (const slice of chunker) {
+ *     console.log(slice);
+ * }
+ */
+export class Chunker {
+    /**
+     * Create a new Chunker.
+     * @param {string | Uint8Array} text - The text to chunk
+     * @param {Object} [options] - Options
+     * @param {number} [options.size=4096] - Target chunk size in bytes
+     * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+     * @param {string | Uint8Array} [options.pattern] - Multi-byte pattern to split on
+     * @param {boolean} [options.prefix=false] - Put delimiter/pattern at start of next chunk
+     * @param {boolean} [options.consecutive=false] - Split at START of consecutive runs
+     * @param {boolean} [options.forwardFallback=false] - Search forward if no pattern in backward window
+     */
+    constructor(text, options = {}) {
+        this._isString = typeof text === 'string';
+        const bytes = toBytes(text);
+        const { size, delimiters, pattern, prefix, consecutive, forwardFallback } = options;
+
+        if (pattern) {
+            const patternBytes = toBytes(pattern);
+            this._chunker = WasmChunker.with_pattern(bytes, size ?? 4096, patternBytes, prefix, consecutive, forwardFallback);
+        } else {
+            this._chunker = new WasmChunker(bytes, size, delimiters, prefix);
+        }
+    }
+
+    /**
+     * Get the next chunk, or undefined if exhausted.
+     * @returns {string | Uint8Array | undefined}
+     */
+    next() {
+        const chunk = this._chunker.next();
+        if (chunk === undefined) return undefined;
+        return this._isString ? decoder.decode(chunk) : chunk;
+    }
+
+    /**
+     * Reset the chunker to iterate from the beginning.
+     */
+    reset() {
+        this._chunker.reset();
+    }
+
+    /**
+     * Collect all chunk offsets as an array of [start, end] pairs.
+     * This is faster than iterating as it makes a single WASM call.
+     * @returns {Array<[number, number]>}
+     */
+    collectOffsets() {
+        const flat = this._chunker.collect_offsets();
+        const pairs = [];
+        for (let i = 0; i < flat.length; i += 2) {
+            pairs.push([flat[i], flat[i + 1]]);
+        }
+        return pairs;
+    }
+
+    /**
+     * Free the underlying WASM memory.
+     */
+    free() {
+        this._chunker.free();
+    }
+
+    /**
+     * Iterator protocol - allows use in for...of loops.
+     */
+    *[Symbol.iterator]() {
+        let chunk;
+        while ((chunk = this._chunker.next()) !== undefined) {
+            yield this._isString ? decoder.decode(chunk) : chunk;
+        }
+    }
+}

package/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@chonkiejs/chunk",
+  "version": "0.5.0",
+  "description": "The fastest semantic text chunking library",
+  "type": "module",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "pkg/"
+  ],
+  "scripts": {
+    "build": "wasm-pack build --target web",
+    "test": "node --test tests/"
+  },
+  "keywords": [
+    "chunking",
+    "text",
+    "simd",
+    "nlp",
+    "tokenization",
+    "rag",
+    "wasm",
+    "webassembly",
+    "chonkie"
+  ],
+  "author": "Bhavnick Minhas",
+  "license": "MIT OR Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/chonkie-inc/chunk"
+  }
+}