memchunk 0.1.0

package/README.md

@@ -0,0 +1,83 @@
+<p align="center">
+  <img src="../../assets/memchunk_wide.png" alt="memchunk" width="500">
+</p>
+
+<h1 align="center">memchunk</h1>
+
+<p align="center">
+  <em>the fastest text chunking library — up to 1 TB/s throughput</em>
+</p>
+
+<p align="center">
+  <a href="https://www.npmjs.com/package/memchunk"><img src="https://img.shields.io/npm/v/memchunk.svg" alt="npm"></a>
+  <a href="https://github.com/chonkie-inc/memchunk"><img src="https://img.shields.io/badge/github-memchunk-blue" alt="GitHub"></a>
+  <a href="LICENSE-MIT"><img src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg" alt="License"></a>
+</p>
+
+---
+
+you know how every chunking library claims to be fast? yeah, we actually meant it.
+
+**memchunk** splits text at semantic boundaries (periods, newlines, the usual suspects) and does it stupid fast. we're talking "chunk the entire english wikipedia in 120ms" fast.
+
+want to know how? [read the blog post](https://minha.sh/posts/so-you-want-to-chunk-really-fast) where we nerd out about SIMD instructions and lookup tables.
+
+## installation
+
+```bash
+npm install memchunk
+```
+
+## usage
+
+```javascript
+import { init, chunk } from 'memchunk';
+
+// initialize wasm (required once)
+await init();
+
+const text = new TextEncoder().encode("Hello world. How are you? I'm fine.\nThanks for asking.");
+
+// with defaults (4KB chunks, split at \n . ?)
+for (const slice of chunk(text)) {
+    console.log(new TextDecoder().decode(slice));
+}
+
+// with custom size
+for (const slice of chunk(text, { size: 1024 })) {
+    console.log(new TextDecoder().decode(slice));
+}
+
+// with custom delimiters
+for (const slice of chunk(text, { delimiters: ".?!\n" })) {
+    console.log(new TextDecoder().decode(slice));
+}
+
+// with both
+for (const slice of chunk(text, { size: 8192, delimiters: "\n" })) {
+    console.log(new TextDecoder().decode(slice));
+}
+
+// collect all chunks
+const chunks = [...chunk(text)];
+```
+
+chunks are returned as `Uint8Array` subarrays (zero-copy views of the original text).
+
+## citation
+
+if you use memchunk in your research, please cite it as follows:
+
+```bibtex
+@software{memchunk2025,
+  author = {Minhas, Bhavnick},
+  title = {memchunk: The fastest text chunking library},
+  year = {2025},
+  publisher = {GitHub},
+  howpublished = {\url{https://github.com/chonkie-inc/memchunk}},
+}
+```
+
+## license
+
+licensed under either of [Apache License, Version 2.0](LICENSE-APACHE) or [MIT license](LICENSE-MIT) at your option.

package/index.js

@@ -0,0 +1,144 @@
+/**
+ * memchunk - The fastest semantic text chunking library
+ *
+ * @example
+ * ```javascript
+ * import { init, chunk } from 'memchunk';
+ *
+ * await init();
+ *
+ * const text = new TextEncoder().encode("Hello. World. Test.");
+ * for (const slice of chunk(text, { size: 10, delimiters: "." })) {
+ *     console.log(new TextDecoder().decode(slice));
+ * }
+ * ```
+ */
+
+import initWasm, {
+    Chunker as WasmChunker,
+    default_target_size,
+    default_delimiters,
+    chunk_offsets as wasmChunkOffsets,
+} from './pkg/memchunk_wasm.js';
+
+export { default_target_size, default_delimiters };
+
+/**
+ * Split text into chunks at delimiter boundaries.
+ * Returns an iterator of zero-copy Uint8Array subarray views.
+ *
+ * @param {Uint8Array} text - The text to chunk as bytes
+ * @param {Object} [options] - Options
+ * @param {number} [options.size=4096] - Target chunk size in bytes
+ * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+ * @yields {Uint8Array} Zero-copy subarray views of the original text
+ *
+ * @example
+ * const text = new TextEncoder().encode("Hello. World. Test.");
+ * for (const slice of chunk(text, { size: 10, delimiters: "." })) {
+ *     console.log(new TextDecoder().decode(slice));
+ * }
+ */
+export function* chunk(text, options = {}) {
+    const { size, delimiters } = options;
+    const flat = wasmChunkOffsets(text, size, delimiters);
+    for (let i = 0; i < flat.length; i += 2) {
+        yield text.subarray(flat[i], flat[i + 1]);
+    }
+}
+
+/**
+ * Get chunk offsets without creating views.
+ * Returns an array of [start, end] offset pairs.
+ *
+ * @param {Uint8Array} text - The text to chunk as bytes
+ * @param {Object} [options] - Options
+ * @param {number} [options.size=4096] - Target chunk size in bytes
+ * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+ * @returns {Array<[number, number]>} Array of [start, end] offset pairs
+ */
+export function chunk_offsets(text, options = {}) {
+    const { size, delimiters } = options;
+    const flat = wasmChunkOffsets(text, size, delimiters);
+    const pairs = [];
+    for (let i = 0; i < flat.length; i += 2) {
+        pairs.push([flat[i], flat[i + 1]]);
+    }
+    return pairs;
+}
+
+let initialized = false;
+
+/**
+ * Initialize the WASM module. Must be called before using chunk functions.
+ */
+export async function init() {
+    if (!initialized) {
+        await initWasm();
+        initialized = true;
+    }
+}
+
+/**
+ * Chunker splits text at delimiter boundaries.
+ * Implements Symbol.iterator for use in for...of loops.
+ */
+export class Chunker {
+    /**
+     * Create a new Chunker.
+     * @param {Uint8Array} text - The text to chunk as bytes
+     * @param {Object} [options] - Options
+     * @param {number} [options.size=4096] - Target chunk size in bytes
+     * @param {string} [options.delimiters="\n.?"] - Delimiter characters
+     */
+    constructor(text, options = {}) {
+        const { size, delimiters } = options;
+        this._chunker = new WasmChunker(text, size, delimiters);
+    }
+
+    /**
+     * Get the next chunk, or undefined if exhausted.
+     * @returns {Uint8Array | undefined}
+     */
+    next() {
+        return this._chunker.next();
+    }
+
+    /**
+     * Reset the chunker to iterate from the beginning.
+     */
+    reset() {
+        this._chunker.reset();
+    }
+
+    /**
+     * Collect all chunk offsets as an array of [start, end] pairs.
+     * This is faster than iterating as it makes a single WASM call.
+     * @returns {Array<[number, number]>}
+     */
+    collectOffsets() {
+        const flat = this._chunker.collect_offsets();
+        const pairs = [];
+        for (let i = 0; i < flat.length; i += 2) {
+            pairs.push([flat[i], flat[i + 1]]);
+        }
+        return pairs;
+    }
+
+    /**
+     * Free the underlying WASM memory.
+     */
+    free() {
+        this._chunker.free();
+    }
+
+    /**
+     * Iterator protocol - allows use in for...of loops.
+     */
+    *[Symbol.iterator]() {
+        let chunk;
+        while ((chunk = this._chunker.next()) !== undefined) {
+            yield chunk;
+        }
+    }
+}

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "memchunk",
+  "version": "0.1.0",
+  "description": "The fastest semantic text chunking library",
+  "type": "module",
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "pkg/"
+  ],
+  "scripts": {
+    "build": "wasm-pack build --target web",
+    "test": "node --test tests/"
+  },
+  "keywords": [
+    "chunking",
+    "text",
+    "simd",
+    "nlp",
+    "tokenization",
+    "rag",
+    "wasm",
+    "webassembly"
+  ],
+  "author": "Bhavnick Minhas",
+  "license": "MIT OR Apache-2.0",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/chonkie-inc/memchunk"
+  }
+}