@pipedog/sentencex-wasm 0.0.0 → 0.2.0

package/dist/api.d.ts

@@ -0,0 +1,11 @@
+/** The public functions, shared by both entries; each entry supplies its own module loader. */
+import { type SentenceBoundary, SentencexEngine } from "./engine.ts";
+export interface SentencexApi {
+    segment(language: string, text: string): string[];
+    getSentenceBoundaries(language: string, text: string): SentenceBoundary[];
+    initSentencex(source?: WebAssembly.Module | BufferSource): SentencexEngine;
+    getSentencex(): SentencexEngine;
+}
+/** @internal */
+export declare function createApi(loadDefault: () => WebAssembly.Module): SentencexApi;
+//# sourceMappingURL=api.d.ts.map

package/dist/api.js

@@ -0,0 +1,24 @@
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+/** The public functions, shared by both entries; each entry supplies its own module loader. */
+import { compileSync, SentencexEngine } from "./engine.js";
+/** @internal */
+export function createApi(loadDefault) {
+    let engine = null;
+    const getSentencex = () => {
+        engine ??= new SentencexEngine(loadDefault());
+        return engine;
+    };
+    return {
+        segment: (language, text) => getSentencex().segment(language, text),
+        getSentenceBoundaries: (language, text) => getSentencex().getSentenceBoundaries(language, text),
+        initSentencex(source) {
+            if (source === undefined)
+                return getSentencex();
+            engine = new SentencexEngine(compileSync(source));
+            return engine;
+        },
+        getSentencex,
+    };
+}
+//# sourceMappingURL=api.js.map

package/dist/engine.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Hand-written wrapper over the sentencex WebAssembly exports.
+ *
+ * The module has no imports and a tiny C ABI (see `rust/src/lib.rs`), so there
+ * is no generated glue: we copy the UTF-8 language code and text into linear
+ * memory, call `sx_segment` / `sx_boundaries`, and read back u32 records of
+ * UTF-16 offsets that index the caller's original string. Everything here is
+ * synchronous.
+ */
+/** One sentence boundary, as in the `sentencex` npm package's typings. */
+export interface SentenceBoundary {
+    /** UTF-16 index (JavaScript string index) where the sentence starts. */
+    start_index: number;
+    /** UTF-16 index (JavaScript string index) where the sentence ends (exclusive). */
+    end_index: number;
+    /** The sentence text. */
+    text: string;
+    /** Punctuation mark that ended the sentence, or null if none. */
+    boundary_symbol: string | null;
+    /** Whether this boundary represents a paragraph break ("\n\n"). */
+    is_paragraph_break: boolean;
+}
+/**
+ * One instantiated sentencex module. All methods are synchronous.
+ *
+ * If the WASM code traps (a Rust panic), the instance is discarded and a fresh
+ * one is created from the same module on the next call.
+ */
+export declare class SentencexEngine {
+    #private;
+    readonly module: WebAssembly.Module;
+    constructor(module: WebAssembly.Module);
+    /** Current size of the instance's linear memory, in bytes (it grows, never shrinks). */
+    get memoryBytes(): number;
+    /** Split `text` into sentences. Concatenating the result gives back `text`. */
+    segment(language: string, text: string): string[];
+    /** Detailed sentence boundaries, with JavaScript (UTF-16) string indices. */
+    getSentenceBoundaries(language: string, text: string): SentenceBoundary[];
+}
+/** Compile WASM bytes synchronously (not allowed on Cloudflare Workers). */
+export declare function compileSync(source: WebAssembly.Module | BufferSource): WebAssembly.Module;
+//# sourceMappingURL=engine.d.ts.map

package/dist/engine.js

@@ -0,0 +1,105 @@
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+const BOUNDARY_FIELDS = 7;
+const NONE = 0xffffffff;
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+function instantiate(module) {
+    const x = new WebAssembly.Instance(module, {}).exports;
+    if (typeof x.sx_segment !== "function") {
+        throw new Error("not the @pipedog/sentencex-wasm module");
+    }
+    return x;
+}
+/**
+ * One instantiated sentencex module. All methods are synchronous.
+ *
+ * If the WASM code traps (a Rust panic), the instance is discarded and a fresh
+ * one is created from the same module on the next call.
+ */
+export class SentencexEngine {
+    module;
+    #x;
+    constructor(module) {
+        this.module = module;
+        this.#x = instantiate(module);
+    }
+    /** Current size of the instance's linear memory, in bytes (it grows, never shrinks). */
+    get memoryBytes() {
+        return this.#x.memory.buffer.byteLength;
+    }
+    /** Split `text` into sentences. Concatenating the result gives back `text`. */
+    segment(language, text) {
+        return this.#call(language, text, false, (x, n) => {
+            const rec = new Uint32Array(x.memory.buffer, x.sx_out(), n * 2);
+            const out = new Array(n);
+            for (let i = 0; i < n; i++)
+                out[i] = text.slice(rec[2 * i], rec[2 * i + 1]);
+            return out;
+        });
+    }
+    /** Detailed sentence boundaries, with JavaScript (UTF-16) string indices. */
+    getSentenceBoundaries(language, text) {
+        return this.#call(language, text, true, (x, n) => {
+            const rec = new Uint32Array(x.memory.buffer, x.sx_out(), n * BOUNDARY_FIELDS);
+            const bytes = new Uint8Array(x.memory.buffer);
+            const out = new Array(n);
+            for (let i = 0; i < n; i++) {
+                const r = i * BOUNDARY_FIELDS;
+                const symLen = rec[r + 5];
+                out[i] = {
+                    start_index: rec[r],
+                    end_index: rec[r + 1],
+                    text: text.slice(rec[r + 2], rec[r + 3]),
+                    boundary_symbol: symLen === NONE
+                        ? null
+                        : decoder.decode(bytes.subarray(rec[r + 4], rec[r + 4] + symLen)),
+                    is_paragraph_break: rec[r + 6] !== 0,
+                };
+            }
+            return out;
+        });
+    }
+    #call(language, text, boundaries, read) {
+        if (typeof language !== "string" || typeof text !== "string") {
+            throw new TypeError("language and text must be strings");
+        }
+        const x = this.#x;
+        // UTF-8 needs at most 3 bytes per UTF-16 code unit.
+        const langCap = language.length * 3;
+        const cap = langCap + text.length * 3;
+        let ptr = 0;
+        try {
+            ptr = x.sx_alloc(cap);
+            // Views are created after the alloc: a memory.grow detaches the old buffer.
+            const mem = new Uint8Array(x.memory.buffer);
+            const lang = encoder.encodeInto(language, mem.subarray(ptr, ptr + langCap)).written;
+            const textPtr = ptr + langCap;
+            const textLen = encoder.encodeInto(text, mem.subarray(textPtr, ptr + cap)).written;
+            const n = boundaries
+                ? x.sx_boundaries(ptr, lang, textPtr, textLen)
+                : x.sx_segment(ptr, lang, textPtr, textLen);
+            if (n < 0)
+                throw new Error(`sentencex failed (code ${n})`);
+            // Read before freeing: boundary symbols point into the input buffer.
+            const result = read(x, n);
+            x.sx_free(ptr, cap);
+            return result;
+        }
+        catch (e) {
+            if (e instanceof WebAssembly.RuntimeError) {
+                // A trap leaves the instance in an unknown state; start over next time.
+                this.#x = instantiate(this.module);
+            }
+            else if (ptr !== 0) {
+                x.sx_free(ptr, cap);
+            }
+            throw e;
+        }
+    }
+}
+/** Compile WASM bytes synchronously (not allowed on Cloudflare Workers). */
+export function compileSync(source) {
+    return source instanceof WebAssembly.Module ? source : new WebAssembly.Module(source);
+}
+//# sourceMappingURL=engine.js.map

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+export type { SentencexApi } from "./api.ts";
+export { type SentenceBoundary, SentencexEngine } from "./engine.ts";
+/** Split `text` into sentences (sentencex rules for `language`, e.g. "en"). */
+export declare const segment: (language: string, text: string) => string[];
+/** Detailed sentence boundaries with JavaScript (UTF-16) string indices. */
+export declare const getSentenceBoundaries: (language: string, text: string) => import("./engine.ts").SentenceBoundary[];
+/** Alias with the `sentencex` npm package's name. */
+export declare const get_sentence_boundaries: (language: string, text: string) => import("./engine.ts").SentenceBoundary[];
+/**
+ * Compile and instantiate now (otherwise done on first use). Passing a
+ * `WebAssembly.Module` or bytes replaces the bundled module.
+ */
+export declare const initSentencex: (source?: WebAssembly.Module | BufferSource) => import("./engine.ts").SentencexEngine;
+/** The shared engine, instantiated on first use. */
+export declare const getSentencex: () => import("./engine.ts").SentencexEngine;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -0,0 +1,49 @@
+// Copyright (c) 2026, pipedog contributors
+// SPDX-License-Identifier: BSD-2-Clause
+/**
+ * Default entry (Bun, Node >= 22, Deno, and bundlers).
+ *
+ * Fully synchronous: the WASM bytes are embedded in `wasm_bytes.ts` (base64)
+ * and compiled with `new WebAssembly.Module()` on the first call.
+ *
+ * @example
+ * ```ts
+ * import { segment } from "@pipedog/sentencex-wasm";
+ * segment("en", "Dr. Smith arrived at 5 p.m. He was late.");
+ * // ["Dr. Smith arrived at 5 p.m. ", "He was late."]
+ * ```
+ *
+ * On Cloudflare Workers the package resolves to `workerd.ts` instead (the
+ * "workerd" export condition), which imports the .wasm as a module because
+ * workerd cannot compile WASM from bytes.
+ */
+import { createApi } from "./api.js";
+import { compileSync } from "./engine.js";
+import { WASM_BASE64 } from "./wasm_bytes.js";
+export { SentencexEngine } from "./engine.js";
+function decodeBase64(b64) {
+    const fromBase64 = Uint8Array
+        .fromBase64;
+    if (typeof fromBase64 === "function")
+        return fromBase64(b64);
+    const bin = atob(b64);
+    const out = new Uint8Array(bin.length);
+    for (let i = 0; i < bin.length; i++)
+        out[i] = bin.charCodeAt(i);
+    return out;
+}
+const api = createApi(() => compileSync(decodeBase64(WASM_BASE64)));
+/** Split `text` into sentences (sentencex rules for `language`, e.g. "en"). */
+export const segment = api.segment;
+/** Detailed sentence boundaries with JavaScript (UTF-16) string indices. */
+export const getSentenceBoundaries = api.getSentenceBoundaries;
+/** Alias with the `sentencex` npm package's name. */
+export const get_sentence_boundaries = api.getSentenceBoundaries;
+/**
+ * Compile and instantiate now (otherwise done on first use). Passing a
+ * `WebAssembly.Module` or bytes replaces the bundled module.
+ */
+export const initSentencex = api.initSentencex;
+/** The shared engine, instantiated on first use. */
+export const getSentencex = api.getSentencex;
+//# sourceMappingURL=index.js.map