mdream 1.1.0 → 1.1.2

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "mdream",
   "type": "module",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "description": "Ultra-performant HTML to Markdown Convertor Optimized for LLMs and llm.txt artifacts.",
   "author": {
     "name": "Harlan Wilton",
@@ -46,27 +46,28 @@
     "bin",
     "dist",
     "napi/",
+    "wasm-bundler/",
     "wasm/"
   ],
   "browser": "./dist/browser.mjs",
   "dependencies": {},
   "optionalDependencies": {
-    "@mdream/rust-android-arm64": "1.1.0",
-    "@mdream/rust-darwin-arm64": "1.1.0",
-    "@mdream/rust-darwin-x64": "1.1.0",
-    "@mdream/rust-linux-arm-gnueabihf": "1.1.0",
-    "@mdream/rust-android-arm-eabi": "1.1.0",
-    "@mdream/rust-linux-arm64-gnu": "1.1.0",
-    "@mdream/rust-freebsd-x64": "1.1.0",
-    "@mdream/rust-linux-arm64-musl": "1.1.0",
-    "@mdream/rust-linux-x64-gnu": "1.1.0",
-    "@mdream/rust-linux-x64-musl": "1.1.0",
-    "@mdream/rust-wasm32-wasi": "1.1.0",
-    "@mdream/rust-win32-x64-msvc": "1.1.0",
-    "@mdream/rust-win32-arm64-msvc": "1.1.0"
+    "@mdream/rust-android-arm-eabi": "1.1.2",
+    "@mdream/rust-darwin-arm64": "1.1.2",
+    "@mdream/rust-darwin-x64": "1.1.2",
+    "@mdream/rust-freebsd-x64": "1.1.2",
+    "@mdream/rust-android-arm64": "1.1.2",
+    "@mdream/rust-linux-arm64-gnu": "1.1.2",
+    "@mdream/rust-linux-arm-gnueabihf": "1.1.2",
+    "@mdream/rust-linux-arm64-musl": "1.1.2",
+    "@mdream/rust-linux-x64-musl": "1.1.2",
+    "@mdream/rust-linux-x64-gnu": "1.1.2",
+    "@mdream/rust-wasm32-wasi": "1.1.2",
+    "@mdream/rust-win32-arm64-msvc": "1.1.2",
+    "@mdream/rust-win32-x64-msvc": "1.1.2"
   },
   "devDependencies": {
-    "@mdream/js": "1.1.0"
+    "@mdream/js": "1.1.2"
   },
   "scripts": {
     "flame": "pnpm build && unbuild bench/bundle && clinic flame -- node bench/bundle/dist/string.mjs 10",
@@ -83,7 +84,7 @@
     "test:wiki-small:file": "cat test/fixtures/wikipedia-small.html | node ./bin/mdream.mjs --origin https://en.wikipedia.org | tee test/wiki-markdown.md",
     "build": "obuild",
     "build:native": "cd ../../crates/node && napi build --platform --release",
-    "build:edge": "cd ../../crates/edge && wasm-pack build --target web --out-dir ../../packages/mdream/wasm --out-name mdream_edge && wasm-opt -Oz --enable-bulk-memory --enable-nontrapping-float-to-int --strip-producers ../../packages/mdream/wasm/mdream_edge_bg.wasm -o ../../packages/mdream/wasm/mdream_edge_bg.wasm",
+    "build:edge": "cd ../../crates/edge && wasm-pack build --target web --out-dir ../../packages/mdream/wasm --out-name mdream_edge && wasm-opt -Oz --enable-bulk-memory --enable-nontrapping-float-to-int --strip-producers ../../packages/mdream/wasm/mdream_edge_bg.wasm -o ../../packages/mdream/wasm/mdream_edge_bg.wasm && wasm-pack build --target bundler --out-dir ../../packages/mdream/wasm-bundler --out-name mdream_edge && wasm-opt -Oz --enable-bulk-memory --enable-nontrapping-float-to-int --strip-producers ../../packages/mdream/wasm-bundler/mdream_edge_bg.wasm -o ../../packages/mdream/wasm-bundler/mdream_edge_bg.wasm",
     "typecheck": "tsc --noEmit",
     "dev:prepare": "obuild --stub",
     "test": "vitest test",

package/wasm/package.json

@@ -2,7 +2,7 @@
   "name": "mdream-edge",
   "type": "module",
   "description": "WebAssembly edge runtime bindings for mdream HTML-to-Markdown converter",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "license": "MIT",
   "files": [
     "mdream_edge_bg.wasm",
@@ -14,4 +14,4 @@
   "sideEffects": [
     "./snippets/*"
   ]
-}
+}

package/wasm-bundler/mdream_edge.d.ts

@@ -0,0 +1,14 @@
+/* tslint:disable */
+/* eslint-disable */
+
+export class MarkdownStream {
+    free(): void;
+    [Symbol.dispose](): void;
+    finish(): string;
+    constructor(options: any);
+    processChunk(chunk: string): string;
+}
+
+export function htmlToMarkdown(html: string, options: any): string;
+
+export function htmlToMarkdownResult(html: string, options: any): any;

package/wasm-bundler/mdream_edge.js

@@ -0,0 +1,9 @@
+/* @ts-self-types="./mdream_edge.d.ts" */
+import * as wasm from "./mdream_edge_bg.wasm";
+import { __wbg_set_wasm } from "./mdream_edge_bg.js";
+
+__wbg_set_wasm(wasm);
+
+export {
+    MarkdownStream, htmlToMarkdown, htmlToMarkdownResult
+} from "./mdream_edge_bg.js";

package/wasm-bundler/mdream_edge_bg.js

@@ -0,0 +1,317 @@
+export class MarkdownStream {
+    __destroy_into_raw() {
+        const ptr = this.__wbg_ptr;
+        this.__wbg_ptr = 0;
+        MarkdownStreamFinalization.unregister(this);
+        return ptr;
+    }
+    free() {
+        const ptr = this.__destroy_into_raw();
+        wasm.__wbg_markdownstream_free(ptr, 0);
+    }
+    /**
+     * @returns {string}
+     */
+    finish() {
+        let deferred1_0;
+        let deferred1_1;
+        try {
+            const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+            wasm.markdownstream_finish(retptr, this.__wbg_ptr);
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            deferred1_0 = r0;
+            deferred1_1 = r1;
+            return getStringFromWasm0(r0, r1);
+        } finally {
+            wasm.__wbindgen_add_to_stack_pointer(16);
+            wasm.__wbindgen_export4(deferred1_0, deferred1_1, 1);
+        }
+    }
+    /**
+     * @param {any} options
+     */
+    constructor(options) {
+        const ret = wasm.markdownstream_new(addHeapObject(options));
+        this.__wbg_ptr = ret >>> 0;
+        MarkdownStreamFinalization.register(this, this.__wbg_ptr, this);
+        return this;
+    }
+    /**
+     * @param {string} chunk
+     * @returns {string}
+     */
+    processChunk(chunk) {
+        let deferred2_0;
+        let deferred2_1;
+        try {
+            const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+            const ptr0 = passStringToWasm0(chunk, wasm.__wbindgen_export, wasm.__wbindgen_export2);
+            const len0 = WASM_VECTOR_LEN;
+            wasm.markdownstream_processChunk(retptr, this.__wbg_ptr, ptr0, len0);
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            deferred2_0 = r0;
+            deferred2_1 = r1;
+            return getStringFromWasm0(r0, r1);
+        } finally {
+            wasm.__wbindgen_add_to_stack_pointer(16);
+            wasm.__wbindgen_export4(deferred2_0, deferred2_1, 1);
+        }
+    }
+}
+if (Symbol.dispose) MarkdownStream.prototype[Symbol.dispose] = MarkdownStream.prototype.free;
+
+/**
+ * @param {string} html
+ * @param {any} options
+ * @returns {string}
+ */
+export function htmlToMarkdown(html, options) {
+    let deferred2_0;
+    let deferred2_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.htmlToMarkdown(retptr, ptr0, len0, addHeapObject(options));
+        var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+        var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+        deferred2_0 = r0;
+        deferred2_1 = r1;
+        return getStringFromWasm0(r0, r1);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_export4(deferred2_0, deferred2_1, 1);
+    }
+}
+
+/**
+ * @param {string} html
+ * @param {any} options
+ * @returns {any}
+ */
+export function htmlToMarkdownResult(html, options) {
+    const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2);
+    const len0 = WASM_VECTOR_LEN;
+    const ret = wasm.htmlToMarkdownResult(ptr0, len0, addHeapObject(options));
+    return takeObject(ret);
+}
+export function __wbg___wbindgen_boolean_get_6ea149f0a8dcc5ff(arg0) {
+    const v = getObject(arg0);
+    const ret = typeof(v) === 'boolean' ? v : undefined;
+    return isLikeNone(ret) ? 0xFFFFFF : ret ? 1 : 0;
+}
+export function __wbg___wbindgen_is_null_52ff4ec04186736f(arg0) {
+    const ret = getObject(arg0) === null;
+    return ret;
+}
+export function __wbg___wbindgen_is_undefined_29a43b4d42920abd(arg0) {
+    const ret = getObject(arg0) === undefined;
+    return ret;
+}
+export function __wbg___wbindgen_number_get_c7f42aed0525c451(arg0, arg1) {
+    const obj = getObject(arg1);
+    const ret = typeof(obj) === 'number' ? obj : undefined;
+    getDataViewMemory0().setFloat64(arg0 + 8 * 1, isLikeNone(ret) ? 0 : ret, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, !isLikeNone(ret), true);
+}
+export function __wbg___wbindgen_string_get_7ed5322991caaec5(arg0, arg1) {
+    const obj = getObject(arg1);
+    const ret = typeof(obj) === 'string' ? obj : undefined;
+    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export, wasm.__wbindgen_export2);
+    var len1 = WASM_VECTOR_LEN;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+}
+export function __wbg___wbindgen_throw_6b64449b9b9ed33c(arg0, arg1) {
+    throw new Error(getStringFromWasm0(arg0, arg1));
+}
+export function __wbg_entries_e0b73aa8571ddb56(arg0) {
+    const ret = Object.entries(getObject(arg0));
+    return addHeapObject(ret);
+}
+export function __wbg_from_0dbf29f09e7fb200(arg0) {
+    const ret = Array.from(getObject(arg0));
+    return addHeapObject(ret);
+}
+export function __wbg_get_6011fa3a58f61074() { return handleError(function (arg0, arg1) {
+    const ret = Reflect.get(getObject(arg0), getObject(arg1));
+    return addHeapObject(ret);
+}, arguments); }
+export function __wbg_get_8360291721e2339f(arg0, arg1) {
+    const ret = getObject(arg0)[arg1 >>> 0];
+    return addHeapObject(ret);
+}
+export function __wbg_isArray_c3109d14ffc06469(arg0) {
+    const ret = Array.isArray(getObject(arg0));
+    return ret;
+}
+export function __wbg_length_3d4ecd04bd8d22f1(arg0) {
+    const ret = getObject(arg0).length;
+    return ret;
+}
+export function __wbg_new_682678e2f47e32bc() {
+    const ret = new Array();
+    return addHeapObject(ret);
+}
+export function __wbg_new_aa8d0fa9762c29bd() {
+    const ret = new Object();
+    return addHeapObject(ret);
+}
+export function __wbg_push_471a5b068a5295f6(arg0, arg1) {
+    const ret = getObject(arg0).push(getObject(arg1));
+    return ret;
+}
+export function __wbg_set_022bee52d0b05b19() { return handleError(function (arg0, arg1, arg2) {
+    const ret = Reflect.set(getObject(arg0), getObject(arg1), getObject(arg2));
+    return ret;
+}, arguments); }
+export function __wbindgen_cast_0000000000000001(arg0, arg1) {
+    // Cast intrinsic for `Ref(String) -> Externref`.
+    const ret = getStringFromWasm0(arg0, arg1);
+    return addHeapObject(ret);
+}
+export function __wbindgen_object_clone_ref(arg0) {
+    const ret = getObject(arg0);
+    return addHeapObject(ret);
+}
+export function __wbindgen_object_drop_ref(arg0) {
+    takeObject(arg0);
+}
+const MarkdownStreamFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_markdownstream_free(ptr >>> 0, 1));
+
+function addHeapObject(obj) {
+    if (heap_next === heap.length) heap.push(heap.length + 1);
+    const idx = heap_next;
+    heap_next = heap[idx];
+
+    heap[idx] = obj;
+    return idx;
+}
+
+function dropObject(idx) {
+    if (idx < 1028) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+let cachedDataViewMemory0 = null;
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
+    }
+    return cachedDataViewMemory0;
+}
+
+function getStringFromWasm0(ptr, len) {
+    ptr = ptr >>> 0;
+    return decodeText(ptr, len);
+}
+
+let cachedUint8ArrayMemory0 = null;
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
+    }
+    return cachedUint8ArrayMemory0;
+}
+
+function getObject(idx) { return heap[idx]; }
+
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        wasm.__wbindgen_export3(addHeapObject(e));
+    }
+}
+
+let heap = new Array(1024).fill(undefined);
+heap.push(undefined, null, true, false);
+
+let heap_next = heap.length;
+
+function isLikeNone(x) {
+    return x === undefined || x === null;
+}
+
+function passStringToWasm0(arg, malloc, realloc) {
+    if (realloc === undefined) {
+        const buf = cachedTextEncoder.encode(arg);
+        const ptr = malloc(buf.length, 1) >>> 0;
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
+        WASM_VECTOR_LEN = buf.length;
+        return ptr;
+    }
+
+    let len = arg.length;
+    let ptr = malloc(len, 1) >>> 0;
+
+    const mem = getUint8ArrayMemory0();
+
+    let offset = 0;
+
+    for (; offset < len; offset++) {
+        const code = arg.charCodeAt(offset);
+        if (code > 0x7F) break;
+        mem[ptr + offset] = code;
+    }
+    if (offset !== len) {
+        if (offset !== 0) {
+            arg = arg.slice(offset);
+        }
+        ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
+        const ret = cachedTextEncoder.encodeInto(arg, view);
+
+        offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
+    }
+
+    WASM_VECTOR_LEN = offset;
+    return ptr;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
+let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+cachedTextDecoder.decode();
+const MAX_SAFARI_DECODE_BYTES = 2146435072;
+let numBytesDecoded = 0;
+function decodeText(ptr, len) {
+    numBytesDecoded += len;
+    if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) {
+        cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true });
+        cachedTextDecoder.decode();
+        numBytesDecoded = len;
+    }
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
+}
+
+const cachedTextEncoder = new TextEncoder();
+
+if (!('encodeInto' in cachedTextEncoder)) {
+    cachedTextEncoder.encodeInto = function (arg, view) {
+        const buf = cachedTextEncoder.encode(arg);
+        view.set(buf);
+        return {
+            read: arg.length,
+            written: buf.length
+        };
+    };
+}
+
+let WASM_VECTOR_LEN = 0;
+
+
+let wasm;
+export function __wbg_set_wasm(val) {
+    wasm = val;
+}

package/wasm-bundler/mdream_edge_bg.wasm.d.ts

@@ -0,0 +1,14 @@
+/* tslint:disable */
+/* eslint-disable */
+export const memory: WebAssembly.Memory;
+export const __wbg_markdownstream_free: (a: number, b: number) => void;
+export const htmlToMarkdown: (a: number, b: number, c: number, d: number) => void;
+export const htmlToMarkdownResult: (a: number, b: number, c: number) => number;
+export const markdownstream_finish: (a: number, b: number) => void;
+export const markdownstream_new: (a: number) => number;
+export const markdownstream_processChunk: (a: number, b: number, c: number, d: number) => void;
+export const __wbindgen_export: (a: number, b: number) => number;
+export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number;
+export const __wbindgen_export3: (a: number) => void;
+export const __wbindgen_add_to_stack_pointer: (a: number) => number;
+export const __wbindgen_export4: (a: number, b: number, c: number) => void;

package/wasm-bundler/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "mdream-edge",
+  "type": "module",
+  "description": "WebAssembly edge runtime bindings for mdream HTML-to-Markdown converter",
+  "version": "1.1.2",
+  "license": "MIT",
+  "files": [
+    "mdream_edge_bg.wasm",
+    "mdream_edge.js",
+    "mdream_edge_bg.js",
+    "mdream_edge.d.ts"
+  ],
+  "main": "mdream_edge.js",
+  "types": "mdream_edge.d.ts",
+  "sideEffects": [
+    "./mdream_edge.js",
+    "./snippets/*"
+  ]
+}