@oh-my-pi/pi-natives 8.12.10 → 9.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@oh-my-pi/pi-natives",
-	"version": "8.12.10",
+	"version": "9.0.0",
 	"description": "Native Rust functionality compiled to WebAssembly via wasm-bindgen",
 	"type": "module",
 	"main": "./src/index.ts",
@@ -30,7 +30,7 @@
 		"directory": "packages/natives"
 	},
 	"dependencies": {
-		"@oh-my-pi/pi-utils": "8.12.10"
+		"@oh-my-pi/pi-utils": "9.0.0"
 	},
 	"devDependencies": {
 		"@types/node": "^25.0.10"

package/src/html/index.ts

@@ -0,0 +1,47 @@
+/**
+ * HTML to Markdown conversion powered by WASM.
+ *
+ * Conversion happens in a worker thread to avoid blocking the main thread.
+ */
+
+import { type RequestOptions, WorkerPool } from "../pool";
+import type { HtmlRequest, HtmlResponse, HtmlToMarkdownOptions } from "./types";
+
+export type { HtmlToMarkdownOptions } from "./types";
+
+const pool = new WorkerPool<HtmlRequest, HtmlResponse>({
+	workerUrl: new URL("./worker.ts", import.meta.url).href,
+	maxWorkers: 2,
+	idleTimeoutMs: 30_000,
+});
+
+/**
+ * Convert HTML to Markdown.
+ *
+ * @param html - HTML content to convert
+ * @param options - Conversion options
+ * @returns Markdown text
+ */
+export async function htmlToMarkdown(
+	html: string,
+	options?: HtmlToMarkdownOptions,
+	req?: RequestOptions,
+): Promise<string> {
+	const response = await pool.request<Extract<HtmlResponse, { type: "converted" }>>(
+		{
+			type: "convert",
+			html,
+			options,
+		},
+		req,
+	);
+	return response.markdown;
+}
+
+/**
+ * Terminate the HTML worker pool.
+ * Call this when shutting down to clean up resources.
+ */
+export function terminate(): void {
+	pool.terminate();
+}

package/src/html/types.ts

@@ -0,0 +1,25 @@
+/**
+ * Types for HTML to Markdown worker communication.
+ */
+
+export interface HtmlToMarkdownOptions {
+	/** Remove navigation elements, forms, headers, footers */
+	cleanContent?: boolean;
+	/** Skip images during conversion */
+	skipImages?: boolean;
+}
+
+export type HtmlRequest =
+	| { type: "init"; id: number }
+	| { type: "destroy" }
+	| {
+			type: "convert";
+			id: number;
+			html: string;
+			options?: HtmlToMarkdownOptions;
+	  };
+
+export type HtmlResponse =
+	| { type: "ready"; id: number }
+	| { type: "error"; id: number; error: string }
+	| { type: "converted"; id: number; markdown: string };

package/src/html/worker.ts

@@ -0,0 +1,40 @@
+/**
+ * Worker for HTML to Markdown conversion.
+ * Uses WASM for actual conversion.
+ */
+
+import { html_to_markdown } from "../../wasm/pi_natives";
+import type { HtmlRequest, HtmlResponse } from "./types";
+
+declare const self: Worker;
+
+function respond(msg: HtmlResponse): void {
+	self.postMessage(msg);
+}
+
+self.addEventListener("message", (e: MessageEvent<HtmlRequest>) => {
+	const msg = e.data;
+
+	switch (msg.type) {
+		case "init":
+			respond({ type: "ready", id: msg.id });
+			break;
+
+		case "destroy":
+			break;
+
+		case "convert": {
+			try {
+				const markdown = html_to_markdown(msg.html, msg.options);
+				respond({ type: "converted", id: msg.id, markdown });
+			} catch (err) {
+				respond({
+					type: "error",
+					id: msg.id,
+					error: err instanceof Error ? err.message : String(err),
+				});
+			}
+			break;
+		}
+	}
+});

package/src/image/index.ts

@@ -49,9 +49,12 @@ export class PhotonImage {
 	 * The bytes are transferred to the worker (zero-copy).
 	 */
 	static async new_from_byteslice(bytes: Uint8Array): Promise<PhotonImage> {
-		const response = await pool.request<Extract<ImageResponse, { type: "loaded" }>>({ type: "load", bytes }, [
-			bytes.buffer,
-		]);
+		const response = await pool.request<Extract<ImageResponse, { type: "loaded" }>>(
+			{ type: "load", bytes },
+			{
+				transfer: [bytes.buffer],
+			},
+		);
 		return new PhotonImage(response.handle, response.width, response.height);
 	}
 

package/src/index.ts

@@ -157,8 +157,18 @@ export {
 	supportsLanguage,
 } from "./highlight/index";
 
+// =============================================================================
+// HTML to Markdown
+// =============================================================================
+
+export {
+	type HtmlToMarkdownOptions,
+	htmlToMarkdown,
+	terminate as terminateHtmlWorker,
+} from "./html/index";
+
 // =============================================================================
 // Worker Pool (shared infrastructure)
 // =============================================================================
 
-export { type BaseRequest, type BaseResponse, WorkerPool, type WorkerPoolOptions } from "./pool";
+export { type BaseRequest, type BaseResponse, type RequestOptions, WorkerPool, type WorkerPoolOptions } from "./pool";

package/src/pool.ts

@@ -27,6 +27,17 @@ export interface WorkerPoolOptions {
 	idleTimeoutMs?: number;
 	/** Timeout for worker initialization in ms (default: 10000). */
 	initTimeoutMs?: number;
+	/** Grace period after request timeout before force-terminating stuck workers (default: 5000). */
+	stuckGracePeriodMs?: number;
+}
+
+export interface RequestOptions {
+	/** Timeout for this request in ms. After this, the promise rejects but worker gets a grace period. */
+	timeoutMs?: number;
+	/** Abort signal for this request. */
+	signal?: AbortSignal;
+	/** Transfer list for postMessage. */
+	transfer?: ArrayBufferLike[];
 }
 
 interface PooledWorker {
@@ -39,7 +50,8 @@ interface PooledWorker {
 interface PendingRequest<T> {
 	resolve: (result: T) => void;
 	reject: (error: Error) => void;
-	timeout?: ReturnType<typeof setTimeout>;
+	worker?: PooledWorker;
+	dispose?: () => void;
 }
 
 /**
@@ -62,26 +74,51 @@ export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
 			maxWorkers: options.maxWorkers ?? 4,
 			idleTimeoutMs: options.idleTimeoutMs ?? 30_000,
 			initTimeoutMs: options.initTimeoutMs ?? 10_000,
+			stuckGracePeriodMs: options.stuckGracePeriodMs ?? 5_000,
 		};
 	}
 
 	/**
 	 * Send a request to a worker and wait for the response.
 	 * Workers are acquired from the pool (or created if under limit).
+	 *
+	 * @param msg - Request message
+	 * @param options - Request options (timeout, transfer)
 	 */
 	async request<T extends TRes = TRes>(
 		msg: TReq | (Omit<TReq, "id"> & { id?: number }),
-		transfer?: ArrayBufferLike[],
+		options?: RequestOptions,
 	): Promise<T> {
+		const { timeoutMs, signal, transfer } = options ?? {};
+		signal?.throwIfAborted();
+
 		const worker = await this.#acquireWorker();
 		const id = msg.id ?? this.#nextRequestId++;
 		const fullMsg = { ...msg, id } as TReq;
 
 		const { promise, resolve, reject } = Promise.withResolvers<T>();
-		this.#pending.set(id, {
+		const pending: PendingRequest<T> = {
 			resolve: resolve as (result: TRes) => void,
 			reject,
-		});
+			worker,
+		};
+		this.#pending.set(id, pending as PendingRequest<TRes>);
+
+		const onAbort = () => {
+			this.#handleRequestAbort(id, worker);
+		};
+
+		if (timeoutMs && timeoutMs > 0 && signal) {
+			const combined = AbortSignal.any([signal, AbortSignal.timeout(timeoutMs)]);
+			combined.addEventListener("abort", onAbort, { once: true });
+			pending.dispose = () => combined.removeEventListener("abort", onAbort);
+		} else if (timeoutMs && timeoutMs > 0) {
+			const timer = setTimeout(onAbort, timeoutMs);
+			pending.dispose = () => clearTimeout(timer);
+		} else if (signal) {
+			signal.addEventListener("abort", onAbort, { once: true });
+			pending.dispose = () => signal.removeEventListener("abort", onAbort);
+		}
 
 		worker.currentRequestId = id;
 		if (transfer) {
@@ -109,7 +146,7 @@ export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
 
 		for (const pending of this.#pending.values()) {
 			pending.reject(new Error("Worker pool terminated"));
-			if (pending.timeout) clearTimeout(pending.timeout);
+			void pending.dispose?.();
 		}
 		this.#pending.clear();
 	}
@@ -144,7 +181,7 @@ export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
 		if (!pending) return;
 
 		this.#pending.delete(msg.id);
-		if (pending.timeout) clearTimeout(pending.timeout);
+		void pending.dispose?.();
 
 		if (msg.type === "error" && "error" in msg) {
 			pending.reject(new Error(msg.error ?? "Unknown error"));
@@ -163,11 +200,58 @@ export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
 		const pending = this.#pending.get(id);
 		if (pending) {
 			this.#pending.delete(id);
-			if (pending.timeout) clearTimeout(pending.timeout);
+			void pending.dispose?.();
 			pending.reject(error);
 		}
 	}
 
+	#handleRequestAbort(id: number, worker: PooledWorker): void {
+		const pending = this.#pending.get(id);
+		if (!pending) return;
+
+		pending.dispose = undefined;
+		pending.reject(new Error("Request timeout"));
+
+		if (this.#options.stuckGracePeriodMs > 0) {
+			const timer = setTimeout(() => {
+				this.#terminateStuckWorker(id, worker);
+			}, this.#options.stuckGracePeriodMs);
+
+			pending.dispose = () => {
+				clearTimeout(timer);
+			};
+		}
+	}
+
+	#terminateStuckWorker(id: number, worker: PooledWorker): void {
+		const pending = this.#pending.get(id);
+		if (pending) {
+			this.#pending.delete(id);
+			void pending.dispose?.();
+		}
+
+		if (worker.currentRequestId !== id) return;
+		if (!this.#pool.includes(worker)) return;
+
+		this.#removeWorker(worker);
+
+		if (this.#pool.length === 0 && this.#waiters.length > 0) {
+			this.#replenishPool();
+		}
+	}
+
+	async #replenishPool(): Promise<void> {
+		const worker = this.#createWorker();
+		worker.busy = true;
+		this.#pool.push(worker);
+		try {
+			await this.#initializeWorker(worker);
+			this.#releaseWorker(worker);
+		} catch {
+			this.#removeWorker(worker);
+		}
+	}
+
 	#removeWorker(pooledWorker: PooledWorker): void {
 		const idx = this.#pool.indexOf(pooledWorker);
 		if (idx !== -1) {
@@ -227,7 +311,7 @@ export class WorkerPool<TReq extends BaseRequest, TRes extends BaseResponse> {
 		this.#pending.set(id, {
 			resolve: () => resolve(),
 			reject,
-			timeout,
+			dispose: () => clearTimeout(timeout),
 		} as PendingRequest<TRes>);
 
 		pooledWorker.worker.postMessage({ type: "init", id } satisfies BaseRequest);

package/wasm/pi_natives.d.ts

@@ -101,6 +101,11 @@ export function has_match(content: string, pattern: string, ignore_case: boolean
  */
 export function highlight_code(code: string, lang: string | null | undefined, colors: any): string;
 
+/**
+ * Convert HTML to Markdown.
+ */
+export function html_to_markdown(html: string, options: any): string;
+
 /**
  * Resize an image to the specified dimensions.
  */

package/wasm/pi_natives.js

@@ -309,6 +309,39 @@ export function highlight_code(code, lang, colors) {
     }
 }
 
+/**
+ * Convert HTML to Markdown.
+ * @param {string} html
+ * @param {any} options
+ * @returns {string}
+ */
+export function html_to_markdown(html, options) {
+    let deferred3_0;
+    let deferred3_1;
+    try {
+        const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
+        const ptr0 = passStringToWasm0(html, wasm.__wbindgen_export, wasm.__wbindgen_export2);
+        const len0 = WASM_VECTOR_LEN;
+        wasm.html_to_markdown(retptr, ptr0, len0, addHeapObject(options));
+        var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+        var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+        var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
+        var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
+        var ptr2 = r0;
+        var len2 = r1;
+        if (r3) {
+            ptr2 = 0; len2 = 0;
+            throw takeObject(r2);
+        }
+        deferred3_0 = ptr2;
+        deferred3_1 = len2;
+        return getStringFromWasm0(ptr2, len2);
+    } finally {
+        wasm.__wbindgen_add_to_stack_pointer(16);
+        wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1);
+    }
+}
+
 /**
  * Resize an image to the specified dimensions.
  * @param {PhotonImage} image
@@ -451,6 +484,10 @@ function __wbg_get_imports() {
             const ret = typeof(getObject(arg0)) === 'bigint';
             return ret;
         },
+        __wbg___wbindgen_is_null_ac34f5003991759a: function(arg0) {
+            const ret = getObject(arg0) === null;
+            return ret;
+        },
         __wbg___wbindgen_is_object_5ae8e5880f2c1fbd: function(arg0) {
             const val = getObject(arg0);
             const ret = typeof(val) === 'object' && val !== null;

package/wasm/pi_natives_bg.wasm.d.ts

@@ -10,6 +10,7 @@ export const extract_segments: (a: number, b: number, c: number, d: number, e: n
 export const get_supported_languages: (a: number) => void;
 export const has_match: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void;
 export const highlight_code: (a: number, b: number, c: number, d: number, e: number, f: number) => void;
+export const html_to_markdown: (a: number, b: number, c: number, d: number) => void;
 export const photonimage_get_bytes: (a: number, b: number) => void;
 export const photonimage_get_bytes_jpeg: (a: number, b: number, c: number) => void;
 export const photonimage_get_height: (a: number) => number;