@kreuzberg/wasm 4.0.0-rc.6 → 4.0.0

package/dist/pkg/README.md

@@ -0,0 +1,503 @@
+# WebAssembly
+
+<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
+  <!-- Language Bindings -->
+  <a href="https://crates.io/crates/kreuzberg">
+    <img src="https://img.shields.io/crates/v/kreuzberg?label=Rust&color=007ec6" alt="Rust">
+  </a>
+  <a href="https://hex.pm/packages/kreuzberg">
+    <img src="https://img.shields.io/hexpm/v/kreuzberg?label=Elixir&color=007ec6" alt="Elixir">
+  </a>
+  <a href="https://pypi.org/project/kreuzberg/">
+    <img src="https://img.shields.io/pypi/v/kreuzberg?label=Python&color=007ec6" alt="Python">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/node">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/node?label=Node.js&color=007ec6" alt="Node.js">
+  </a>
+  <a href="https://www.npmjs.com/package/@kreuzberg/wasm">
+    <img src="https://img.shields.io/npm/v/@kreuzberg/wasm?label=WASM&color=007ec6" alt="WASM">
+  </a>
+
+  <a href="https://central.sonatype.com/artifact/dev.kreuzberg/kreuzberg">
+    <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
+  </a>
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.0.0" alt="Go">
+  </a>
+  <a href="https://www.nuget.org/packages/Kreuzberg/">
+    <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">
+  </a>
+  <a href="https://packagist.org/packages/kreuzberg/kreuzberg">
+    <img src="https://img.shields.io/packagist/v/kreuzberg/kreuzberg?label=PHP&color=007ec6" alt="PHP">
+  </a>
+  <a href="https://rubygems.org/gems/kreuzberg">
+    <img src="https://img.shields.io/gem/v/kreuzberg?label=Ruby&color=007ec6" alt="Ruby">
+  </a>
+
+  <!-- Project Info -->
+  <a href="https://github.com/kreuzberg-dev/kreuzberg/blob/main/LICENSE">
+    <img src="https://img.shields.io/badge/License-MIT-blue.svg" alt="License">
+  </a>
+  <a href="https://docs.kreuzberg.dev">
+    <img src="https://img.shields.io/badge/docs-kreuzberg.dev-blue" alt="Documentation">
+  </a>
+</div>
+
+<img width="1128" height="191" alt="Banner2" src="https://github.com/user-attachments/assets/419fc06c-8313-4324-b159-4b4d3cfce5c0" />
+
+<div align="center" style="margin-top: 20px;">
+  <a href="https://discord.gg/pXxagNK2zN">
+      <img height="22" src="https://img.shields.io/badge/Discord-Join%20our%20community-7289da?logo=discord&logoColor=white" alt="Discord">
+  </a>
+</div>
+
+
+Extract text, tables, images, and metadata from 56 file formats including PDF, Office documents, and images. WebAssembly bindings for browsers, Deno, and Cloudflare Workers with portable deployment and multi-threading support.
+
+
+## Installation
+
+### Package Installation
+
+
+Install via one of the supported package managers:
+
+
+
+**npm:**
+```bash
+npm install @kreuzberg/wasm
+```
+
+
+
+
+**pnpm:**
+```bash
+pnpm add @kreuzberg/wasm
+```
+
+
+
+
+**yarn:**
+```bash
+yarn add @kreuzberg/wasm
+```
+
+
+
+
+
+### System Requirements
+
+- Modern browser with WebAssembly support, or Deno 1.0+, or Cloudflare Workers
+- Optional: [Tesseract WASM](https://github.com/naptha/tesseract.js) for OCR functionality
+
+
+
+## Quick Start
+
+### Basic Extraction
+
+Extract text, metadata, and structure from any supported document format:
+
+```ts
+import { extractBytes, initWasm } from "@kreuzberg/wasm";
+
+async function main() {
+	await initWasm();
+
+	const buffer = await fetch("document.pdf").then((r) => r.arrayBuffer());
+	const bytes = new Uint8Array(buffer);
+
+	const result = await extractBytes(bytes, "application/pdf");
+
+	console.log("Extracted content:");
+	console.log(result.content);
+	console.log("MIME type:", result.mimeType);
+	console.log("Metadata:", result.metadata);
+}
+
+main().catch(console.error);
+```
+
+
+### Common Use Cases
+
+#### Extract with Custom Configuration
+
+Most use cases benefit from configuration to control extraction behavior:
+
+
+**With OCR (for scanned documents):**
+
+```ts
+import { enableOcr, extractBytes, initWasm } from "@kreuzberg/wasm";
+
+async function extractWithOcr() {
+	await initWasm();
+
+	try {
+		await enableOcr();
+		console.log("OCR enabled successfully");
+	} catch (error) {
+		console.error("Failed to enable OCR:", error);
+		return;
+	}
+
+	const bytes = new Uint8Array(await fetch("scanned-page.png").then((r) => r.arrayBuffer()));
+
+	const result = await extractBytes(bytes, "image/png", {
+		ocr: {
+			backend: "tesseract-wasm",
+			language: "eng",
+		},
+	});
+
+	console.log("Extracted text:");
+	console.log(result.content);
+}
+
+extractWithOcr().catch(console.error);
+```
+
+
+
+
+#### Table Extraction
+
+
+See [Table Extraction Guide](https://kreuzberg.dev/features/table-extraction/) for detailed examples.
+
+
+
+#### Processing Multiple Files
+
+
+```ts
+import { extractBytes, initWasm } from "@kreuzberg/wasm";
+
+interface DocumentJob {
+	name: string;
+	bytes: Uint8Array;
+	mimeType: string;
+}
+
+async function _processBatch(documents: DocumentJob[], concurrency: number = 3) {
+	await initWasm();
+
+	const results: Record<string, string> = {};
+	const queue = [...documents];
+
+	const workers = Array(concurrency)
+		.fill(null)
+		.map(async () => {
+			while (queue.length > 0) {
+				const doc = queue.shift();
+				if (!doc) break;
+
+				try {
+					const result = await extractBytes(doc.bytes, doc.mimeType);
+					results[doc.name] = result.content;
+				} catch (error) {
+					console.error(`Failed to process ${doc.name}:`, error);
+				}
+			}
+		});
+
+	await Promise.all(workers);
+	return results;
+}
+```
+
+
+
+
+
+#### Async Processing
+
+For non-blocking document processing:
+
+```ts
+import { extractBytes, getWasmCapabilities, initWasm } from "@kreuzberg/wasm";
+
+async function extractDocuments(files: Uint8Array[], mimeTypes: string[]) {
+	const caps = getWasmCapabilities();
+	if (!caps.hasWasm) {
+		throw new Error("WebAssembly not supported");
+	}
+
+	await initWasm();
+
+	const results = await Promise.all(files.map((bytes, index) => extractBytes(bytes, mimeTypes[index])));
+
+	return results.map((r) => ({
+		content: r.content,
+		pageCount: r.metadata?.pageCount,
+	}));
+}
+
+const fileBytes = [new Uint8Array([1, 2, 3])];
+const mimes = ["application/pdf"];
+
+extractDocuments(fileBytes, mimes)
+	.then((results) => console.log(results))
+	.catch(console.error);
+```
+
+
+
+
+
+
+### Next Steps
+
+- **[Installation Guide](https://kreuzberg.dev/getting-started/installation/)** - Platform-specific setup
+- **[API Documentation](https://kreuzberg.dev/api/)** - Complete API reference
+- **[Examples & Guides](https://kreuzberg.dev/guides/)** - Full code examples and usage guides
+- **[Configuration Guide](https://kreuzberg.dev/configuration/)** - Advanced configuration options
+- **[Troubleshooting](https://kreuzberg.dev/troubleshooting/)** - Common issues and solutions
+
+
+
+## Features
+
+### Supported File Formats (56+)
+
+56 file formats across 8 major categories with intelligent format detection and comprehensive metadata extraction.
+
+#### Office Documents
+
+| Category | Formats | Capabilities |
+|----------|---------|--------------|
+| **Word Processing** | `.docx`, `.odt` | Full text, tables, images, metadata, styles |
+| **Spreadsheets** | `.xlsx`, `.xlsm`, `.xlsb`, `.xls`, `.xla`, `.xlam`, `.xltm`, `.ods` | Sheet data, formulas, cell metadata, charts |
+| **Presentations** | `.pptx`, `.ppt`, `.ppsx` | Slides, speaker notes, images, metadata |
+| **PDF** | `.pdf` | Text, tables, images, metadata, OCR support |
+| **eBooks** | `.epub`, `.fb2` | Chapters, metadata, embedded resources |
+
+#### Images (OCR-Enabled)
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Raster** | `.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, `.bmp`, `.tiff`, `.tif` | OCR, table detection, EXIF metadata, dimensions, color space |
+| **Advanced** | `.jp2`, `.jpx`, `.jpm`, `.mj2`, `.pnm`, `.pbm`, `.pgm`, `.ppm` | OCR, table detection, format-specific metadata |
+| **Vector** | `.svg` | DOM parsing, embedded text, graphics metadata |
+
+#### Web & Data
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Markup** | `.html`, `.htm`, `.xhtml`, `.xml`, `.svg` | DOM parsing, metadata (Open Graph, Twitter Card), link extraction |
+| **Structured Data** | `.json`, `.yaml`, `.yml`, `.toml`, `.csv`, `.tsv` | Schema detection, nested structures, validation |
+| **Text & Markdown** | `.txt`, `.md`, `.markdown`, `.rst`, `.org`, `.rtf` | CommonMark, GFM, reStructuredText, Org Mode |
+
+#### Email & Archives
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Email** | `.eml`, `.msg` | Headers, body (HTML/plain), attachments, threading |
+| **Archives** | `.zip`, `.tar`, `.tgz`, `.gz`, `.7z` | File listing, nested archives, metadata |
+
+#### Academic & Scientific
+
+| Category | Formats | Features |
+|----------|---------|----------|
+| **Citations** | `.bib`, `.biblatex`, `.ris`, `.enw`, `.csl` | Bibliography parsing, citation extraction |
+| **Scientific** | `.tex`, `.latex`, `.typst`, `.jats`, `.ipynb`, `.docbook` | LaTeX, Jupyter notebooks, PubMed JATS |
+| **Documentation** | `.opml`, `.pod`, `.mdoc`, `.troff` | Technical documentation formats |
+
+**[Complete Format Reference](https://kreuzberg.dev/reference/formats/)**
+
+### Key Capabilities
+
+- **Text Extraction** - Extract all text content with position and formatting information
+- **Metadata Extraction** - Retrieve document properties, creation date, author, etc.
+- **Table Extraction** - Parse tables with structure and cell content preservation
+- **Image Extraction** - Extract embedded images and render page previews
+- **OCR Support** - Integrate multiple OCR backends for scanned documents
+
+- **Async/Await** - Non-blocking document processing with concurrent operations
+
+
+- **Plugin System** - Extensible post-processing for custom text transformation
+
+
+- **Batch Processing** - Efficiently process multiple documents in parallel
+- **Memory Efficient** - Stream large files without loading entirely into memory
+- **Language Detection** - Detect and support multiple languages in documents
+- **Configuration** - Fine-grained control over extraction behavior
+
+### Performance Characteristics
+
+| Format | Speed | Memory | Notes |
+|--------|-------|--------|-------|
+| **PDF (text)** | 10-100 MB/s | ~50MB per doc | Fastest extraction |
+| **Office docs** | 20-200 MB/s | ~100MB per doc | DOCX, XLSX, PPTX |
+| **Images (OCR)** | 1-5 MB/s | Variable | Depends on OCR backend |
+| **Archives** | 5-50 MB/s | ~200MB per doc | ZIP, TAR, etc. |
+| **Web formats** | 50-200 MB/s | Streaming | HTML, XML, JSON |
+
+
+
+## OCR Support
+
+Kreuzberg supports multiple OCR backends for extracting text from scanned documents and images:
+
+
+- **Tesseract-Wasm**
+
+
+### OCR Configuration Example
+
+```ts
+import { enableOcr, extractBytes, initWasm } from "@kreuzberg/wasm";
+
+async function extractWithOcr() {
+	await initWasm();
+
+	try {
+		await enableOcr();
+		console.log("OCR enabled successfully");
+	} catch (error) {
+		console.error("Failed to enable OCR:", error);
+		return;
+	}
+
+	const bytes = new Uint8Array(await fetch("scanned-page.png").then((r) => r.arrayBuffer()));
+
+	const result = await extractBytes(bytes, "image/png", {
+		ocr: {
+			backend: "tesseract-wasm",
+			language: "eng",
+		},
+	});
+
+	console.log("Extracted text:");
+	console.log(result.content);
+}
+
+extractWithOcr().catch(console.error);
+```
+
+
+
+
+## Async Support
+
+This binding provides full async/await support for non-blocking document processing:
+
+```ts
+import { extractBytes, getWasmCapabilities, initWasm } from "@kreuzberg/wasm";
+
+async function extractDocuments(files: Uint8Array[], mimeTypes: string[]) {
+	const caps = getWasmCapabilities();
+	if (!caps.hasWasm) {
+		throw new Error("WebAssembly not supported");
+	}
+
+	await initWasm();
+
+	const results = await Promise.all(files.map((bytes, index) => extractBytes(bytes, mimeTypes[index])));
+
+	return results.map((r) => ({
+		content: r.content,
+		pageCount: r.metadata?.pageCount,
+	}));
+}
+
+const fileBytes = [new Uint8Array([1, 2, 3])];
+const mimes = ["application/pdf"];
+
+extractDocuments(fileBytes, mimes)
+	.then((results) => console.log(results))
+	.catch(console.error);
+```
+
+
+
+
+## Plugin System
+
+Kreuzberg supports extensible post-processing plugins for custom text transformation and filtering.
+
+For detailed plugin documentation, visit [Plugin System Guide](https://kreuzberg.dev/plugins/).
+
+
+
+
+
+
+## Batch Processing
+
+Process multiple documents efficiently:
+
+```ts
+import { extractBytes, initWasm } from "@kreuzberg/wasm";
+
+interface DocumentJob {
+	name: string;
+	bytes: Uint8Array;
+	mimeType: string;
+}
+
+async function _processBatch(documents: DocumentJob[], concurrency: number = 3) {
+	await initWasm();
+
+	const results: Record<string, string> = {};
+	const queue = [...documents];
+
+	const workers = Array(concurrency)
+		.fill(null)
+		.map(async () => {
+			while (queue.length > 0) {
+				const doc = queue.shift();
+				if (!doc) break;
+
+				try {
+					const result = await extractBytes(doc.bytes, doc.mimeType);
+					results[doc.name] = result.content;
+				} catch (error) {
+					console.error(`Failed to process ${doc.name}:`, error);
+				}
+			}
+		});
+
+	await Promise.all(workers);
+	return results;
+}
+```
+
+
+
+
+## Configuration
+
+For advanced configuration options including language detection, table extraction, OCR settings, and more:
+
+**[Configuration Guide](https://kreuzberg.dev/configuration/)**
+
+## Documentation
+
+- **[Official Documentation](https://kreuzberg.dev/)**
+- **[API Reference](https://kreuzberg.dev/reference/api-wasm/)**
+- **[Examples & Guides](https://kreuzberg.dev/guides/)**
+
+## Troubleshooting
+
+For common issues and solutions, visit [Troubleshooting Guide](https://kreuzberg.dev/troubleshooting/).
+
+## Contributing
+
+Contributions are welcome! See [Contributing Guide](https://github.com/kreuzberg-dev/kreuzberg/blob/main/CONTRIBUTING.md).
+
+## License
+
+MIT License - see LICENSE file for details.
+
+## Support
+
+- **Discord Community**: [Join our Discord](https://discord.gg/pXxagNK2zN)
+- **GitHub Issues**: [Report bugs](https://github.com/kreuzberg-dev/kreuzberg/issues)
+- **Discussions**: [Ask questions](https://github.com/kreuzberg-dev/kreuzberg/discussions)

package/dist/{kreuzberg_wasm.d.ts → pkg/kreuzberg_wasm.d.ts}

@@ -429,7 +429,7 @@ export function getExtensionsForMime(mime_type: string): Array<any>;
  * console.log(unknownMime); // null
  * ```
  */
-export function getMimeFromExtension(extension: string): string | undefined;
+export function getMimeFromExtension(extension: string): string;
 
 /**
  * Get module information
@@ -442,7 +442,7 @@ export function get_module_info(): ModuleInfo;
  */
 export function init(): void;
 
-export function initThreadPool(num_threads: number): Promise<any>;
+export function initThreadPool(_num_threads: number): Promise<any>;
 
 /**
  * Helper function to initialize the thread pool with error handling
@@ -455,6 +455,15 @@ export function initThreadPool(num_threads: number): Promise<any>;
  */
 export function init_thread_pool_safe(num_threads: number): boolean;
 
+/**
+ * Establishes a binding between an external Pdfium WASM module and `pdfium-render`'s WASM module.
+ * This function should be called from Javascript once the external Pdfium WASM module has been loaded
+ * into the browser. It is essential that this function is called _before_ initializing
+ * `pdfium-render` from within Rust code. For an example, see:
+ * <https://github.com/ajrcarey/pdfium-render/blob/master/examples/index.html>
+ */
+export function initialize_pdfium_render(pdfium_wasm_module: any, local_wasm_module: any, debug: boolean): boolean;
+
 /**
  * List all registered OCR backend names.
  *
@@ -586,6 +595,13 @@ export function loadConfigFromString(content: string, format: string): any;
  */
 export function normalizeMimeType(mime_type: string): string;
 
+/**
+ * A callback function that can be invoked by Pdfium's `FPDF_LoadCustomDocument()` function,
+ * wrapping around `crate::utils::files::read_block_from_callback()` to shuffle data buffers
+ * from our WASM memory heap to Pdfium's WASM memory heap as they are loaded.
+ */
+export function read_block_from_callback_wasm(param: number, position: number, pBuf: number, size: number): number;
+
 /**
  * Register a custom OCR backend.
  *
@@ -746,13 +762,9 @@ export function unregister_validator(name: string): void;
  */
 export function version(): string;
 
-export class wbg_rayon_PoolBuilder {
-  private constructor();
-  free(): void;
-  [Symbol.dispose](): void;
-  numThreads(): number;
-  build(): void;
-  receiver(): number;
-}
-
-export function wbg_rayon_start_worker(receiver: number): void;
+/**
+ * A callback function that can be invoked by Pdfium's `FPDF_SaveAsCopy()` and `FPDF_SaveWithVersion()`
+ * functions, wrapping around `crate::utils::files::write_block_from_callback()` to shuffle data buffers
+ * from Pdfium's WASM memory heap to our WASM memory heap as they are written.
+ */
+export function write_block_from_callback_wasm(param: number, buf: number, size: number): number;