kreuzberg 4.0.0.rc1

data/examples/async_patterns.rb

@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+# Async Patterns for Kreuzberg Ruby Bindings
+#
+# This example demonstrates async patterns and concurrency approaches for Ruby,
+# with comparison to the underlying Rust implementation.
+
+require 'kreuzberg'
+
+# NOTE: Ruby bindings use Tokio runtime with block_on() internally.
+# The "async" functions block the Ruby GVL during execution, so there's
+# no performance benefit over the _sync variants from Ruby's perspective.
+
+# ============================================================================
+# Pattern 1: Synchronous Extraction (Recommended)
+# ============================================================================
+
+def basic_sync_extraction
+  result = Kreuzberg.extract_file_sync('document.pdf')
+  puts "Content: #{result[:content]}"
+  puts "MIME type: #{result[:mime_type]}"
+end
+
+# ============================================================================
+# Pattern 2: "Async" Extraction (Same Performance as Sync)
+# ============================================================================
+
+def basic_async_extraction
+  # This LOOKS async but actually blocks the Ruby thread
+  # Internally uses: runtime.block_on(async { ... })
+  result = Kreuzberg.extract_file('document.pdf')
+  puts "Content: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 3: Concurrent Processing with Ruby Threads
+# ============================================================================
+
+def concurrent_with_threads
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  # Use Ruby threads to achieve parallelism
+  # Each thread calls the synchronous API
+  threads = files.map do |file|
+    Thread.new do
+      Kreuzberg.extract_file_sync(file)
+    end
+  end
+
+  results = threads.map(&:value)
+  results.each_with_index do |result, index|
+    puts "File #{index + 1}: #{result[:content][0..100]}"
+  end
+end
+
+# ============================================================================
+# Pattern 4: Batch Processing (Preferred for Multiple Files)
+# ============================================================================
+
+def batch_processing
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  # The batch API handles concurrency internally via Rust/Tokio
+  # This is more efficient than Ruby threads
+  results = Kreuzberg.batch_extract_files_sync(files)
+
+  puts "Processed #{results.length} files"
+  results.each do |result|
+    puts "Content preview: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 5: Extraction with Configuration
+# ============================================================================
+
+def extraction_with_config
+  # Configure OCR
+  config = {
+    ocr: {
+      backend: 'tesseract',
+      language: 'eng'
+    },
+    force_ocr: false
+  }
+
+  result = Kreuzberg.extract_file_sync('scanned.pdf', **config)
+  puts "Extracted with OCR: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 6: Extract from Bytes
+# ============================================================================
+
+def extract_from_bytes
+  data = File.binread('document.pdf')
+  result = Kreuzberg.extract_bytes_sync(data, 'application/pdf')
+  puts "Extracted from memory: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 7: Batch Extract from Bytes
+# ============================================================================
+
+def batch_extract_from_bytes
+  files = ['doc1.pdf', 'doc2.pdf']
+  bytes_array = files.map { |f| File.binread(f) }
+  mime_types = ['application/pdf', 'application/pdf']
+
+  results = Kreuzberg.batch_extract_bytes_sync(bytes_array, mime_types)
+  puts "Processed #{results.length} files from memory"
+end
+
+# ============================================================================
+# Pattern 8: Error Handling
+# ============================================================================
+
+def error_handling
+  Kreuzberg.extract_file_sync('nonexistent.pdf')
+rescue StandardError => e
+  puts "Extraction failed: #{e.message}"
+end
+
+# ============================================================================
+# Pattern 9: Sequential Processing
+# ============================================================================
+
+def sequential_processing
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  files.each do |file|
+    result = Kreuzberg.extract_file_sync(file)
+    puts "Processed #{file}: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 10: Background Processing with ActiveJob (Rails)
+# ============================================================================
+
+# Example ActiveJob for async processing in Rails
+# < ApplicationJob
+class DocumentExtractionJob
+  # queue_as :default
+
+  def perform(file_path)
+    result = Kreuzberg.extract_file_sync(file_path)
+    # Store result in database or process further
+    puts "Background extraction complete: #{result[:content][0..100]}"
+  end
+end
+
+# Usage in Rails controller:
+# DocumentExtractionJob.perform_later('document.pdf')
+
+# ============================================================================
+# Pattern 11: Concurrent Processing with Parallel Gem
+# ============================================================================
+
+def concurrent_with_parallel_gem
+  require 'parallel'
+
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf', 'doc4.pdf']
+
+  # Process files in parallel using multiple CPU cores
+  results = Parallel.map(files, in_processes: 4) do |file|
+    Kreuzberg.extract_file_sync(file)
+  end
+
+  results.each do |result|
+    puts "Content: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 12: Timeout Wrapper
+# ============================================================================
+
+def extraction_with_timeout(file_path, timeout_seconds = 30)
+  require 'timeout'
+
+  Timeout.timeout(timeout_seconds) do
+    Kreuzberg.extract_file_sync(file_path)
+  end
+rescue Timeout::Error
+  puts "Extraction timed out after #{timeout_seconds} seconds"
+  nil
+end
+
+# ============================================================================
+# Pattern 13: Custom Ruby PostProcessor Plugin
+# ============================================================================
+
+def register_postprocessor
+  # Register a Ruby-based post-processor
+  uppercase_processor = lambda do |result|
+    result[:content] = result[:content].upcase
+    result
+  end
+
+  Kreuzberg.register_post_processor('uppercase', uppercase_processor, 100)
+
+  # Now all extractions will use the uppercase processor
+  result = Kreuzberg.extract_file_sync('document.pdf')
+  puts "Uppercase content: #{result[:content]}"
+
+  # Clean up
+  Kreuzberg.unregister_post_processor('uppercase')
+end
+
+# ============================================================================
+# Pattern 14: Custom Ruby Validator Plugin
+# ============================================================================
+
+def register_validator
+  # Register a Ruby-based validator
+  min_length_validator = lambda do |result|
+    raise 'Content too short' if result[:content].length < 100
+  end
+
+  Kreuzberg.register_validator('min_length', min_length_validator, 100)
+
+  # Validation will run automatically during extraction
+  begin
+    result = Kreuzberg.extract_file_sync('short_document.pdf')
+    puts "Validation passed: #{result[:content]}"
+  rescue StandardError => e
+    puts "Validation failed: #{e.message}"
+  end
+
+  # Clean up
+  Kreuzberg.unregister_validator('min_length')
+end
+
+# ============================================================================
+# Pattern 15: Custom Ruby OCR Backend Plugin
+# ============================================================================
+
+class CustomOcrBackend
+  def process_image(image_bytes, language)
+    # In a real implementation, you would:
+    # 1. Call an external OCR service
+    # 2. Use an HTTP API
+    # 3. Process with a Ruby gem
+    "Extracted text from #{image_bytes.length} bytes using #{language}"
+  end
+
+  def supports_language?(lang)
+    %w[eng deu fra].include?(lang)
+  end
+end
+
+def register_ocr_backend
+  backend = CustomOcrBackend.new
+  Kreuzberg.register_ocr_backend('custom', backend)
+
+  # Now you can use the custom backend
+  config = {
+    ocr: {
+      backend: 'custom',
+      language: 'eng'
+    },
+    force_ocr: true
+  }
+
+  result = Kreuzberg.extract_file_sync('scanned.pdf', **config)
+  puts "Custom OCR result: #{result[:content]}"
+end
+
+# ============================================================================
+# Main Demonstration
+# ============================================================================
+
+def main
+  puts '=== Basic Sync Extraction ==='
+  basic_sync_extraction
+
+  puts '\n=== Basic Async Extraction (Blocks GVL) ==='
+  basic_async_extraction
+
+  puts '\n=== Concurrent with Ruby Threads ==='
+  concurrent_with_threads
+
+  puts '\n=== Batch Processing (Preferred) ==='
+  batch_processing
+
+  puts '\n=== Extraction with Config ==='
+  extraction_with_config
+
+  puts '\n=== Extract from Bytes ==='
+  extract_from_bytes
+
+  puts '\n=== Error Handling ==='
+  error_handling
+
+  puts '\n=== Sequential Processing ==='
+  sequential_processing
+
+  puts '\n=== Extraction with Timeout ==='
+  extraction_with_timeout('large_document.pdf', 30)
+
+  puts '\n=== Custom PostProcessor ==='
+  register_postprocessor
+
+  puts '\n=== Custom Validator ==='
+  register_validator
+end
+
+# Run if executed directly
+main if __FILE__ == $PROGRAM_NAME
+
+# ============================================================================
+# Key Takeaways:
+#
+# 1. Ruby bindings use Tokio runtime with block_on() internally
+# 2. "Async" functions block the Ruby GVL - no concurrency benefit
+# 3. Use _sync variants for clarity (same performance)
+# 4. Use Ruby threads or Parallel gem for concurrent processing
+# 5. Batch API is most efficient for multiple files
+# 6. ActiveJob for background processing in Rails
+# 7. Ruby plugins (PostProcessor, Validator, OCR) are fully supported
+#
+# Performance Comparison:
+# - Magnus: Blocks GVL, same overhead as sync (~Xms per call)
+# - PyO3 (optimized): ~0.17ms overhead, GIL released during await
+# - NAPI-RS: ~0ms overhead, automatic Promise conversion
+#
+# When to Use Ruby Bindings:
+# ✅ Rails applications (ActiveJob for background processing)
+# ✅ Ruby scripts (existing Ruby codebases)
+# ✅ Simple extraction (single-file processing)
+# ✅ Batch processing (batch API handles concurrency)
+#
+# Consider Other Bindings For:
+# ❌ High concurrency (use Node.js/NAPI-RS instead)
+# ❌ Real-time processing (use Node.js/NAPI-RS instead)
+# ❌ I/O-bound workloads (use Python/PyO3 or Node.js/NAPI-RS)
+#
+# See packages/ruby/ext/kreuzberg_rb/native/README.md for detailed async runtime documentation.
+# ============================================================================

data/ext/kreuzberg_rb/extconf.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+require 'rbconfig'
+
+if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
+  devkit = ENV.fetch('RI_DEVKIT', nil)
+  prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
+  compat_include = File.expand_path('native/include/msvc_compat', __dir__).tr('\\', '/')
+
+  extra_args = []
+  extra_args << "-I#{compat_include}"
+
+  if devkit
+    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
+    extra_args.concat([
+                        '--target=x86_64-pc-windows-gnu',
+                        "--sysroot=#{sysroot}"
+                      ])
+  end
+
+  unless extra_args.empty?
+    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/).reject(&:empty?)
+    ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
+  end
+end
+
+default_profile = ENV.fetch('CARGO_PROFILE', 'release')
+native_dir = 'native'
+
+create_rust_makefile('kreuzberg_rb') do |config|
+  config.profile = default_profile.to_sym
+  config.ext_dir = native_dir
+end

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+name = "kreuzberg-rb"
+version = "4.0.0-rc.1"
+edition = "2024"
+rust-version = "1.85"
+authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
+license = "MIT"
+repository = "https://github.com/Goldziher/kreuzberg"
+homepage = "https://kreuzberg.dev"
+documentation = "https://docs.rs/kreuzberg"
+readme = "README.md"
+description = "Ruby bindings (Magnus) for Kreuzberg - high-performance document intelligence framework"
+keywords = ["document", "extraction", "ocr", "pdf", "ruby"]
+categories = ["api-bindings"]
+
+[lib]
+name = "kreuzberg_rb"
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+
+[dependencies]
+async-trait = "0.1"
+kreuzberg = { version = "4.0.0-rc.1", features = ["full", "embeddings"] }
+magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = ["rb-sys"] }
+rb-sys = { version = "0.9.117", default-features = false, features = ["stable-api-compiled-fallback"] }
+serde_json = "1.0"
+tokio = { version = "1.48", features = ["rt", "macros"] }
+html-to-markdown-rs = { version = "2.9.1", default-features = false }
+
+[dev-dependencies]
+pretty_assertions = "1.4"
+
+[patch.crates-io]
+kreuzberg = { path = "../../../../vendor/kreuzberg" }