kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/Rakefile

@@ -6,13 +6,6 @@ require 'rspec/core/rake_task'
 
 GEMSPEC = Gem::Specification.load(File.expand_path('kreuzberg.gemspec', __dir__))
 
-# Vendor kreuzberg core crates before compilation
-task :vendor do
-  vendor_script = File.expand_path('../../scripts/ci/ruby/vendor-kreuzberg-core.sh', __dir__)
-  puts 'Vendoring kreuzberg core crates...'
-  sh "bash #{vendor_script}"
-end
-
 Rake::ExtensionTask.new('kreuzberg_rb', GEMSPEC) do |ext|
   ext.lib_dir = 'lib'
   ext.ext_dir = 'ext/kreuzberg_rb'
@@ -23,12 +16,10 @@ Rake::ExtensionTask.new('kreuzberg_rb', GEMSPEC) do |ext|
     x86_64-darwin
     arm64-darwin
     x64-mingw32
-    x64-mingw-ucrt
   ]
 end
 
 RSpec::Core::RakeTask.new(:spec)
 
-task compile: :vendor
 task spec: :compile
 task default: :spec

data/Steepfile

@@ -15,23 +15,19 @@ target :lib do
 
   # Strategic ignores for steep limitations (not fixable, safe to ignore):
 
-  # 1. Sorbet type annotations - Steep doesn't recognize Sorbet's T::Struct and T::Sig
-  #    This file uses Sorbet exclusively for type definitions
-  ignore 'lib/kreuzberg/types.rb'
-
-  # 2. Struct.new with keyword_init - steep cannot understand implicit attr_readers
+  # 1. Struct.new with keyword_init - steep cannot understand implicit attr_readers
   #    defined by Struct.new in blocks (Table and Chunk classes)
   ignore 'lib/kreuzberg/result.rb'
 
-  # 3. Generic type parameters in normalize_config - steep has difficulty with
+  # 2. Generic type parameters in normalize_config - steep has difficulty with
   #    methods that take Class as parameter and return instances
   ignore 'lib/kreuzberg/config.rb'
 
-  # 4. Interface types - steep doesn't recognize that all Ruby objects have nil? and is_a?
+  # 3. Interface types - steep doesn't recognize that all Ruby objects have nil? and is_a?
   #    even for interface types like _ToH
   ignore 'lib/kreuzberg/extraction_api.rb'
 
-  # 5. Open3 methods - steep's built-in Open3 RBS signatures incomplete
+  # 4. Open3 methods - steep's built-in Open3 RBS signatures incomplete
   #    (capture2, capture3, popen3 are standard library methods)
   ignore 'lib/kreuzberg/setup_lib_path.rb'
   ignore 'lib/kreuzberg/cli_proxy.rb'

data/examples/async_patterns.rb

@@ -1,5 +1,10 @@
 # frozen_string_literal: true
 
+# Async Patterns for Kreuzberg Ruby Bindings
+#
+# This example demonstrates async patterns and concurrency approaches for Ruby,
+# with comparison to the underlying Rust implementation.
+
 require 'kreuzberg'
 
 # NOTE: Ruby bindings use Tokio runtime with block_on() internally.
@@ -21,6 +26,8 @@ end
 # ============================================================================
 
 def basic_async_extraction
+  # This LOOKS async but actually blocks the Ruby thread
+  # Internally uses: runtime.block_on(async { ... })
   result = Kreuzberg.extract_file('document.pdf')
   puts "Content: #{result[:content]}"
 end
@@ -32,6 +39,8 @@ end
 def concurrent_with_threads
   files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
 
+  # Use Ruby threads to achieve parallelism
+  # Each thread calls the synchronous API
   threads = files.map do |file|
     Thread.new do
       Kreuzberg.extract_file_sync(file)
@@ -51,6 +60,8 @@ end
 def batch_processing
   files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
 
+  # The batch API handles concurrency internally via Rust/Tokio
+  # This is more efficient than Ruby threads
   results = Kreuzberg.batch_extract_files_sync(files)
 
   puts "Processed #{results.length} files"
@@ -64,6 +75,7 @@ end
 # ============================================================================
 
 def extraction_with_config
+  # Configure OCR
   config = {
     ocr: {
       backend: 'tesseract',
@@ -129,8 +141,11 @@ end
 # Example ActiveJob for async processing in Rails
 # < ApplicationJob
 class DocumentExtractionJob
+  # queue_as :default
+
   def perform(file_path)
     result = Kreuzberg.extract_file_sync(file_path)
+    # Store result in database or process further
     puts "Background extraction complete: #{result[:content][0..100]}"
   end
 end
@@ -147,6 +162,7 @@ def concurrent_with_parallel_gem
 
   files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf', 'doc4.pdf']
 
+  # Process files in parallel using multiple CPU cores
   results = Parallel.map(files, in_processes: 4) do |file|
     Kreuzberg.extract_file_sync(file)
   end
@@ -176,6 +192,7 @@ end
 # ============================================================================
 
 def register_postprocessor
+  # Register a Ruby-based post-processor
   uppercase_processor = lambda do |result|
     result[:content] = result[:content].upcase
     result
@@ -183,9 +200,11 @@ def register_postprocessor
 
   Kreuzberg.register_post_processor('uppercase', uppercase_processor, 100)
 
+  # Now all extractions will use the uppercase processor
   result = Kreuzberg.extract_file_sync('document.pdf')
   puts "Uppercase content: #{result[:content]}"
 
+  # Clean up
   Kreuzberg.unregister_post_processor('uppercase')
 end
 
@@ -194,12 +213,14 @@ end
 # ============================================================================
 
 def register_validator
+  # Register a Ruby-based validator
   min_length_validator = lambda do |result|
     raise 'Content too short' if result[:content].length < 100
   end
 
   Kreuzberg.register_validator('min_length', min_length_validator, 100)
 
+  # Validation will run automatically during extraction
   begin
     result = Kreuzberg.extract_file_sync('short_document.pdf')
     puts "Validation passed: #{result[:content]}"
@@ -207,6 +228,7 @@ def register_validator
     puts "Validation failed: #{e.message}"
   end
 
+  # Clean up
   Kreuzberg.unregister_validator('min_length')
 end
 
@@ -214,9 +236,12 @@ end
 # Pattern 15: Custom Ruby OCR Backend Plugin
 # ============================================================================
 
-# Example OCR backend implementation for custom processing.
 class CustomOcrBackend
   def process_image(image_bytes, language)
+    # In a real implementation, you would:
+    # 1. Call an external OCR service
+    # 2. Use an HTTP API
+    # 3. Process with a Ruby gem
     "Extracted text from #{image_bytes.length} bytes using #{language}"
   end
 
@@ -229,6 +254,7 @@ def register_ocr_backend
   backend = CustomOcrBackend.new
   Kreuzberg.register_ocr_backend('custom', backend)
 
+  # Now you can use the custom backend
   config = {
     ocr: {
       backend: 'custom',
@@ -280,4 +306,35 @@ def main
   register_validator
 end
 
+# Run if executed directly
 main if __FILE__ == $PROGRAM_NAME
+
+# ============================================================================
+# Key Takeaways:
+#
+# 1. Ruby bindings use Tokio runtime with block_on() internally
+# 2. "Async" functions block the Ruby GVL - no concurrency benefit
+# 3. Use _sync variants for clarity (same performance)
+# 4. Use Ruby threads or Parallel gem for concurrent processing
+# 5. Batch API is most efficient for multiple files
+# 6. ActiveJob for background processing in Rails
+# 7. Ruby plugins (PostProcessor, Validator, OCR) are fully supported
+#
+# Performance Comparison:
+# - Magnus: Blocks GVL, same overhead as sync (~Xms per call)
+# - PyO3 (optimized): ~0.17ms overhead, GIL released during await
+# - NAPI-RS: ~0ms overhead, automatic Promise conversion
+#
+# When to Use Ruby Bindings:
+# ✅ Rails applications (ActiveJob for background processing)
+# ✅ Ruby scripts (existing Ruby codebases)
+# ✅ Simple extraction (single-file processing)
+# ✅ Batch processing (batch API handles concurrency)
+#
+# Consider Other Bindings For:
+# ❌ High concurrency (use Node.js/NAPI-RS instead)
+# ❌ Real-time processing (use Node.js/NAPI-RS instead)
+# ❌ I/O-bound workloads (use Python/PyO3 or Node.js/NAPI-RS)
+#
+# See packages/ruby/ext/kreuzberg_rb/native/README.md for detailed async runtime documentation.
+# ============================================================================

data/ext/kreuzberg_rb/extconf.rb

@@ -3,57 +3,27 @@
 require 'mkmf'
 require 'rb_sys/mkmf'
 require 'rbconfig'
-require 'fileutils'
-
-if Gem.win_platform?
-  # Use CI-provided CARGO_TARGET_DIR if available, otherwise use a short path
-  # GitHub Actions sets CARGO_TARGET_DIR=C:\t for MAX_PATH mitigation
-  if ENV['CARGO_TARGET_DIR']
-    puts "Windows detected: Using existing CARGO_TARGET_DIR=#{ENV['CARGO_TARGET_DIR']}"
-  else
-    # Try C:\t first (CI convention), fall back to D:/kz-build
-    short_target_dir = Dir.exist?('C:/t') ? 'C:/t' : 'C:/kz-build'
-    begin
-      FileUtils.mkdir_p(short_target_dir)
-      ENV['CARGO_TARGET_DIR'] = short_target_dir
-      ENV['OUT_DIR'] = short_target_dir
-      puts "Windows detected: Using short build path #{short_target_dir}"
-    rescue StandardError => e
-      puts "Warning: Could not create short path #{short_target_dir}: #{e.message}"
-      # Fall back to relative path which rb_sys will handle
-    end
-  end
-end
 
 if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
   devkit = ENV.fetch('RI_DEVKIT', nil)
   prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
-
-  # Set up include paths for MSVC compatibility headers
-  native_include = File.expand_path('native/include', __dir__).tr('\\', '/')
   compat_include = File.expand_path('native/include/msvc_compat', __dir__).tr('\\', '/')
 
   extra_args = []
-  extra_args << "-I#{native_include}"
   extra_args << "-I#{compat_include}"
 
-  # Add Windows-specific flags for better compatibility
-  extra_args << '-fms-extensions'
-  extra_args << '-fno-omit-frame-pointer'
-
   if devkit
-    sysroot = "#{devkit}#{prefix}".tr('\\', '/')
-    extra_args.push('--target=x86_64-pc-windows-gnu', "--sysroot=#{sysroot}")
+    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
+    extra_args.concat([
+                        '--target=x86_64-pc-windows-gnu',
+                        "--sysroot=#{sysroot}"
+                      ])
   end
 
   unless extra_args.empty?
     existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/).reject(&:empty?)
     ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
-    puts "BINDGEN_EXTRA_CLANG_ARGS set to: #{ENV.fetch('BINDGEN_EXTRA_CLANG_ARGS', nil)}"
   end
-
-  # Set target for Windows GNU toolchain if not already set
-  ENV['CARGO_BUILD_TARGET'] ||= 'x86_64-pc-windows-gnu' if devkit || ENV['MSYSTEM']
 end
 
 default_profile = ENV.fetch('CARGO_PROFILE', 'release')

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,75 +1,36 @@
-[workspace]
-
-[workspace.lints.clippy]
-collapsible_if = "allow"
-
 [package]
 name = "kreuzberg-rb"
-version = "4.0.0-rc.28"
+version = "4.0.0-rc.1"
 edition = "2024"
-rust-version = "1.91"
+rust-version = "1.85"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
 license = "MIT"
-repository = "https://github.com/kreuzberg-dev/kreuzberg"
+repository = "https://github.com/Goldziher/kreuzberg"
 homepage = "https://kreuzberg.dev"
 documentation = "https://docs.rs/kreuzberg"
 readme = "README.md"
 description = "Ruby bindings (Magnus) for Kreuzberg - high-performance document intelligence framework"
-keywords = ["ruby", "magnus", "document", "extraction", "bindings"]
-categories = ["api-bindings", "text-processing"]
-
-[lints]
-workspace = true
+keywords = ["document", "extraction", "ocr", "pdf", "ruby"]
+categories = ["api-bindings"]
 
 [lib]
 name = "kreuzberg_rb"
 crate-type = ["cdylib", "rlib"]
 
 [features]
-default = ["embeddings"]
-embeddings = ["kreuzberg/embeddings"]
+default = []
 
 [dependencies]
-async-trait = "0.1.89"
-kreuzberg = { path = "../../../vendor/kreuzberg", default-features = false, features = [
-    "pdf",
-    "excel",
-    "office",
-    "email",
-    "html",
-    "xml",
-    "archives",
-    "ocr",
-    "language-detection",
-    "chunking",
-    "embeddings",
-    "quality",
-    "keywords",
-    "api",
-    "mcp",
-    "otel",
-    "bundled-pdfium",
-    "tokio-runtime",
-] }
-kreuzberg-ffi = { path = "../../../vendor/kreuzberg-ffi" }
-magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = [
-    "rb-sys",
-] }
-rb-sys = { version = "0.9.119", default-features = false, features = [
-    "stable-api-compiled-fallback",
-] }
-serde_json = "1.0.145"
-tokio = { version = "1.48.0", features = [
-    "rt",
-    "rt-multi-thread",
-    "macros",
-    "sync",
-    "process",
-    "fs",
-    "time",
-    "io-util",
-] }
-html-to-markdown-rs = { version = "2.14.2", default-features = false }
+async-trait = "0.1"
+kreuzberg = { version = "4.0.0-rc.1", features = ["full", "embeddings"] }
+magnus = { git = "https://github.com/matsadler/magnus", rev = "f6db11769efb517427bf7f121f9c32e18b059b38", features = ["rb-sys"] }
+rb-sys = { version = "0.9.117", default-features = false, features = ["stable-api-compiled-fallback"] }
+serde_json = "1.0"
+tokio = { version = "1.48", features = ["rt", "macros"] }
+html-to-markdown-rs = { version = "2.9.1", default-features = false }
 
 [dev-dependencies]
 pretty_assertions = "1.4"
+
+[patch.crates-io]
+kreuzberg = { path = "../../../../vendor/kreuzberg" }

data/ext/kreuzberg_rb/native/build.rs

@@ -1,15 +1,17 @@
+#[cfg(target_os = "macos")]
 fn main() {
-    let target = std::env::var("TARGET").unwrap();
-
-    if target.contains("darwin") {
-        println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
-        println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
-    } else if target.contains("linux") {
-        println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
-    } else if target.contains("windows") {
-        // Windows doesn't need rpath or dynamic_lookup equivalents
-        // The linker flags are already configured in .cargo/config.toml
-    }
+    println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
+    // Set rpath to look for libpdfium.dylib in the same directory as the Ruby extension
+    println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path");
+    println!("cargo:rustc-link-arg=-Wl,-rpath,@loader_path/.");
+}
 
-    println!("cargo:rerun-if-changed=build.rs");
+#[cfg(target_os = "linux")]
+fn main() {
+    // Set rpath to look for libpdfium.so in the same directory as the Ruby extension
+    println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN");
+    println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/.");
 }
+
+#[cfg(not(any(target_os = "macos", target_os = "linux")))]
+fn main() {}

data/ext/kreuzberg_rb/native/include/ieeefp.h

@@ -8,4 +8,4 @@
 #include_next <ieeefp.h>
 #endif
 
-#endif
+#endif  // KREUZBERG_RUBY_IEEFP_H

data/ext/kreuzberg_rb/native/include/msvc_compat/strings.h

@@ -11,4 +11,4 @@
 #define strncasecmp _strnicmp
 #endif
 
-#endif
+#endif /* KREUZBERG_RB_MSVC_COMPAT_STRINGS_H */

data/ext/kreuzberg_rb/native/include/strings.h

@@ -15,6 +15,6 @@
 #ifndef bzero
 #define bzero(ptr, size) memset((ptr), 0, (size))
 #endif
-#endif
+#endif  // _MSC_VER
 
-#endif
+#endif  // KREUZBERG_RUBY_STRINGS_H

data/ext/kreuzberg_rb/native/include/unistd.h

@@ -44,4 +44,4 @@ typedef long ssize_t;
 #include_next <unistd.h>
 #endif
 
-#endif
+#endif  // KREUZBERG_RUBY_UNISTD_H