kreuzberg 4.3.5-aarch64-linux

data/Steepfile

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Steep configuration for Kreuzberg Ruby package
+
+target :lib do
+  signature 'sig'
+
+  check 'lib'
+
+  # Core library
+  library 'pathname'
+  library 'json'
+  library 'fileutils'
+  library 'open3'
+
+  # Strategic ignores for steep limitations (not fixable, safe to ignore):
+
+  # 1. Sorbet type annotations - Steep doesn't recognize Sorbet's T::Struct and T::Sig
+  #    This file uses Sorbet exclusively for type definitions
+  ignore 'lib/kreuzberg/types.rb'
+
+  # 2. Struct.new with keyword_init - steep cannot understand implicit attr_readers
+  #    defined by Struct.new in blocks (Table and Chunk classes)
+  ignore 'lib/kreuzberg/result.rb'
+
+  # 3. Generic type parameters in normalize_config - steep has difficulty with
+  #    methods that take Class as parameter and return instances
+  ignore 'lib/kreuzberg/config.rb'
+
+  # 4. Interface types - steep doesn't recognize that all Ruby objects have nil? and is_a?
+  #    even for interface types like _ToH
+  ignore 'lib/kreuzberg/extraction_api.rb'
+
+  # 5. Open3 methods - steep's built-in Open3 RBS signatures incomplete
+  #    (capture2, capture3, popen3 are standard library methods)
+  ignore 'lib/kreuzberg/setup_lib_path.rb'
+  ignore 'lib/kreuzberg/cli_proxy.rb'
+  ignore 'lib/kreuzberg/mcp_proxy.rb'
+
+  # Ignore Rust extension methods (defined in native code)
+  ignore 'ext'
+
+  # Ignore vendored code
+  ignore 'vendor'
+
+  # Ignore test files
+  ignore 'spec'
+
+  # Ignore examples
+  ignore 'examples'
+end

data/examples/async_patterns.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+require 'kreuzberg'
+
+# NOTE: Ruby bindings use Tokio runtime with block_on() internally.
+# The "async" functions block the Ruby GVL during execution, so there's
+# no performance benefit over the _sync variants from Ruby's perspective.
+
+# ============================================================================
+# Pattern 1: Synchronous Extraction (Recommended)
+# ============================================================================
+
+def basic_sync_extraction
+  result = Kreuzberg.extract_file_sync('document.pdf')
+  puts "Content: #{result[:content]}"
+  puts "MIME type: #{result[:mime_type]}"
+end
+
+# ============================================================================
+# Pattern 2: "Async" Extraction (Same Performance as Sync)
+# ============================================================================
+
+def basic_async_extraction
+  result = Kreuzberg.extract_file('document.pdf')
+  puts "Content: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 3: Concurrent Processing with Ruby Threads
+# ============================================================================
+
+def concurrent_with_threads
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  threads = files.map do |file|
+    Thread.new do
+      Kreuzberg.extract_file_sync(file)
+    end
+  end
+
+  results = threads.map(&:value)
+  results.each_with_index do |result, index|
+    puts "File #{index + 1}: #{result[:content][0..100]}"
+  end
+end
+
+# ============================================================================
+# Pattern 4: Batch Processing (Preferred for Multiple Files)
+# ============================================================================
+
+def batch_processing
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  results = Kreuzberg.batch_extract_files_sync(files)
+
+  puts "Processed #{results.length} files"
+  results.each do |result|
+    puts "Content preview: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 5: Extraction with Configuration
+# ============================================================================
+
+def extraction_with_config
+  config = {
+    ocr: {
+      backend: 'tesseract',
+      language: 'eng'
+    },
+    force_ocr: false
+  }
+
+  result = Kreuzberg.extract_file_sync('scanned.pdf', **config)
+  puts "Extracted with OCR: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 6: Extract from Bytes
+# ============================================================================
+
+def extract_from_bytes
+  data = File.binread('document.pdf')
+  result = Kreuzberg.extract_bytes_sync(data, 'application/pdf')
+  puts "Extracted from memory: #{result[:content]}"
+end
+
+# ============================================================================
+# Pattern 7: Batch Extract from Bytes
+# ============================================================================
+
+def batch_extract_from_bytes
+  files = ['doc1.pdf', 'doc2.pdf']
+  bytes_array = files.map { |f| File.binread(f) }
+  mime_types = ['application/pdf', 'application/pdf']
+
+  results = Kreuzberg.batch_extract_bytes_sync(bytes_array, mime_types)
+  puts "Processed #{results.length} files from memory"
+end
+
+# ============================================================================
+# Pattern 8: Error Handling
+# ============================================================================
+
+def error_handling
+  Kreuzberg.extract_file_sync('nonexistent.pdf')
+rescue StandardError => e
+  puts "Extraction failed: #{e.message}"
+end
+
+# ============================================================================
+# Pattern 9: Sequential Processing
+# ============================================================================
+
+def sequential_processing
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf']
+
+  files.each do |file|
+    result = Kreuzberg.extract_file_sync(file)
+    puts "Processed #{file}: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 10: Background Processing with ActiveJob (Rails)
+# ============================================================================
+
+# Example ActiveJob for async processing in Rails
+# < ApplicationJob
+class DocumentExtractionJob
+  def perform(file_path)
+    result = Kreuzberg.extract_file_sync(file_path)
+    puts "Background extraction complete: #{result[:content][0..100]}"
+  end
+end
+
+# Usage in Rails controller:
+# DocumentExtractionJob.perform_later('document.pdf')
+
+# ============================================================================
+# Pattern 11: Concurrent Processing with Parallel Gem
+# ============================================================================
+
+def concurrent_with_parallel_gem
+  require 'parallel'
+
+  files = ['doc1.pdf', 'doc2.pdf', 'doc3.pdf', 'doc4.pdf']
+
+  results = Parallel.map(files, in_processes: 4) do |file|
+    Kreuzberg.extract_file_sync(file)
+  end
+
+  results.each do |result|
+    puts "Content: #{result[:content][0..50]}"
+  end
+end
+
+# ============================================================================
+# Pattern 12: Timeout Wrapper
+# ============================================================================
+
+def extraction_with_timeout(file_path, timeout_seconds = 30)
+  require 'timeout'
+
+  Timeout.timeout(timeout_seconds) do
+    Kreuzberg.extract_file_sync(file_path)
+  end
+rescue Timeout::Error
+  puts "Extraction timed out after #{timeout_seconds} seconds"
+  nil
+end
+
+# ============================================================================
+# Pattern 13: Custom Ruby PostProcessor Plugin
+# ============================================================================
+
+def register_postprocessor
+  uppercase_processor = lambda do |result|
+    result[:content] = result[:content].upcase
+    result
+  end
+
+  Kreuzberg.register_post_processor('uppercase', uppercase_processor, 100)
+
+  result = Kreuzberg.extract_file_sync('document.pdf')
+  puts "Uppercase content: #{result[:content]}"
+
+  Kreuzberg.unregister_post_processor('uppercase')
+end
+
+# ============================================================================
+# Pattern 14: Custom Ruby Validator Plugin
+# ============================================================================
+
+def register_validator
+  min_length_validator = lambda do |result|
+    raise 'Content too short' if result[:content].length < 100
+  end
+
+  Kreuzberg.register_validator('min_length', min_length_validator, 100)
+
+  begin
+    result = Kreuzberg.extract_file_sync('short_document.pdf')
+    puts "Validation passed: #{result[:content]}"
+  rescue StandardError => e
+    puts "Validation failed: #{e.message}"
+  end
+
+  Kreuzberg.unregister_validator('min_length')
+end
+
+# ============================================================================
+# Pattern 15: Custom Ruby OCR Backend Plugin
+# ============================================================================
+
+# Example OCR backend implementation for custom processing.
+class CustomOcrBackend
+  def process_image(image_bytes, language)
+    "Extracted text from #{image_bytes.length} bytes using #{language}"
+  end
+
+  def supports_language?(lang)
+    %w[eng deu fra].include?(lang)
+  end
+end
+
+def register_ocr_backend
+  backend = CustomOcrBackend.new
+  Kreuzberg.register_ocr_backend('custom', backend)
+
+  config = {
+    ocr: {
+      backend: 'custom',
+      language: 'eng'
+    },
+    force_ocr: true
+  }
+
+  result = Kreuzberg.extract_file_sync('scanned.pdf', **config)
+  puts "Custom OCR result: #{result[:content]}"
+end
+
+# ============================================================================
+# Main Demonstration
+# ============================================================================
+
+def main
+  puts '=== Basic Sync Extraction ==='
+  basic_sync_extraction
+
+  puts '\n=== Basic Async Extraction (Blocks GVL) ==='
+  basic_async_extraction
+
+  puts '\n=== Concurrent with Ruby Threads ==='
+  concurrent_with_threads
+
+  puts '\n=== Batch Processing (Preferred) ==='
+  batch_processing
+
+  puts '\n=== Extraction with Config ==='
+  extraction_with_config
+
+  puts '\n=== Extract from Bytes ==='
+  extract_from_bytes
+
+  puts '\n=== Error Handling ==='
+  error_handling
+
+  puts '\n=== Sequential Processing ==='
+  sequential_processing
+
+  puts '\n=== Extraction with Timeout ==='
+  extraction_with_timeout('large_document.pdf', 30)
+
+  puts '\n=== Custom PostProcessor ==='
+  register_postprocessor
+
+  puts '\n=== Custom Validator ==='
+  register_validator
+end
+
+main if __FILE__ == $PROGRAM_NAME

data/extconf.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+require 'rbconfig'
+require 'fileutils'
+
+if Gem.win_platform?
+  # Use CI-provided CARGO_TARGET_DIR if available, otherwise use a short path
+  # GitHub Actions sets CARGO_TARGET_DIR=C:\t for MAX_PATH mitigation
+  if ENV['CARGO_TARGET_DIR']
+    puts "Windows detected: Using existing CARGO_TARGET_DIR=#{ENV['CARGO_TARGET_DIR']}"
+  else
+    short_target_dir = Dir.exist?('C:/t') ? 'C:/t' : 'C:/kz-build'
+    begin
+      FileUtils.mkdir_p(short_target_dir)
+      ENV['CARGO_TARGET_DIR'] = short_target_dir
+      ENV['OUT_DIR'] = short_target_dir
+      puts "Windows detected: Using short build path #{short_target_dir}"
+    rescue StandardError => e
+      puts "Warning: Could not create short path #{short_target_dir}: #{e.message}"
+    end
+  end
+end
+
+if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
+  devkit = ENV.fetch('RI_DEVKIT', nil)
+  prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
+
+  # Set up include paths for MSVC compatibility headers
+  native_include = File.expand_path('ext/kreuzberg_rb/native/include', __dir__).tr('\\', '/')
+  compat_include = File.expand_path('ext/kreuzberg_rb/native/include/msvc_compat', __dir__).tr('\\', '/')
+
+  extra_args = []
+  extra_args << "-I#{native_include}"
+  extra_args << "-I#{compat_include}"
+  extra_args << '-fms-extensions'
+  extra_args << '-fno-omit-frame-pointer'
+
+  if devkit
+    sysroot = "#{devkit}#{prefix}".tr('\\', '/')
+    extra_args.push('--target=x86_64-pc-windows-gnu', "--sysroot=#{sysroot}")
+  end
+
+  unless extra_args.empty?
+    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/).reject(&:empty?)
+    ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
+    puts "BINDGEN_EXTRA_CLANG_ARGS set to: #{ENV.fetch('BINDGEN_EXTRA_CLANG_ARGS', nil)}"
+  end
+
+  # Set target for Windows GNU toolchain if not already set
+  ENV['CARGO_BUILD_TARGET'] ||= 'x86_64-pc-windows-gnu' if devkit || ENV['MSYSTEM']
+end
+
+default_profile = ENV.fetch('CARGO_PROFILE', 'release')
+
+create_rust_makefile('kreuzberg_rb') do |config|
+  config.profile = default_profile.to_sym
+  config.ext_dir = File.expand_path('ext/kreuzberg_rb/native', __dir__)
+end

data/kreuzberg.gemspec

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+
+require_relative 'lib/kreuzberg/version'
+
+repo_root = File.expand_path('../..', __dir__)
+
+ruby_prefix = 'packages/ruby/'
+ruby_cmd = %(git -C "#{repo_root}" ls-files -z #{ruby_prefix})
+ruby_files =
+  `#{ruby_cmd}`.split("\x0")
+               .select { |path| path.start_with?(ruby_prefix) }
+               .map { |path| path.delete_prefix(ruby_prefix) }
+
+core_prefix = 'crates/kreuzberg/'
+core_cmd = %(git -C "#{repo_root}" ls-files -z #{core_prefix})
+core_files =
+  `#{core_cmd}`.split("\x0")
+               .select { |path| path.start_with?(core_prefix) }
+               .map { |path| path.delete_prefix('crates/') }
+               .map { |path| "vendor/#{path}" }
+
+ffi_prefix = 'crates/kreuzberg-ffi/'
+ffi_cmd = %(git -C "#{repo_root}" ls-files -z #{ffi_prefix})
+ffi_files =
+  `#{ffi_cmd}`.split("\x0")
+              .select { |path| path.start_with?(ffi_prefix) }
+              .map { |path| path.delete_prefix('crates/') }
+              .map { |path| "vendor/#{path}" }
+
+fallback_files = Dir.chdir(__dir__) do
+  ruby_fallback = Dir.glob(
+    %w[
+      README.md
+      LICENSE
+      ext/**/*.rs
+      ext/**/*.rb
+      ext/**/*.toml
+      ext/**/*.lock
+      ext/**/*.md
+      ext/**/build.rs
+      ext/**/Cargo.*
+      exe/*
+      lib/**/*.rb
+      sig/**/*.rbs
+      spec/**/*.rb
+    ],
+    File::FNM_DOTMATCH
+  )
+
+  core_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/.fastembed_cache/') }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  ffi_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  tesseract_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg-tesseract/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  paddle_ocr_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg-paddle-ocr/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  pdfium_render_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg-pdfium-render/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .reject { |f| f.include?('/target/') }
+       .grep_v(/\.(swp|bak|tmp)$/)
+       .grep_v(/~$/)
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  ruby_fallback + core_fallback + ffi_fallback + tesseract_fallback + paddle_ocr_fallback + pdfium_render_fallback
+end
+
+vendor_files = Dir.chdir(__dir__) do
+  kreuzberg_files = if Dir.exist?('vendor/kreuzberg')
+                      Dir.glob('vendor/kreuzberg/**/*', File::FNM_DOTMATCH)
+                         .reject { |f| File.directory?(f) }
+                         .reject { |f| f.include?('/.fastembed_cache/') }
+                         .reject { |f| f.include?('/.kreuzberg/') }
+                         .reject { |f| f.include?('/target/') }
+                         .grep_v(/\.(swp|bak|tmp)$/)
+                         .grep_v(/~$/)
+                    else
+                      []
+                    end
+
+  kreuzberg_ffi_files = if Dir.exist?('vendor/kreuzberg-ffi')
+                          Dir.glob('vendor/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
+                             .reject { |f| File.directory?(f) }
+                             .reject { |f| f.include?('/target/') }
+                             .grep_v(/\.(swp|bak|tmp)$/)
+                             .grep_v(/~$/)
+                        else
+                          []
+                        end
+
+  kreuzberg_tesseract_files = if Dir.exist?('vendor/kreuzberg-tesseract')
+                                Dir.glob('vendor/kreuzberg-tesseract/**/*', File::FNM_DOTMATCH)
+                                   .reject { |f| File.directory?(f) }
+                                   .reject { |f| f.include?('/target/') }
+                                   .grep_v(/\.(swp|bak|tmp)$/)
+                                   .grep_v(/~$/)
+                              else
+                                []
+                              end
+
+  kreuzberg_paddle_ocr_files = if Dir.exist?('vendor/kreuzberg-paddle-ocr')
+                                 Dir.glob('vendor/kreuzberg-paddle-ocr/**/*', File::FNM_DOTMATCH)
+                                    .reject { |f| File.directory?(f) }
+                                    .reject { |f| f.include?('/target/') }
+                                    .grep_v(/\.(swp|bak|tmp)$/)
+                                    .grep_v(/~$/)
+                               else
+                                 []
+                               end
+
+  kreuzberg_pdfium_render_files = if Dir.exist?('vendor/kreuzberg-pdfium-render')
+                                    Dir.glob('vendor/kreuzberg-pdfium-render/**/*', File::FNM_DOTMATCH)
+                                       .reject { |f| File.directory?(f) }
+                                       .reject { |f| f.include?('/target/') }
+                                       .grep_v(/\.(swp|bak|tmp)$/)
+                                       .grep_v(/~$/)
+                                  else
+                                    []
+                                  end
+
+  rb_sys_files = if Dir.exist?('vendor/rb-sys')
+                   Dir.glob('vendor/rb-sys/**/*', File::FNM_DOTMATCH)
+                      .reject { |f| File.directory?(f) }
+                      .reject { |f| f.include?('/target/') }
+                      .grep_v(/\.(swp|bak|tmp)$/)
+                      .grep_v(/~$/)
+                 else
+                   []
+                 end
+
+  workspace_toml = if File.exist?('vendor/Cargo.toml')
+                     ['vendor/Cargo.toml']
+                   else
+                     []
+                   end
+
+  kreuzberg_files + kreuzberg_ffi_files + kreuzberg_tesseract_files +
+    kreuzberg_paddle_ocr_files + kreuzberg_pdfium_render_files + rb_sys_files + workspace_toml
+end
+
+# When vendor files exist, get ext/ files from filesystem (to include modified Cargo.toml
+# with vendor paths) instead of from git (which has original 5-level crate paths)
+ext_files_from_fs = Dir.chdir(__dir__) do
+  Dir.glob('ext/**/*', File::FNM_DOTMATCH)
+     .reject { |f| File.directory?(f) }
+     .reject { |f| f.include?('/target/') }
+     .grep_v(/\.(swp|bak|tmp)$/)
+     .grep_v(/~$/)
+end
+
+files = if (ruby_files + core_files + ffi_files).empty?
+          fallback_files
+        elsif vendor_files.any?
+          # Use ext/ files from filesystem (modified by vendor script) + non-ext ruby files from git
+          non_ext_ruby_files = ruby_files.reject { |f| f.start_with?('ext/') }
+          non_ext_ruby_files + ext_files_from_fs + vendor_files
+        else
+          ruby_files + core_files + ffi_files
+        end
+
+native_artifacts = Dir.chdir(__dir__) do
+  Dir.glob(%w[
+             lib/**/*.bundle
+             lib/**/*.so
+             lib/**/*.dll
+             lib/**/*.dylib
+           ])
+end
+files.concat(native_artifacts)
+
+files = files.select { |f| File.exist?(f) }
+files = files.uniq
+
+Gem::Specification.new do |spec|
+  spec.name = 'kreuzberg'
+  spec.version = Kreuzberg::VERSION
+  spec.authors = ['Na\'aman Hirschfeld']
+  spec.email = ['nhirschfeld@gmail.com']
+
+  spec.summary = 'Document intelligence library — extract text from PDFs, Office docs, images, and 75+ formats'
+  spec.description = <<~DESC
+    Kreuzberg is a high-performance document intelligence library with a Rust core and native
+    Ruby bindings via Magnus. Extract text, metadata, and structured data from 75+ file formats
+    including PDF, DOCX, PPTX, XLSX, HTML, RTF, images (with OCR), email, archives, and more.
+    Features async/sync APIs, text chunking, language detection, and keyword extraction.
+  DESC
+  spec.homepage = 'https://github.com/kreuzberg-dev/kreuzberg'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.2.0', '< 5.0'
+
+  spec.metadata = {
+    'homepage_uri' => spec.homepage,
+    'source_code_uri' => 'https://github.com/kreuzberg-dev/kreuzberg',
+    'changelog_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/blob/main/CHANGELOG.md',
+    'documentation_uri' => 'https://docs.kreuzberg.dev',
+    'bug_tracker_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/issues',
+    'rubygems_mfa_required' => 'true',
+    'keywords' => 'document-intelligence,document-extraction,text-extraction,ocr,pdf,rust,native-extension,nlp,rag'
+  }
+
+  spec.files = files
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.extensions = ['ext/kreuzberg_rb/extconf.rb']
+
+  spec.add_dependency 'rb_sys', '~> 0.9.119'
+
+  spec.add_development_dependency 'bundler', '~> 4.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rake-compiler', '~> 1.2'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  spec.add_development_dependency 'sorbet-runtime', '~> 0.5'
+  unless Gem.win_platform?
+    spec.add_development_dependency 'rbs', '~> 3.0'
+    spec.add_development_dependency 'rubocop', '~> 1.66'
+    spec.add_development_dependency 'rubocop-performance', '~> 1.21'
+    spec.add_development_dependency 'rubocop-rspec', '~> 3.0'
+    spec.add_development_dependency 'steep', '~> 1.8'
+  end
+  spec.add_development_dependency 'yard', '~> 0.9'
+end