kreuzberg 4.0.0.pre.rc.29 → 4.0.0.rc1

data/extconf.rb

@@ -3,53 +3,21 @@
 require 'mkmf'
 require 'rb_sys/mkmf'
 require 'rbconfig'
-require 'fileutils'
-
-if Gem.win_platform?
-  # Use CI-provided CARGO_TARGET_DIR if available, otherwise use a short path
-  # GitHub Actions sets CARGO_TARGET_DIR=C:\t for MAX_PATH mitigation
-  if ENV['CARGO_TARGET_DIR']
-    puts "Windows detected: Using existing CARGO_TARGET_DIR=#{ENV['CARGO_TARGET_DIR']}"
-  else
-    short_target_dir = Dir.exist?('C:/t') ? 'C:/t' : 'C:/kz-build'
-    begin
-      FileUtils.mkdir_p(short_target_dir)
-      ENV['CARGO_TARGET_DIR'] = short_target_dir
-      ENV['OUT_DIR'] = short_target_dir
-      puts "Windows detected: Using short build path #{short_target_dir}"
-    rescue StandardError => e
-      puts "Warning: Could not create short path #{short_target_dir}: #{e.message}"
-    end
-  end
-end
 
 if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
   devkit = ENV.fetch('RI_DEVKIT', nil)
   prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
 
-  # Set up include paths for MSVC compatibility headers
-  native_include = File.expand_path('ext/kreuzberg_rb/native/include', __dir__).tr('\\', '/')
-  compat_include = File.expand_path('ext/kreuzberg_rb/native/include/msvc_compat', __dir__).tr('\\', '/')
-
-  extra_args = []
-  extra_args << "-I#{native_include}"
-  extra_args << "-I#{compat_include}"
-  extra_args << '-fms-extensions'
-  extra_args << '-fno-omit-frame-pointer'
-
   if devkit
-    sysroot = "#{devkit}#{prefix}".tr('\\', '/')
-    extra_args.push('--target=x86_64-pc-windows-gnu', "--sysroot=#{sysroot}")
-  end
+    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
+    extra_args = [
+      '--target=x86_64-pc-windows-gnu',
+      "--sysroot=#{sysroot}"
+    ]
 
-  unless extra_args.empty?
-    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/).reject(&:empty?)
+    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/)
     ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
-    puts "BINDGEN_EXTRA_CLANG_ARGS set to: #{ENV.fetch('BINDGEN_EXTRA_CLANG_ARGS', nil)}"
   end
-
-  # Set target for Windows GNU toolchain if not already set
-  ENV['CARGO_BUILD_TARGET'] ||= 'x86_64-pc-windows-gnu' if devkit || ENV['MSYSTEM']
 end
 
 default_profile = ENV.fetch('CARGO_PROFILE', 'release')

data/kreuzberg.gemspec

@@ -4,26 +4,20 @@ require_relative 'lib/kreuzberg/version'
 
 repo_root = File.expand_path('../..', __dir__)
 
+# Include files from packages/ruby
 ruby_prefix = 'packages/ruby/'
 ruby_cmd = %(git -C "#{repo_root}" ls-files -z #{ruby_prefix})
 ruby_files =
   `#{ruby_cmd}`.split("\x0")
-               .select { |path| path.start_with?(ruby_prefix) }
-               .map { |path| path.delete_prefix(ruby_prefix) }
+              .select { |path| path.start_with?(ruby_prefix) }
+              .map { |path| path.delete_prefix(ruby_prefix) }
 
+# Include the kreuzberg core crate (needed for path patch in Cargo.toml)
 core_prefix = 'crates/kreuzberg/'
 core_cmd = %(git -C "#{repo_root}" ls-files -z #{core_prefix})
 core_files =
   `#{core_cmd}`.split("\x0")
-               .select { |path| path.start_with?(core_prefix) }
-               .map { |path| path.delete_prefix('crates/') }
-               .map { |path| "vendor/#{path}" }
-
-ffi_prefix = 'crates/kreuzberg-ffi/'
-ffi_cmd = %(git -C "#{repo_root}" ls-files -z #{ffi_prefix})
-ffi_files =
-  `#{ffi_cmd}`.split("\x0")
-              .select { |path| path.start_with?(ffi_prefix) }
+              .select { |path| path.start_with?(core_prefix) }
               .map { |path| path.delete_prefix('crates/') }
               .map { |path| "vendor/#{path}" }
 
@@ -47,109 +41,17 @@ fallback_files = Dir.chdir(__dir__) do
     File::FNM_DOTMATCH
   )
 
+  # Fallback for core crate - copy from repo root
   core_fallback = Dir.chdir(repo_root) do
     Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
-       .reject { |f| File.directory?(f) }
-       .reject { |f| f.include?('/.fastembed_cache/') }
-       .reject { |f| f.include?('/target/') }
-       .grep_v(/\.(swp|bak|tmp)$/)
-       .grep_v(/~$/)
-       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
-  end
-
-  ffi_fallback = Dir.chdir(repo_root) do
-    Dir.glob('crates/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
-       .reject { |f| File.directory?(f) }
-       .reject { |f| f.include?('/target/') }
-       .grep_v(/\.(swp|bak|tmp)$/)
-       .grep_v(/~$/)
-       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
-  end
-
-  tesseract_fallback = Dir.chdir(repo_root) do
-    Dir.glob('crates/kreuzberg-tesseract/**/*', File::FNM_DOTMATCH)
-       .reject { |f| File.directory?(f) }
-       .reject { |f| f.include?('/target/') }
-       .grep_v(/\.(swp|bak|tmp)$/)
-       .grep_v(/~$/)
-       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+      .reject { |f| File.directory?(f) }
+      .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
   end
 
-  ruby_fallback + core_fallback + ffi_fallback + tesseract_fallback
-end
-
-vendor_files = Dir.chdir(__dir__) do
-  kreuzberg_files = if Dir.exist?('vendor/kreuzberg')
-                      Dir.glob('vendor/kreuzberg/**/*', File::FNM_DOTMATCH)
-                         .reject { |f| File.directory?(f) }
-                         .reject { |f| f.include?('/.fastembed_cache/') }
-                         .reject { |f| f.include?('/.kreuzberg/') }
-                         .reject { |f| f.include?('/target/') }
-                         .grep_v(/\.(swp|bak|tmp)$/)
-                         .grep_v(/~$/)
-                    else
-                      []
-                    end
-
-  kreuzberg_ffi_files = if Dir.exist?('vendor/kreuzberg-ffi')
-                          Dir.glob('vendor/kreuzberg-ffi/**/*', File::FNM_DOTMATCH)
-                             .reject { |f| File.directory?(f) }
-                             .reject { |f| f.include?('/target/') }
-                             .grep_v(/\.(swp|bak|tmp)$/)
-                             .grep_v(/~$/)
-                        else
-                          []
-                        end
-
-  kreuzberg_tesseract_files = if Dir.exist?('vendor/kreuzberg-tesseract')
-                                Dir.glob('vendor/kreuzberg-tesseract/**/*', File::FNM_DOTMATCH)
-                                   .reject { |f| File.directory?(f) }
-                                   .reject { |f| f.include?('/target/') }
-                                   .grep_v(/\.(swp|bak|tmp)$/)
-                                   .grep_v(/~$/)
-                              else
-                                []
-                              end
-
-  rb_sys_files = if Dir.exist?('vendor/rb-sys')
-                   Dir.glob('vendor/rb-sys/**/*', File::FNM_DOTMATCH)
-                      .reject { |f| File.directory?(f) }
-                      .reject { |f| f.include?('/target/') }
-                      .grep_v(/\.(swp|bak|tmp)$/)
-                      .grep_v(/~$/)
-                 else
-                   []
-                 end
-
-  workspace_toml = if File.exist?('vendor/Cargo.toml')
-                     ['vendor/Cargo.toml']
-                   else
-                     []
-                   end
-
-  kreuzberg_files + kreuzberg_ffi_files + kreuzberg_tesseract_files + rb_sys_files + workspace_toml
+  ruby_fallback + core_fallback
 end
 
-files = if (ruby_files + core_files + ffi_files).empty?
-          fallback_files
-        elsif vendor_files.any?
-          ruby_files + vendor_files
-        else
-          ruby_files + core_files + ffi_files
-        end
-
-native_artifacts = Dir.chdir(__dir__) do
-  Dir.glob(%w[
-             lib/**/*.bundle
-             lib/**/*.so
-             lib/**/*.dll
-             lib/**/*.dylib
-           ])
-end
-files.concat(native_artifacts)
-
-files = files.select { |f| File.exist?(f) }
-files = files.uniq
+files = (ruby_files + core_files).empty? ? fallback_files : (ruby_files + core_files)
 
 Gem::Specification.new do |spec|
   spec.name = 'kreuzberg'
@@ -163,16 +65,16 @@ Gem::Specification.new do |spec|
     Rust core. Supports extraction, OCR, chunking, and language detection for 30+ file formats
     including PDF, DOCX, PPTX, XLSX, images, and more.
   DESC
-  spec.homepage = 'https://github.com/kreuzberg-dev/kreuzberg'
+  spec.homepage = 'https://github.com/Goldziher/kreuzberg'
   spec.license = 'MIT'
   spec.required_ruby_version = '>= 3.2.0'
 
   spec.metadata = {
     'homepage_uri' => spec.homepage,
-    'source_code_uri' => 'https://github.com/kreuzberg-dev/kreuzberg',
-    'changelog_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/blob/main/CHANGELOG.md',
+    'source_code_uri' => 'https://github.com/Goldziher/kreuzberg',
+    'changelog_uri' => 'https://github.com/Goldziher/kreuzberg/blob/main/CHANGELOG.md',
     'documentation_uri' => 'https://docs.kreuzberg.dev',
-    'bug_tracker_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/issues',
+    'bug_tracker_uri' => 'https://github.com/Goldziher/kreuzberg/issues',
     'rubygems_mfa_required' => 'true',
     'keywords' => 'document-intelligence,document-extraction,ocr,rust,bindings'
   }
@@ -183,10 +85,14 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
   spec.extensions = ['ext/kreuzberg_rb/extconf.rb']
 
-  spec.add_development_dependency 'bundler', '~> 4.0'
+  # Runtime dependencies
+  # None - the gem is self-contained with the Rust extension
+
+  # Development dependencies
+  spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'rake', '~> 13.0'
   spec.add_development_dependency 'rake-compiler', '~> 1.2'
-  spec.add_development_dependency 'rb_sys', '0.9.119'
+  spec.add_development_dependency 'rb_sys', '~> 0.9'
   spec.add_development_dependency 'rspec', '~> 3.12'
   unless Gem.win_platform?
     spec.add_development_dependency 'rbs', '~> 3.0'

data/lib/kreuzberg/api_proxy.rb

@@ -4,8 +4,22 @@ require 'open3'
 require 'pathname'
 
 module Kreuzberg
+  # API server proxy
+  #
+  # Starts and manages the Kreuzberg API server (Litestar/Python-based or Rust-based).
+  #
   # @example Start the server
+  #   server = Kreuzberg::APIProxy.new(port: 8000)
+  #   server.start
+  #   # Server runs in background
+  #   server.stop
+  #
   # @example With block
+  #   Kreuzberg::APIProxy.run(port: 8000) do |server|
+  #     # Server runs while block executes
+  #     response = Net::HTTP.get(URI('http://localhost:8000/health'))
+  #   end
+  #
   module APIProxy
     Error = Class.new(Kreuzberg::Errors::Error)
     MissingBinaryError = Class.new(Error)
@@ -43,7 +57,7 @@ module Kreuzberg
           err: $stderr
         )
         Process.detach(@pid)
-        sleep 1
+        sleep 1 # Give server time to start
         @pid
       end
 
@@ -56,7 +70,8 @@ module Kreuzberg
 
         Process.kill('TERM', @pid)
         Process.wait(@pid)
-      rescue Errno::ESRCH, Errno::ECHILD # rubocop:disable Lint/SuppressedException
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Process already dead
       ensure
         @pid = nil
       end
@@ -103,6 +118,7 @@ module Kreuzberg
     # @raise [MissingBinaryError] If not found
     #
     def find_api_binary
+      # API might be served by kreuzberg CLI or a separate binary
       binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
       found = CLIProxy.search_paths(binary_name).find(&:file?)
       return found if found

data/lib/kreuzberg/cache_api.rb

@@ -2,33 +2,11 @@
 
 module Kreuzberg
   module CacheAPI
-    # @return [void] No meaningful return value
-    # @example Clear cache
     def clear_cache
       native_clear_cache
       reset_cache_tracker!
     end
 
-    # Retrieve cache statistics.
-    #
-    # Returns information about the current state of the extraction result cache,
-    # including the number of cached entries and total memory used. Statistics include
-    # both native Rust cache metrics and local tracker metrics.
-    #
-    # @return [Hash{Symbol | String => Integer}] Cache statistics hash containing:
-    #   - :total_entries [Integer] Total number of cached extraction results
-    #   - :total_size_bytes [Integer] Total memory used by cached results in bytes
-    #
-    # @example Get cache statistics
-    #   stats = Kreuzberg.cache_stats
-    #   puts "Cached entries: #{stats[:total_entries]}"
-    #   puts "Cache size: #{stats[:total_size_bytes]} bytes"
-    #
-    # @example Check if cache is full
-    #   stats = Kreuzberg.cache_stats
-    #   if stats[:total_size_bytes] > 1_000_000_000  # 1GB
-    #     Kreuzberg.clear_cache
-    #   end
     def cache_stats
       stats = native_cache_stats
       total_entries = (stats['total_entries'] || stats[:total_entries] || 0) + @__cache_tracker[:entries]

data/lib/kreuzberg/cli.rb

@@ -1,8 +1,16 @@
 # frozen_string_literal: true
 
 module Kreuzberg
+  # Command-line interface wrapper
+  #
+  # Provides a Ruby API for the Kreuzberg CLI commands.
+  #
   # @example Extract a file
+  #   Kreuzberg::CLI.extract('document.pdf', output: 'text')
+  #
   # @example Detect file type
+  #   mime_type = Kreuzberg::CLI.detect('document.pdf')
+  #
   module CLI
     module_function
 
@@ -14,8 +22,8 @@ module Kreuzberg
     # @return [String] Extracted content
     #
     def extract(path, output: 'text', ocr: false)
-      args = ['extract', path, '--format', output]
-      args.push('--ocr', ocr ? 'true' : 'false')
+      args = ['extract', path, '--output', output]
+      args << '--ocr' if ocr
       CLIProxy.call(args)
     end
 

data/lib/kreuzberg/cli_proxy.rb

@@ -4,7 +4,14 @@ require 'open3'
 require 'pathname'
 
 module Kreuzberg
+  # CLI binary proxy
+  #
+  # Provides access to the Kreuzberg CLI binary built from crates/kreuzberg-cli.
+  #
   # @example
+  #   output = Kreuzberg::CLIProxy.call(['extract', 'document.pdf'])
+  #   puts output
+  #
   module CLIProxy
     Error = Class.new(Kreuzberg::Errors::Error)
     MissingBinaryError = Class.new(Error)
@@ -89,12 +96,15 @@ module Kreuzberg
     #
     def search_paths(binary_name)
       paths = [
+        # In lib/bin (for packaged gems)
         lib_path.join('bin', binary_name),
         lib_path.join(binary_name),
+        # In local development (packages/ruby)
         root_path.join('../../crates/kreuzberg-cli/target/release', binary_name),
         root_path.join('../../target/release', binary_name)
       ]
 
+      # Try workspace root
       workspace_root = root_path.parent&.parent
       paths << workspace_root.join('target', 'release', binary_name) if workspace_root