kreuzberg 4.0.0.rc1 → 4.0.0.rc2

data/extconf.rb

@@ -1,28 +1,28 @@
-# frozen_string_literal: true
-
-require 'mkmf'
-require 'rb_sys/mkmf'
-require 'rbconfig'
-
-if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
-  devkit = ENV.fetch('RI_DEVKIT', nil)
-  prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
-
-  if devkit
-    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
-    extra_args = [
-      '--target=x86_64-pc-windows-gnu',
-      "--sysroot=#{sysroot}"
-    ]
-
-    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/)
-    ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
-  end
-end
-
-default_profile = ENV.fetch('CARGO_PROFILE', 'release')
-
-create_rust_makefile('kreuzberg_rb') do |config|
-  config.profile = default_profile.to_sym
-  config.ext_dir = File.expand_path('ext/kreuzberg_rb/native', __dir__)
-end
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+require 'rbconfig'
+
+if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
+  devkit = ENV.fetch('RI_DEVKIT', nil)
+  prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
+
+  if devkit
+    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
+    extra_args = [
+      '--target=x86_64-pc-windows-gnu',
+      "--sysroot=#{sysroot}"
+    ]
+
+    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/)
+    ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
+  end
+end
+
+default_profile = ENV.fetch('CARGO_PROFILE', 'release')
+
+create_rust_makefile('kreuzberg_rb') do |config|
+  config.profile = default_profile.to_sym
+  config.ext_dir = File.expand_path('ext/kreuzberg_rb/native', __dir__)
+end

data/kreuzberg.gemspec

@@ -1,105 +1,148 @@
-# frozen_string_literal: true
-
-require_relative 'lib/kreuzberg/version'
-
-repo_root = File.expand_path('../..', __dir__)
-
-# Include files from packages/ruby
-ruby_prefix = 'packages/ruby/'
-ruby_cmd = %(git -C "#{repo_root}" ls-files -z #{ruby_prefix})
-ruby_files =
-  `#{ruby_cmd}`.split("\x0")
-              .select { |path| path.start_with?(ruby_prefix) }
-              .map { |path| path.delete_prefix(ruby_prefix) }
-
-# Include the kreuzberg core crate (needed for path patch in Cargo.toml)
-core_prefix = 'crates/kreuzberg/'
-core_cmd = %(git -C "#{repo_root}" ls-files -z #{core_prefix})
-core_files =
-  `#{core_cmd}`.split("\x0")
-              .select { |path| path.start_with?(core_prefix) }
-              .map { |path| path.delete_prefix('crates/') }
-              .map { |path| "vendor/#{path}" }
-
-fallback_files = Dir.chdir(__dir__) do
-  ruby_fallback = Dir.glob(
-    %w[
-      README.md
-      LICENSE
-      ext/**/*.rs
-      ext/**/*.rb
-      ext/**/*.toml
-      ext/**/*.lock
-      ext/**/*.md
-      ext/**/build.rs
-      ext/**/Cargo.*
-      exe/*
-      lib/**/*.rb
-      sig/**/*.rbs
-      spec/**/*.rb
-    ],
-    File::FNM_DOTMATCH
-  )
-
-  # Fallback for core crate - copy from repo root
-  core_fallback = Dir.chdir(repo_root) do
-    Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
-      .reject { |f| File.directory?(f) }
-      .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
-  end
-
-  ruby_fallback + core_fallback
-end
-
-files = (ruby_files + core_files).empty? ? fallback_files : (ruby_files + core_files)
-
-Gem::Specification.new do |spec|
-  spec.name = 'kreuzberg'
-  spec.version = Kreuzberg::VERSION
-  spec.authors = ['Na\'aman Hirschfeld']
-  spec.email = ['nhirschfeld@gmail.com']
-
-  spec.summary = 'High-performance document intelligence framework'
-  spec.description = <<~DESC
-    Kreuzberg is a multi-language document intelligence framework with a high-performance
-    Rust core. Supports extraction, OCR, chunking, and language detection for 30+ file formats
-    including PDF, DOCX, PPTX, XLSX, images, and more.
-  DESC
-  spec.homepage = 'https://github.com/Goldziher/kreuzberg'
-  spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.2.0'
-
-  spec.metadata = {
-    'homepage_uri' => spec.homepage,
-    'source_code_uri' => 'https://github.com/Goldziher/kreuzberg',
-    'changelog_uri' => 'https://github.com/Goldziher/kreuzberg/blob/main/CHANGELOG.md',
-    'documentation_uri' => 'https://docs.kreuzberg.dev',
-    'bug_tracker_uri' => 'https://github.com/Goldziher/kreuzberg/issues',
-    'rubygems_mfa_required' => 'true',
-    'keywords' => 'document-intelligence,document-extraction,ocr,rust,bindings'
-  }
-
-  spec.files = files
-  spec.bindir = 'exe'
-  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ['lib']
-  spec.extensions = ['ext/kreuzberg_rb/extconf.rb']
-
-  # Runtime dependencies
-  # None - the gem is self-contained with the Rust extension
-
-  # Development dependencies
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'rake', '~> 13.0'
-  spec.add_development_dependency 'rake-compiler', '~> 1.2'
-  spec.add_development_dependency 'rb_sys', '~> 0.9'
-  spec.add_development_dependency 'rspec', '~> 3.12'
-  unless Gem.win_platform?
-    spec.add_development_dependency 'rbs', '~> 3.0'
-    spec.add_development_dependency 'rubocop', '~> 1.66'
-    spec.add_development_dependency 'rubocop-performance', '~> 1.21'
-    spec.add_development_dependency 'rubocop-rspec', '~> 3.0'
-    spec.add_development_dependency 'steep', '~> 1.8'
-  end
-  spec.add_development_dependency 'yard', '~> 0.9'
-end
+# frozen_string_literal: true
+
+require_relative 'lib/kreuzberg/version'
+
+repo_root = File.expand_path('../..', __dir__)
+
+# Include files from packages/ruby
+ruby_prefix = 'packages/ruby/'
+ruby_cmd = %(git -C "#{repo_root}" ls-files -z #{ruby_prefix})
+ruby_files =
+  `#{ruby_cmd}`.split("\x0")
+               .select { |path| path.start_with?(ruby_prefix) }
+               .map { |path| path.delete_prefix(ruby_prefix) }
+
+# Include the kreuzberg core crate (needed for path patch in Cargo.toml)
+core_prefix = 'crates/kreuzberg/'
+core_cmd = %(git -C "#{repo_root}" ls-files -z #{core_prefix})
+core_files =
+  `#{core_cmd}`.split("\x0")
+               .select { |path| path.start_with?(core_prefix) }
+               .map { |path| path.delete_prefix('crates/') }
+               .map { |path| "vendor/#{path}" }
+
+fallback_files = Dir.chdir(__dir__) do
+  ruby_fallback = Dir.glob(
+    %w[
+      README.md
+      LICENSE
+      ext/**/*.rs
+      ext/**/*.rb
+      ext/**/*.toml
+      ext/**/*.lock
+      ext/**/*.md
+      ext/**/build.rs
+      ext/**/Cargo.*
+      exe/*
+      lib/**/*.rb
+      sig/**/*.rbs
+      spec/**/*.rb
+    ],
+    File::FNM_DOTMATCH
+  )
+
+  # Fallback for core crate - copy from repo root
+  core_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  ruby_fallback + core_fallback
+end
+
+# Check for vendored crates (copied during CI/packaging)
+vendor_files = Dir.chdir(__dir__) do
+  kreuzberg_files = if Dir.exist?('vendor/kreuzberg')
+                      Dir.glob('vendor/kreuzberg/**/*', File::FNM_DOTMATCH)
+                         .reject { |f| File.directory?(f) }
+                         .reject { |f| f.include?('/.fastembed_cache/') }
+                         .reject { |f| f.include?('/target/') }
+                         .grep_v(/\.(swp|bak|tmp)$/)
+                         .grep_v(/~$/)
+                    else
+                      []
+                    end
+
+  rb_sys_files = if Dir.exist?('vendor/rb-sys')
+                   Dir.glob('vendor/rb-sys/**/*', File::FNM_DOTMATCH)
+                      .reject { |f| File.directory?(f) }
+                      .reject { |f| f.include?('/target/') }
+                      .grep_v(/\.(swp|bak|tmp)$/)
+                      .grep_v(/~$/)
+                 else
+                   []
+                 end
+
+  workspace_toml = if File.exist?('vendor/Cargo.toml')
+                     ['vendor/Cargo.toml']
+                   else
+                     []
+                   end
+
+  kreuzberg_files + rb_sys_files + workspace_toml
+end
+
+# Use git-tracked files if available, otherwise fallback to glob
+# Always include vendored files if they exist on disk (for CI packaging)
+files = if (ruby_files + core_files).empty?
+          fallback_files
+        elsif vendor_files.any?
+          ruby_files + vendor_files
+        else
+          ruby_files + core_files
+        end
+
+# Filter to only include files that actually exist
+files = files.select { |f| File.exist?(f) }
+
+Gem::Specification.new do |spec|
+  spec.name = 'kreuzberg'
+  spec.version = Kreuzberg::VERSION
+  spec.authors = ['Na\'aman Hirschfeld']
+  spec.email = ['nhirschfeld@gmail.com']
+
+  spec.summary = 'High-performance document intelligence framework'
+  spec.description = <<~DESC
+    Kreuzberg is a multi-language document intelligence framework with a high-performance
+    Rust core. Supports extraction, OCR, chunking, and language detection for 30+ file formats
+    including PDF, DOCX, PPTX, XLSX, images, and more.
+  DESC
+  spec.homepage = 'https://github.com/kreuzberg-dev/kreuzberg'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.2.0'
+
+  spec.metadata = {
+    'homepage_uri' => spec.homepage,
+    'source_code_uri' => 'https://github.com/kreuzberg-dev/kreuzberg',
+    'changelog_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/blob/main/CHANGELOG.md',
+    'documentation_uri' => 'https://docs.kreuzberg.dev',
+    'bug_tracker_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/issues',
+    'rubygems_mfa_required' => 'true',
+    'keywords' => 'document-intelligence,document-extraction,ocr,rust,bindings'
+  }
+
+  spec.files = files
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.extensions = ['ext/kreuzberg_rb/extconf.rb']
+
+  # Runtime dependencies
+  # None - the gem is self-contained with the Rust extension
+
+  # Development dependencies
+  spec.add_development_dependency 'bundler', '~> 4.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rake-compiler', '~> 1.2'
+  spec.add_development_dependency 'rb_sys', '~> 0.9.119'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  unless Gem.win_platform?
+    spec.add_development_dependency 'rbs', '~> 3.0'
+    spec.add_development_dependency 'rubocop', '~> 1.66'
+    spec.add_development_dependency 'rubocop-performance', '~> 1.21'
+    spec.add_development_dependency 'rubocop-rspec', '~> 3.0'
+    spec.add_development_dependency 'steep', '~> 1.8'
+  end
+  spec.add_development_dependency 'yard', '~> 0.9'
+end

data/lib/kreuzberg/api_proxy.rb

@@ -1,142 +1,142 @@
-# frozen_string_literal: true
-
-require 'open3'
-require 'pathname'
-
-module Kreuzberg
-  # API server proxy
-  #
-  # Starts and manages the Kreuzberg API server (Litestar/Python-based or Rust-based).
-  #
-  # @example Start the server
-  #   server = Kreuzberg::APIProxy.new(port: 8000)
-  #   server.start
-  #   # Server runs in background
-  #   server.stop
-  #
-  # @example With block
-  #   Kreuzberg::APIProxy.run(port: 8000) do |server|
-  #     # Server runs while block executes
-  #     response = Net::HTTP.get(URI('http://localhost:8000/health'))
-  #   end
-  #
-  module APIProxy
-    Error = Class.new(Kreuzberg::Errors::Error)
-    MissingBinaryError = Class.new(Error)
-    ServerError = Class.new(Error)
-
-    # API server instance
-    class Server
-      attr_reader :port, :host, :pid
-
-      # Initialize server
-      #
-      # @param port [Integer] Port to run on (default: 8000)
-      # @param host [String] Host to bind to (default: "0.0.0.0")
-      #
-      def initialize(port: 8000, host: '0.0.0.0')
-        @port = port
-        @host = host
-        @pid = nil
-        @process = nil
-      end
-
-      # Start the server in the background
-      #
-      # @return [Integer] Process ID
-      # @raise [ServerError] If server fails to start
-      #
-      def start
-        binary = APIProxy.find_api_binary
-        @pid = spawn(
-          binary.to_s,
-          'api',
-          '--host', @host,
-          '--port', @port.to_s,
-          out: $stdout,
-          err: $stderr
-        )
-        Process.detach(@pid)
-        sleep 1 # Give server time to start
-        @pid
-      end
-
-      # Stop the server
-      #
-      # @return [void]
-      #
-      def stop
-        return unless @pid
-
-        Process.kill('TERM', @pid)
-        Process.wait(@pid)
-      rescue Errno::ESRCH, Errno::ECHILD
-        # Process already dead
-      ensure
-        @pid = nil
-      end
-
-      # Check if server is running
-      #
-      # @return [Boolean]
-      #
-      def running?
-        return false unless @pid
-
-        Process.kill(0, @pid)
-        true
-      rescue Errno::ESRCH, Errno::EPERM
-        false
-      end
-    end
-
-    module_function
-
-    # Run server with a block
-    #
-    # @param port [Integer] Port to run on
-    # @param host [String] Host to bind to
-    # @yield [Server] Yields server instance
-    # @return [Object] Block result
-    #
-    # @example
-    #   Kreuzberg::APIProxy.run(port: 8000) do |server|
-    #     # Make API requests
-    #   end
-    #
-    def run(port: 8000, host: '0.0.0.0')
-      server = Server.new(port: port, host: host)
-      server.start
-      yield server
-    ensure
-      server&.stop
-    end
-
-    # Find the API binary
-    #
-    # @return [Pathname] Path to binary
-    # @raise [MissingBinaryError] If not found
-    #
-    def find_api_binary
-      # API might be served by kreuzberg CLI or a separate binary
-      binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
-      found = CLIProxy.search_paths(binary_name).find(&:file?)
-      return found if found
-
-      raise MissingBinaryError, missing_binary_message
-    end
-
-    # Error message for missing binary
-    #
-    # @return [String]
-    #
-    def missing_binary_message
-      <<~MSG.strip
-        kreuzberg binary not found for API server. Build it with:
-        `cargo build --release --package kreuzberg-cli`
-
-        Or ensure kreuzberg is installed with API support.
-      MSG
-    end
-  end
-end
+# frozen_string_literal: true
+
+require 'open3'
+require 'pathname'
+
+module Kreuzberg
+  # API server proxy
+  #
+  # Starts and manages the Kreuzberg API server (Litestar/Python-based or Rust-based).
+  #
+  # @example Start the server
+  #   server = Kreuzberg::APIProxy.new(port: 8000)
+  #   server.start
+  #   # Server runs in background
+  #   server.stop
+  #
+  # @example With block
+  #   Kreuzberg::APIProxy.run(port: 8000) do |server|
+  #     # Server runs while block executes
+  #     response = Net::HTTP.get(URI('http://localhost:8000/health'))
+  #   end
+  #
+  module APIProxy
+    Error = Class.new(Kreuzberg::Errors::Error)
+    MissingBinaryError = Class.new(Error)
+    ServerError = Class.new(Error)
+
+    # API server instance
+    class Server
+      attr_reader :port, :host, :pid
+
+      # Initialize server
+      #
+      # @param port [Integer] Port to run on (default: 8000)
+      # @param host [String] Host to bind to (default: "0.0.0.0")
+      #
+      def initialize(port: 8000, host: '0.0.0.0')
+        @port = port
+        @host = host
+        @pid = nil
+        @process = nil
+      end
+
+      # Start the server in the background
+      #
+      # @return [Integer] Process ID
+      # @raise [ServerError] If server fails to start
+      #
+      def start
+        binary = APIProxy.find_api_binary
+        @pid = spawn(
+          binary.to_s,
+          'api',
+          '--host', @host,
+          '--port', @port.to_s,
+          out: $stdout,
+          err: $stderr
+        )
+        Process.detach(@pid)
+        sleep 1 # Give server time to start
+        @pid
+      end
+
+      # Stop the server
+      #
+      # @return [void]
+      #
+      def stop
+        return unless @pid
+
+        Process.kill('TERM', @pid)
+        Process.wait(@pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Process already dead
+      ensure
+        @pid = nil
+      end
+
+      # Check if server is running
+      #
+      # @return [Boolean]
+      #
+      def running?
+        return false unless @pid
+
+        Process.kill(0, @pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+    end
+
+    module_function
+
+    # Run server with a block
+    #
+    # @param port [Integer] Port to run on
+    # @param host [String] Host to bind to
+    # @yield [Server] Yields server instance
+    # @return [Object] Block result
+    #
+    # @example
+    #   Kreuzberg::APIProxy.run(port: 8000) do |server|
+    #     # Make API requests
+    #   end
+    #
+    def run(port: 8000, host: '0.0.0.0')
+      server = Server.new(port: port, host: host)
+      server.start
+      yield server
+    ensure
+      server&.stop
+    end
+
+    # Find the API binary
+    #
+    # @return [Pathname] Path to binary
+    # @raise [MissingBinaryError] If not found
+    #
+    def find_api_binary
+      # API might be served by kreuzberg CLI or a separate binary
+      binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
+      found = CLIProxy.search_paths(binary_name).find(&:file?)
+      return found if found
+
+      raise MissingBinaryError, missing_binary_message
+    end
+
+    # Error message for missing binary
+    #
+    # @return [String]
+    #
+    def missing_binary_message
+      <<~MSG.strip
+        kreuzberg binary not found for API server. Build it with:
+        `cargo build --release --package kreuzberg-cli`
+
+        Or ensure kreuzberg is installed with API support.
+      MSG
+    end
+  end
+end