kreuzberg 4.0.0.pre.rc.6

data/extconf.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'mkmf'
+require 'rb_sys/mkmf'
+require 'rbconfig'
+
+if /mswin|mingw/.match?(RbConfig::CONFIG['host_os'])
+  devkit = ENV.fetch('RI_DEVKIT', nil)
+  prefix = ENV['MSYSTEM_PREFIX'] || '/ucrt64'
+
+  if devkit
+    sysroot = "#{devkit}#{prefix}".tr('\\\\', '/')
+    extra_args = [
+      '--target=x86_64-pc-windows-gnu',
+      "--sysroot=#{sysroot}"
+    ]
+
+    existing = ENV['BINDGEN_EXTRA_CLANG_ARGS'].to_s.split(/\s+/)
+    ENV['BINDGEN_EXTRA_CLANG_ARGS'] = (existing + extra_args).uniq.join(' ')
+  end
+end
+
+default_profile = ENV.fetch('CARGO_PROFILE', 'release')
+
+create_rust_makefile('kreuzberg_rb') do |config|
+  config.profile = default_profile.to_sym
+  config.ext_dir = File.expand_path('ext/kreuzberg_rb/native', __dir__)
+end

data/kreuzberg.gemspec

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+require_relative 'lib/kreuzberg/version'
+
+repo_root = File.expand_path('../..', __dir__)
+
+# Include files from packages/ruby
+ruby_prefix = 'packages/ruby/'
+ruby_cmd = %(git -C "#{repo_root}" ls-files -z #{ruby_prefix})
+ruby_files =
+  `#{ruby_cmd}`.split("\x0")
+               .select { |path| path.start_with?(ruby_prefix) }
+               .map { |path| path.delete_prefix(ruby_prefix) }
+
+# Include the kreuzberg core crate (needed for path patch in Cargo.toml)
+core_prefix = 'crates/kreuzberg/'
+core_cmd = %(git -C "#{repo_root}" ls-files -z #{core_prefix})
+core_files =
+  `#{core_cmd}`.split("\x0")
+               .select { |path| path.start_with?(core_prefix) }
+               .map { |path| path.delete_prefix('crates/') }
+               .map { |path| "vendor/#{path}" }
+
+fallback_files = Dir.chdir(__dir__) do
+  ruby_fallback = Dir.glob(
+    %w[
+      README.md
+      LICENSE
+      ext/**/*.rs
+      ext/**/*.rb
+      ext/**/*.toml
+      ext/**/*.lock
+      ext/**/*.md
+      ext/**/build.rs
+      ext/**/Cargo.*
+      exe/*
+      lib/**/*.rb
+      sig/**/*.rbs
+      spec/**/*.rb
+    ],
+    File::FNM_DOTMATCH
+  )
+
+  # Fallback for core crate - copy from repo root
+  core_fallback = Dir.chdir(repo_root) do
+    Dir.glob('crates/kreuzberg/**/*', File::FNM_DOTMATCH)
+       .reject { |f| File.directory?(f) }
+       .map { |path| "vendor/#{path.delete_prefix('crates/')}" }
+  end
+
+  ruby_fallback + core_fallback
+end
+
+# Check for vendored crates (copied during CI/packaging)
+vendor_files = Dir.chdir(__dir__) do
+  kreuzberg_files = if Dir.exist?('vendor/kreuzberg')
+                      Dir.glob('vendor/kreuzberg/**/*', File::FNM_DOTMATCH)
+                         .reject { |f| File.directory?(f) }
+                         .reject { |f| f.include?('/.fastembed_cache/') }
+                         .reject { |f| f.include?('/target/') }
+                         .grep_v(/\.(swp|bak|tmp)$/)
+                         .grep_v(/~$/)
+                    else
+                      []
+                    end
+
+  rb_sys_files = if Dir.exist?('vendor/rb-sys')
+                   Dir.glob('vendor/rb-sys/**/*', File::FNM_DOTMATCH)
+                      .reject { |f| File.directory?(f) }
+                      .reject { |f| f.include?('/target/') }
+                      .grep_v(/\.(swp|bak|tmp)$/)
+                      .grep_v(/~$/)
+                 else
+                   []
+                 end
+
+  workspace_toml = if File.exist?('vendor/Cargo.toml')
+                     ['vendor/Cargo.toml']
+                   else
+                     []
+                   end
+
+  kreuzberg_files + rb_sys_files + workspace_toml
+end
+
+# Use git-tracked files if available, otherwise fallback to glob
+# Always include vendored files if they exist on disk (for CI packaging)
+files = if (ruby_files + core_files).empty?
+          fallback_files
+        elsif vendor_files.any?
+          ruby_files + vendor_files
+        else
+          ruby_files + core_files
+        end
+
+# Filter to only include files that actually exist
+files = files.select { |f| File.exist?(f) }
+
+Gem::Specification.new do |spec|
+  spec.name = 'kreuzberg'
+  spec.version = Kreuzberg::VERSION
+  spec.authors = ['Na\'aman Hirschfeld']
+  spec.email = ['nhirschfeld@gmail.com']
+
+  spec.summary = 'High-performance document intelligence framework'
+  spec.description = <<~DESC
+    Kreuzberg is a multi-language document intelligence framework with a high-performance
+    Rust core. Supports extraction, OCR, chunking, and language detection for 30+ file formats
+    including PDF, DOCX, PPTX, XLSX, images, and more.
+  DESC
+  spec.homepage = 'https://github.com/kreuzberg-dev/kreuzberg'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.2.0'
+
+  spec.metadata = {
+    'homepage_uri' => spec.homepage,
+    'source_code_uri' => 'https://github.com/kreuzberg-dev/kreuzberg',
+    'changelog_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/blob/main/CHANGELOG.md',
+    'documentation_uri' => 'https://docs.kreuzberg.dev',
+    'bug_tracker_uri' => 'https://github.com/kreuzberg-dev/kreuzberg/issues',
+    'rubygems_mfa_required' => 'true',
+    'keywords' => 'document-intelligence,document-extraction,ocr,rust,bindings'
+  }
+
+  spec.files = files
+  spec.bindir = 'exe'
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+  spec.extensions = ['ext/kreuzberg_rb/extconf.rb']
+
+  # Runtime dependencies
+  # None - the gem is self-contained with the Rust extension
+
+  # Development dependencies
+  spec.add_development_dependency 'bundler', '~> 4.0'
+  spec.add_development_dependency 'rake', '~> 13.0'
+  spec.add_development_dependency 'rake-compiler', '~> 1.2'
+  spec.add_development_dependency 'rb_sys', '~> 0.9.119'
+  spec.add_development_dependency 'rspec', '~> 3.12'
+  unless Gem.win_platform?
+    spec.add_development_dependency 'rbs', '~> 3.0'
+    spec.add_development_dependency 'rubocop', '~> 1.66'
+    spec.add_development_dependency 'rubocop-performance', '~> 1.21'
+    spec.add_development_dependency 'rubocop-rspec', '~> 3.0'
+    spec.add_development_dependency 'steep', '~> 1.8'
+  end
+  spec.add_development_dependency 'yard', '~> 0.9'
+end

data/lib/kreuzberg/api_proxy.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'pathname'
+
+module Kreuzberg
+  # API server proxy
+  #
+  # Starts and manages the Kreuzberg API server (Litestar/Python-based or Rust-based).
+  #
+  # @example Start the server
+  #   server = Kreuzberg::APIProxy.new(port: 8000)
+  #   server.start
+  #   # Server runs in background
+  #   server.stop
+  #
+  # @example With block
+  #   Kreuzberg::APIProxy.run(port: 8000) do |server|
+  #     # Server runs while block executes
+  #     response = Net::HTTP.get(URI('http://localhost:8000/health'))
+  #   end
+  #
+  module APIProxy
+    Error = Class.new(Kreuzberg::Errors::Error)
+    MissingBinaryError = Class.new(Error)
+    ServerError = Class.new(Error)
+
+    # API server instance
+    class Server
+      attr_reader :port, :host, :pid
+
+      # Initialize server
+      #
+      # @param port [Integer] Port to run on (default: 8000)
+      # @param host [String] Host to bind to (default: "0.0.0.0")
+      #
+      def initialize(port: 8000, host: '0.0.0.0')
+        @port = port
+        @host = host
+        @pid = nil
+        @process = nil
+      end
+
+      # Start the server in the background
+      #
+      # @return [Integer] Process ID
+      # @raise [ServerError] If server fails to start
+      #
+      def start
+        binary = APIProxy.find_api_binary
+        @pid = spawn(
+          binary.to_s,
+          'api',
+          '--host', @host,
+          '--port', @port.to_s,
+          out: $stdout,
+          err: $stderr
+        )
+        Process.detach(@pid)
+        sleep 1 # Give server time to start
+        @pid
+      end
+
+      # Stop the server
+      #
+      # @return [void]
+      #
+      def stop
+        return unless @pid
+
+        Process.kill('TERM', @pid)
+        Process.wait(@pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        # Process already dead
+      ensure
+        @pid = nil
+      end
+
+      # Check if server is running
+      #
+      # @return [Boolean]
+      #
+      def running?
+        return false unless @pid
+
+        Process.kill(0, @pid)
+        true
+      rescue Errno::ESRCH, Errno::EPERM
+        false
+      end
+    end
+
+    module_function
+
+    # Run server with a block
+    #
+    # @param port [Integer] Port to run on
+    # @param host [String] Host to bind to
+    # @yield [Server] Yields server instance
+    # @return [Object] Block result
+    #
+    # @example
+    #   Kreuzberg::APIProxy.run(port: 8000) do |server|
+    #     # Make API requests
+    #   end
+    #
+    def run(port: 8000, host: '0.0.0.0')
+      server = Server.new(port: port, host: host)
+      server.start
+      yield server
+    ensure
+      server&.stop
+    end
+
+    # Find the API binary
+    #
+    # @return [Pathname] Path to binary
+    # @raise [MissingBinaryError] If not found
+    #
+    def find_api_binary
+      # API might be served by kreuzberg CLI or a separate binary
+      binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
+      found = CLIProxy.search_paths(binary_name).find(&:file?)
+      return found if found
+
+      raise MissingBinaryError, missing_binary_message
+    end
+
+    # Error message for missing binary
+    #
+    # @return [String]
+    #
+    def missing_binary_message
+      <<~MSG.strip
+        kreuzberg binary not found for API server. Build it with:
+        `cargo build --release --package kreuzberg-cli`
+
+        Or ensure kreuzberg is installed with API support.
+      MSG
+    end
+  end
+end

data/lib/kreuzberg/cache_api.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # Provides caching capabilities for extraction results.
+  module CacheAPI
+    def clear_cache
+      native_clear_cache
+      reset_cache_tracker!
+    end
+
+    def cache_stats
+      stats = native_cache_stats
+      total_entries = (stats['total_entries'] || stats[:total_entries] || 0) + @__cache_tracker[:entries]
+      total_size = (stats['total_size_bytes'] || stats[:total_size_bytes] || 0) + @__cache_tracker[:bytes]
+
+      stats['total_entries'] = total_entries
+      stats[:total_entries] = total_entries
+      stats['total_size_bytes'] = total_size
+      stats[:total_size_bytes] = total_size
+
+      stats
+    end
+
+    private
+
+    def record_cache_entry!(results, opts)
+      use_cache = opts.key?(:use_cache) ? opts[:use_cache] : true
+      return unless use_cache
+
+      results_array = results.is_a?(Array) ? results : [results]
+      results_array.each do |result|
+        # @type var result: Result
+        next unless result.respond_to?(:content)
+
+        @__cache_tracker[:entries] += 1
+        @__cache_tracker[:bytes] += result.content.to_s.bytesize
+      end
+    end
+
+    def reset_cache_tracker!
+      @__cache_tracker[:entries] = 0
+      @__cache_tracker[:bytes] = 0
+      nil
+    end
+  end
+end

data/lib/kreuzberg/cli.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # Command-line interface wrapper
+  #
+  # Provides a Ruby API for the Kreuzberg CLI commands.
+  #
+  # @example Extract a file
+  #   Kreuzberg::CLI.extract('document.pdf', output: 'text')
+  #
+  # @example Detect file type
+  #   mime_type = Kreuzberg::CLI.detect('document.pdf')
+  #
+  module CLI
+    module_function
+
+    # Extract content from a file using the CLI
+    #
+    # @param path [String] Path to the file
+    # @param output [String] Output format ("text", "json", "markdown")
+    # @param ocr [Boolean] Enable OCR
+    # @return [String] Extracted content
+    #
+    def extract(path, output: 'text', ocr: false)
+      args = ['extract', path, '--format', output]
+      args.push('--ocr', ocr ? 'true' : 'false')
+      CLIProxy.call(args)
+    end
+
+    # Detect MIME type of a file using the CLI
+    #
+    # @param path [String] Path to the file
+    # @return [String] MIME type
+    #
+    def detect(path)
+      CLIProxy.call(['detect', path]).strip
+    end
+
+    # Get CLI version
+    #
+    # @return [String] Version string
+    #
+    def version
+      CLIProxy.call(['--version']).strip
+    end
+
+    # Get CLI help text
+    #
+    # @return [String] Help text
+    #
+    def help
+      CLIProxy.call(['--help'])
+    end
+  end
+end

data/lib/kreuzberg/cli_proxy.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'open3'
+require 'pathname'
+
+module Kreuzberg
+  # CLI binary proxy
+  #
+  # Provides access to the Kreuzberg CLI binary built from crates/kreuzberg-cli.
+  #
+  # @example
+  #   output = Kreuzberg::CLIProxy.call(['extract', 'document.pdf'])
+  #   puts output
+  #
+  module CLIProxy
+    Error = Class.new(Kreuzberg::Errors::Error)
+    MissingBinaryError = Class.new(Error)
+
+    # CLI execution error with stderr and exit status
+    class CLIExecutionError < Error
+      attr_reader :stderr, :status
+
+      def initialize(message, stderr:, status:)
+        super(message)
+        @stderr = stderr
+        @status = status
+      end
+    end
+
+    module_function
+
+    # Execute the Kreuzberg CLI with given arguments
+    #
+    # @param argv [Array<String>] Command-line arguments
+    # @return [String] Standard output from the CLI
+    # @raise [CLIExecutionError] If the CLI exits with non-zero status
+    # @raise [MissingBinaryError] If the CLI binary is not found
+    #
+    # @example Extract a file
+    #   output = Kreuzberg::CLIProxy.call(['extract', 'document.pdf'])
+    #
+    # @example Detect file type
+    #   output = Kreuzberg::CLIProxy.call(['detect', 'document.pdf'])
+    #
+    def call(argv)
+      binary = find_cli_binary
+      args = Array(argv).map(&:to_s)
+      stdout, stderr, status = Open3.capture3(binary.to_s, *args)
+      return stdout if status.success?
+
+      raise CLIExecutionError.new(
+        "kreuzberg CLI exited with status #{status.exitstatus}",
+        stderr: stderr,
+        status: status.exitstatus
+      )
+    end
+
+    # Find the kreuzberg CLI binary
+    #
+    # Searches in multiple locations:
+    # - crates/kreuzberg-cli/target/release/
+    # - packages/ruby/lib/bin/
+    # - workspace root target/release/
+    #
+    # @return [Pathname] Path to the CLI binary
+    # @raise [MissingBinaryError] If binary not found
+    #
+    def find_cli_binary
+      binary_name = Gem.win_platform? ? 'kreuzberg.exe' : 'kreuzberg'
+      found = search_paths(binary_name).find(&:file?)
+      return found if found
+
+      raise MissingBinaryError, missing_binary_message
+    end
+
+    # Get the root path of the Ruby package
+    #
+    # @return [Pathname] Root path
+    #
+    def root_path
+      @root_path ||= Pathname(__dir__ || '.').join('../..').expand_path
+    end
+
+    # Get the lib path
+    #
+    # @return [Pathname] Lib path
+    #
+    def lib_path
+      @lib_path ||= Pathname(__dir__ || '.').join('..').expand_path
+    end
+
+    # Search paths for the CLI binary
+    #
+    # @param binary_name [String] Name of the binary
+    # @return [Array<Pathname>] List of paths to search
+    #
+    def search_paths(binary_name)
+      paths = [
+        # In lib/bin (for packaged gems)
+        lib_path.join('bin', binary_name),
+        lib_path.join(binary_name),
+        # In local development (packages/ruby)
+        root_path.join('../../crates/kreuzberg-cli/target/release', binary_name),
+        root_path.join('../../target/release', binary_name)
+      ]
+
+      # Try workspace root
+      workspace_root = root_path.parent&.parent
+      paths << workspace_root.join('target', 'release', binary_name) if workspace_root
+
+      paths
+    end
+
+    # Error message when binary is missing
+    #
+    # @return [String] Error message
+    #
+    def missing_binary_message
+      <<~MSG.strip
+        kreuzberg CLI binary not found. Build it with:
+        `cargo build --release --package kreuzberg-cli`
+
+        Or install the gem with pre-built binaries.
+      MSG
+    end
+  end
+end