kreuzberg 4.0.0.pre.rc.6

data/spec/binding/plugins/validator_spec.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Validator Plugin System' do
+  let(:test_pdf) { test_document_path('text/contract_test.txt') }
+
+  after do
+    Kreuzberg.clear_validators
+  end
+
+  describe 'registering validator as Proc' do
+    it 'registers and executes Proc validator during extraction' do
+      validator_called = false
+      validator = lambda do |_result|
+        validator_called = true
+      end
+
+      Kreuzberg.register_validator('check_called', validator)
+      Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(validator_called).to be true
+    end
+
+    it 'allows extraction to proceed when validator passes' do
+      validator = lambda do |result|
+        # Validation passes - do nothing
+      end
+
+      Kreuzberg.register_validator('pass_validator', validator)
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result).to be_a(Kreuzberg::Result)
+      expect(result.content).not_to be_empty
+    end
+
+    it 'prevents extraction when validator raises ValidationError' do
+      validator = lambda do |result|
+        if result['content'].length < 10_000_000
+          raise Kreuzberg::Errors::ValidationError, 'Content too short for this test'
+        end
+      end
+
+      Kreuzberg.register_validator('min_length', validator)
+
+      expect do
+        Kreuzberg.extract_file_sync(test_pdf)
+      end.to raise_error(Kreuzberg::Errors::ValidationError, /Content too short/)
+    end
+  end
+
+  describe 'registering validator as class' do
+    it 'registers and executes class-based validator' do
+      class MinimumLengthValidator
+        include Kreuzberg::ValidatorProtocol
+
+        def initialize(min_length)
+          @min_length = min_length
+        end
+
+        def call(result)
+          return unless result['content'].length < @min_length
+
+          raise Kreuzberg::Errors::ValidationError, "Content too short: #{result['content'].length} < #{@min_length}"
+        end
+      end
+
+      validator = MinimumLengthValidator.new(10)
+      Kreuzberg.register_validator('min_length', validator)
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result).to be_a(Kreuzberg::Result)
+      expect(result.content.length).to be >= 10
+    end
+
+    it 'validates based on content characteristics' do
+      class NonEmptyValidator
+        include Kreuzberg::ValidatorProtocol
+
+        def call(result)
+          return unless result['content'].strip.empty?
+
+          raise Kreuzberg::Errors::ValidationError, 'Content cannot be empty'
+        end
+      end
+
+      validator = NonEmptyValidator.new
+      Kreuzberg.register_validator('non_empty', validator)
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result.content.strip).not_to be_empty
+    end
+  end
+
+  describe 'validator receives correct parameters' do
+    it 'receives result hash with all required fields' do
+      received_result = nil
+      validator = lambda do |result|
+        received_result = result
+      end
+
+      Kreuzberg.register_validator('capture', validator)
+      Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(received_result).to be_a(Hash)
+      expect(received_result).to have_key('content')
+      expect(received_result).to have_key('mime_type')
+      expect(received_result).to have_key('metadata')
+      expect(received_result).to have_key('tables')
+    end
+
+    it 'receives correct content in result hash' do
+      received_content = nil
+      validator = lambda do |result|
+        received_content = result['content']
+      end
+
+      Kreuzberg.register_validator('capture_content', validator)
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(received_content).to eq(result.content)
+    end
+  end
+
+  describe 'multiple validators' do
+    it 'executes all registered validators' do
+      validator1_called = false
+      validator2_called = false
+
+      validator1 = lambda do |_result|
+        validator1_called = true
+      end
+
+      validator2 = lambda do |_result|
+        validator2_called = true
+      end
+
+      Kreuzberg.register_validator('val1', validator1)
+      Kreuzberg.register_validator('val2', validator2)
+      Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(validator1_called).to be true
+      expect(validator2_called).to be true
+    end
+
+    it 'stops execution if any validator fails' do
+      validator1 = lambda do |_result|
+        raise Kreuzberg::Errors::ValidationError, 'First validator failed'
+      end
+
+      validator2 = lambda do |_result|
+        raise StandardError, 'This should not be reached'
+      end
+
+      Kreuzberg.register_validator('fail_first', validator1)
+      Kreuzberg.register_validator('never_reached', validator2)
+
+      expect do
+        Kreuzberg.extract_file_sync(test_pdf)
+      end.to raise_error(Kreuzberg::Errors::ValidationError, /First validator failed/)
+    end
+  end
+
+  describe 'unregister_validator' do
+    it 'removes a registered validator by name' do
+      validator = lambda do |_result|
+        raise Kreuzberg::Errors::ValidationError, 'Should not be called'
+      end
+
+      Kreuzberg.register_validator('removable', validator)
+      Kreuzberg.unregister_validator('removable')
+
+      expect do
+        Kreuzberg.extract_file_sync(test_pdf)
+      end.not_to raise_error
+    end
+
+    it 'does not affect other registered validators' do
+      validator1_called = false
+      validator3_called = false
+
+      validator1 = lambda do |_result|
+        validator1_called = true
+      end
+
+      validator2 = lambda do |_result|
+        raise Kreuzberg::Errors::ValidationError, 'Should not be called'
+      end
+
+      validator3 = lambda do |_result|
+        validator3_called = true
+      end
+
+      Kreuzberg.register_validator('keep1', validator1)
+      Kreuzberg.register_validator('remove', validator2)
+      Kreuzberg.register_validator('keep3', validator3)
+
+      Kreuzberg.unregister_validator('remove')
+      Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(validator1_called).to be true
+      expect(validator3_called).to be true
+    end
+  end
+
+  describe 'clear_validators' do
+    it 'removes all registered validators' do
+      validator1 = lambda do |_result|
+        raise Kreuzberg::Errors::ValidationError, 'Should not be called 1'
+      end
+
+      validator2 = lambda do |_result|
+        raise Kreuzberg::Errors::ValidationError, 'Should not be called 2'
+      end
+
+      Kreuzberg.register_validator('val1', validator1)
+      Kreuzberg.register_validator('val2', validator2)
+
+      Kreuzberg.clear_validators
+
+      expect do
+        Kreuzberg.extract_file_sync(test_pdf)
+      end.not_to raise_error
+    end
+  end
+
+  describe 'list_validators' do
+    it 'returns empty array when no validators registered' do
+      Kreuzberg.clear_validators
+      validators = Kreuzberg.list_validators
+      expect(validators).to be_an(Array)
+      expect(validators).to be_empty
+    end
+
+    it 'returns validator names after registration' do
+      Kreuzberg.clear_validators
+      validator = ->(result) {}
+      Kreuzberg.register_validator('test-validator', validator)
+      validators = Kreuzberg.list_validators
+      expect(validators).to include('test-validator')
+      Kreuzberg.clear_validators
+    end
+
+    it 'returns all registered validator names' do
+      Kreuzberg.clear_validators
+      validator1 = ->(result) {}
+      validator2 = ->(result) {}
+      validator3 = ->(result) {}
+
+      Kreuzberg.register_validator('validator-one', validator1)
+      Kreuzberg.register_validator('validator-two', validator2)
+      Kreuzberg.register_validator('validator-three', validator3)
+
+      validators = Kreuzberg.list_validators
+      expect(validators).to contain_exactly('validator-one', 'validator-two', 'validator-three')
+      Kreuzberg.clear_validators
+    end
+
+    it 'reflects changes after unregistration' do
+      Kreuzberg.clear_validators
+      validator = ->(result) {}
+      Kreuzberg.register_validator('temp-validator', validator)
+
+      validators_before = Kreuzberg.list_validators
+      expect(validators_before).to include('temp-validator')
+
+      Kreuzberg.unregister_validator('temp-validator')
+
+      validators_after = Kreuzberg.list_validators
+      expect(validators_after).not_to include('temp-validator')
+      Kreuzberg.clear_validators
+    end
+  end
+end

data/spec/fixtures/config.toml

@@ -0,0 +1,39 @@
+# Test configuration file for Kreuzberg Ruby bindings
+
+use_cache = false
+enable_quality_processing = true
+force_ocr = true
+
+[ocr]
+backend = "tesseract"
+language = "deu"
+
+[chunking]
+max_chars = 500
+max_overlap = 100
+preset = "fast"
+
+[language_detection]
+enabled = true
+min_confidence = 0.9
+
+[pdf_options]
+extract_images = true
+passwords = ["secret", "backup"]
+extract_metadata = true
+
+[images]
+extract_images = true
+target_dpi = 600
+max_image_dimension = 2000
+auto_adjust_dpi = false
+min_dpi = 150
+max_dpi = 600
+
+[postprocessor]
+enabled = true
+enabled_processors = ["quality", "formatting"]
+
+[token_reduction]
+mode = "moderate"
+preserve_important_words = true

data/spec/fixtures/config.yaml

@@ -0,0 +1,41 @@
+
+use_cache: false
+enable_quality_processing: true
+force_ocr: true
+
+ocr:
+  backend: tesseract
+  language: fra
+
+chunking:
+  max_chars: 750
+  max_overlap: 150
+  preset: balanced
+
+language_detection:
+  enabled: true
+  min_confidence: 0.85
+
+pdf_options:
+  extract_images: false
+  passwords:
+    - password1
+    - password2
+  extract_metadata: true
+
+images:
+  extract_images: true
+  target_dpi: 300
+  max_image_dimension: 4096
+  auto_adjust_dpi: true
+  min_dpi: 72
+  max_dpi: 600
+
+postprocessor:
+  enabled: false
+  disabled_processors:
+    - token_reduction
+
+token_reduction:
+  mode: light
+  preserve_important_words: false

data/spec/fixtures/invalid_config.toml

@@ -0,0 +1,4 @@
+# Invalid TOML file for testing error handling
+use_cache = "not_a_boolean"
+[ocr
+backend = "tesseract"

data/spec/smoke/package_spec.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+# Basic smoke tests to verify package structure and imports work
+require 'stringio'
+
+RSpec.describe 'Kreuzberg package' do
+  describe 'import and structure' do
+    it 'can be required without errors' do
+      expect { require 'kreuzberg' }.not_to raise_error
+    end
+
+    it 'has a version constant' do
+      expect(Kreuzberg::VERSION).not_to be_nil
+      expect(Kreuzberg::VERSION).to be_a(String)
+      expect(Kreuzberg::VERSION).to match(/^\d+\.\d+\.\d+/)
+    end
+  end
+
+  describe 'public API exports' do
+    describe 'configuration classes' do
+      it 'exports Config::Extraction' do
+        expect(defined?(Kreuzberg::Config::Extraction)).to eq('constant')
+      end
+
+      it 'exports Config::OCR' do
+        expect(defined?(Kreuzberg::Config::OCR)).to eq('constant')
+      end
+
+      it 'exports Config::Chunking' do
+        expect(defined?(Kreuzberg::Config::Chunking)).to eq('constant')
+      end
+
+      it 'exports Config::LanguageDetection' do
+        expect(defined?(Kreuzberg::Config::LanguageDetection)).to eq('constant')
+      end
+
+      it 'exports Config::PDF' do
+        expect(defined?(Kreuzberg::Config::PDF)).to eq('constant')
+      end
+
+      it 'exports Config::HtmlOptions' do
+        expect(defined?(Kreuzberg::Config::HtmlOptions)).to eq('constant')
+      end
+
+      it 'exports Config::Keywords' do
+        expect(defined?(Kreuzberg::Config::Keywords)).to eq('constant')
+      end
+    end
+
+    describe 'result classes' do
+      it 'exports Result' do
+        expect(defined?(Kreuzberg::Result)).to eq('constant')
+      end
+
+      it 'exports Result::Table' do
+        expect(defined?(Kreuzberg::Result::Table)).to eq('constant')
+      end
+
+      it 'exports Result::Chunk' do
+        expect(defined?(Kreuzberg::Result::Chunk)).to eq('constant')
+      end
+
+      it 'exports Result::Image' do
+        expect(defined?(Kreuzberg::Result::Image)).to eq('constant')
+      end
+    end
+
+    describe 'exception classes' do
+      it 'exports Errors::Error' do
+        expect(defined?(Kreuzberg::Errors::Error)).to eq('constant')
+      end
+
+      it 'exports Errors::ValidationError' do
+        expect(defined?(Kreuzberg::Errors::ValidationError)).to eq('constant')
+      end
+
+      it 'exports Errors::ParsingError' do
+        expect(defined?(Kreuzberg::Errors::ParsingError)).to eq('constant')
+      end
+
+      it 'exports Errors::OCRError' do
+        expect(defined?(Kreuzberg::Errors::OCRError)).to eq('constant')
+      end
+
+      it 'exports Errors::MissingDependencyError' do
+        expect(defined?(Kreuzberg::Errors::MissingDependencyError)).to eq('constant')
+      end
+
+      it 'exports Errors::IOError' do
+        expect(defined?(Kreuzberg::Errors::IOError)).to eq('constant')
+      end
+
+      it 'exports Errors::PluginError' do
+        expect(defined?(Kreuzberg::Errors::PluginError)).to eq('constant')
+      end
+    end
+
+    describe 'extraction functions (sync)' do
+      it 'exports extract_file_sync' do
+        expect(Kreuzberg).to respond_to(:extract_file_sync)
+      end
+
+      it 'exports extract_bytes_sync' do
+        expect(Kreuzberg).to respond_to(:extract_bytes_sync)
+      end
+
+      it 'exports batch_extract_files_sync' do
+        expect(Kreuzberg).to respond_to(:batch_extract_files_sync)
+      end
+    end
+
+    describe 'extraction functions (async)' do
+      it 'exports extract_file' do
+        expect(Kreuzberg).to respond_to(:extract_file)
+      end
+
+      it 'exports extract_bytes' do
+        expect(Kreuzberg).to respond_to(:extract_bytes)
+      end
+
+      it 'exports batch_extract_files' do
+        expect(Kreuzberg).to respond_to(:batch_extract_files)
+      end
+    end
+
+    describe 'utility modules' do
+      it 'exports CLI' do
+        expect(defined?(Kreuzberg::CLI)).to eq('constant')
+      end
+
+      it 'exports CLIProxy' do
+        expect(defined?(Kreuzberg::CLIProxy)).to eq('constant')
+      end
+
+      it 'exports APIProxy' do
+        expect(defined?(Kreuzberg::APIProxy)).to eq('constant')
+      end
+
+      it 'exports MCPProxy' do
+        expect(defined?(Kreuzberg::MCPProxy)).to eq('constant')
+      end
+    end
+  end
+
+  describe 'module structure' do
+    it 'defines Kreuzberg as a module' do
+      expect(Kreuzberg).to be_a(Module)
+    end
+
+    it 'defines Kreuzberg::Config as a module' do
+      expect(Kreuzberg::Config).to be_a(Module)
+    end
+
+    it 'defines Kreuzberg::Errors as a module' do
+      expect(Kreuzberg::Errors).to be_a(Module)
+    end
+  end
+
+  describe 'basic extraction smoke tests' do
+    it 'extracts inline text via bytes API' do
+      bytes = StringIO.new('Hello from Kreuzberg')
+      result = Kreuzberg.extract_bytes_sync(bytes.string, 'text/plain')
+
+      expect(result.content).to include('Hello')
+      expect(result.mime_type).to eq('text/plain')
+    end
+
+    it 'extracts from small temp file via sync API' do
+      file = create_test_file('Simple document for smoke testing')
+      result = Kreuzberg.extract_file_sync(file)
+
+      expect(result.content).to include('Simple document')
+      expect(result.mime_type).to eq('text/plain')
+    ensure
+      File.delete(file) if file && File.exist?(file)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'kreuzberg'
+require 'tmpdir'
+require 'fileutils'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |expectations|
+    expectations.include_chain_clauses_in_custom_matcher_descriptions = true
+  end
+
+  config.mock_with :rspec do |mocks|
+    mocks.verify_partial_doubles = true
+  end
+
+  config.shared_context_metadata_behavior = :apply_to_host_groups
+  config.filter_run_when_matching :focus
+  config.example_status_persistence_file_path = 'spec/examples.txt'
+  config.disable_monkey_patching!
+  config.warnings = true
+  config.default_formatter = 'doc' if config.files_to_run.one?
+  config.order = :random
+  Kernel.srand config.seed
+
+  # Helpers
+  config.include(Module.new do
+    def fixture_path(filename)
+      File.join(__dir__, 'fixtures', filename)
+    end
+
+    def test_document_path(relative_path)
+      # Go up from packages/ruby/spec to project root, then into test_documents
+      File.expand_path(File.join(__dir__, '..', '..', '..', 'test_documents', relative_path))
+    end
+
+    def create_test_file(content, filename: 'test.txt')
+      path = File.join(Dir.tmpdir, filename)
+      File.write(path, content)
+      path
+    end
+  end)
+end