kreuzberg 4.0.0.pre.rc.6

data/spec/binding/cache_spec.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Cache Management' do
+  let(:test_pdf) do
+    test_document_path('pdfs/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf')
+  end
+  let(:test_text) { test_document_path('text/contract_test.txt') }
+  let(:test_docx) { test_document_path('documents/contract.docx') }
+
+  before do
+    Kreuzberg.clear_cache
+  end
+
+  after do
+    Kreuzberg.clear_cache
+  end
+
+  describe 'clear_cache' do
+    it 'removes all cached results' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.extract_file_sync(test_text)
+
+      stats_before = Kreuzberg.cache_stats
+      expect(stats_before['total_entries']).to be_positive
+
+      Kreuzberg.clear_cache
+
+      stats_after = Kreuzberg.cache_stats
+      expect(stats_after['total_entries']).to eq(0)
+      expect(stats_after['total_size_bytes']).to eq(0)
+    end
+
+    it 'returns nil (void return)' do
+      result = Kreuzberg.clear_cache
+      expect(result).to be_nil
+    end
+
+    it 'can be called multiple times safely' do
+      Kreuzberg.clear_cache
+      Kreuzberg.clear_cache
+      Kreuzberg.clear_cache
+
+      stats = Kreuzberg.cache_stats
+      expect(stats['total_entries']).to eq(0)
+    end
+
+    it 'does not affect future extractions' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.clear_cache
+
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result).to be_a(Kreuzberg::Result)
+      expect(result.content).not_to be_empty
+    end
+  end
+
+  describe 'cache_stats' do
+    it 'returns hash with correct structure' do
+      stats = Kreuzberg.cache_stats
+
+      expect(stats).to be_a(Hash)
+      expect(stats).to have_key('total_entries')
+      expect(stats).to have_key('total_size_bytes')
+    end
+
+    it 'returns zero stats when cache is empty' do
+      Kreuzberg.clear_cache
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to eq(0)
+      expect(stats['total_size_bytes']).to eq(0)
+    end
+
+    it 'shows entries after extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to be_positive
+    end
+
+    it 'shows total size in bytes' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_size_bytes']).to be_positive
+    end
+
+    it 'increases stats with multiple extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_one = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats_after_two = Kreuzberg.cache_stats
+
+      expect(stats_after_two['total_entries']).to be >= stats_after_one['total_entries']
+    end
+  end
+
+  describe 'cache behavior across extractions' do
+    it 'caches extraction results' do
+      Kreuzberg.clear_cache
+      stats_initial = Kreuzberg.cache_stats
+      expect(stats_initial['total_entries']).to eq(0)
+
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_first = Kreuzberg.cache_stats
+      expect(stats_after_first['total_entries']).to be_positive
+
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_second = Kreuzberg.cache_stats
+
+      expect(result1.content).to eq(result2.content)
+      expect(stats_after_second['total_entries']).to eq(stats_after_first['total_entries'] + 1)
+    end
+
+    it 'tracks different files separately' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_pdf = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats_after_text = Kreuzberg.cache_stats
+
+      expect(stats_after_text['total_entries']).to be >= stats_after_pdf['total_entries']
+    end
+
+    it 'second extraction of same file may use cache' do
+      Kreuzberg.clear_cache
+
+      Time.now
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+      Time.now
+
+      Time.now
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+      Time.now
+
+      expect(result1.content).to eq(result2.content)
+      expect(result1.mime_type).to eq(result2.mime_type)
+    end
+
+    it 'clears cache between extractions when requested' do
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+
+      Kreuzberg.clear_cache
+
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result1.content).to eq(result2.content)
+    end
+  end
+
+  describe 'cache with different configurations' do
+    it 'respects use_cache flag in configs' do
+      Kreuzberg.clear_cache
+
+      config1 = Kreuzberg::Config::Extraction.new(use_cache: true)
+      config2 = Kreuzberg::Config::Extraction.new(use_cache: false)
+
+      Kreuzberg.extract_file_sync(test_pdf, config: config1)
+      stats_after_first = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_pdf, config: config2)
+      stats_after_second = Kreuzberg.cache_stats
+
+      expect(stats_after_second['total_entries']).to eq(stats_after_first['total_entries'])
+    end
+  end
+
+  describe 'cache stats consistency' do
+    it 'stats remain consistent after clear' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.extract_file_sync(test_text)
+
+      Kreuzberg.clear_cache
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to eq(0)
+      expect(stats['total_size_bytes']).to eq(0)
+    end
+
+    it 'stats update correctly after new extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.cache_stats
+
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats2 = Kreuzberg.cache_stats
+
+      expect(stats2['total_entries']).to be_positive
+    end
+  end
+
+  describe 'integration with batch operations' do
+    it 'caches batch extraction results' do
+      Kreuzberg.clear_cache
+
+      results = Kreuzberg.batch_extract_files_sync([test_pdf, test_text])
+      stats = Kreuzberg.cache_stats
+
+      expect(results.length).to eq(2)
+      expect(stats['total_entries']).to be_positive
+    end
+
+    it 'clear_cache affects batch extractions' do
+      Kreuzberg.batch_extract_files_sync([test_pdf, test_text])
+
+      Kreuzberg.clear_cache
+
+      stats = Kreuzberg.cache_stats
+      expect(stats['total_entries']).to eq(0)
+    end
+  end
+end

data/spec/binding/cli_proxy_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+RSpec.describe Kreuzberg::CLIProxy do
+  describe '.find_cli_binary' do
+    context 'when binary exists' do
+      it 'finds the binary in search paths' do
+        binary = described_class.find_cli_binary
+        expect(binary).to be_a(Pathname)
+        expect(binary.file?).to be true
+      end
+    end
+
+    context 'when binary does not exist' do
+      before do
+        allow(described_class).to receive(:search_paths).and_return([])
+      end
+
+      it 'raises MissingBinaryError' do
+        expect do
+          described_class.find_cli_binary
+        end.to raise_error(Kreuzberg::CLIProxy::MissingBinaryError, /not found/)
+      end
+    end
+  end
+
+  describe '.call' do
+    context 'when binary is available' do
+      it 'executes CLI command successfully' do
+        output = described_class.call(['--version'])
+        expect(output).to be_a(String)
+        expect(output).not_to be_empty
+      end
+
+      it 'raises CLIExecutionError on failure' do
+        expect do
+          described_class.call(['invalid-command'])
+        end.to raise_error(Kreuzberg::CLIProxy::CLIExecutionError)
+      end
+    end
+  end
+
+  describe '.search_paths' do
+    it 'returns an array of Pathname objects' do
+      paths = described_class.search_paths('kreuzberg')
+      expect(paths).to be_an(Array)
+      expect(paths).to all(be_a(Pathname))
+    end
+
+    it 'includes expected search locations' do
+      paths = described_class.search_paths('kreuzberg')
+      path_strings = paths.map(&:to_s)
+
+      expect(path_strings.any? { |p| p.include?('lib/bin') }).to be true
+      expect(path_strings.any? { |p| p.include?('target/release') }).to be true
+    end
+  end
+
+  describe '.root_path' do
+    it 'returns a Pathname' do
+      expect(described_class.root_path).to be_a(Pathname)
+    end
+
+    it 'points to an existing directory' do
+      expect(described_class.root_path.directory?).to be true
+    end
+  end
+
+  describe '.lib_path' do
+    it 'returns a Pathname' do
+      expect(described_class.lib_path).to be_a(Pathname)
+    end
+
+    it 'points to an existing directory' do
+      expect(described_class.lib_path.directory?).to be true
+    end
+  end
+
+  describe '.missing_binary_message' do
+    it 'returns helpful error message' do
+      message = described_class.missing_binary_message
+      expect(message).to include('cargo build')
+      expect(message).to include('kreuzberg-cli')
+    end
+  end
+end

data/spec/binding/cli_spec.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+RSpec.describe Kreuzberg::CLI do
+  describe '.extract' do
+    it 'extracts content from a file' do
+      path = test_document_path('documents/simple.odt')
+      output = described_class.extract(path)
+
+      expect(output).to be_a(String)
+      expect(output).not_to be_empty
+    end
+
+    it 'accepts output format option' do
+      path = test_document_path('documents/simple.odt')
+      output = described_class.extract(path, output: 'json')
+
+      expect(output).to be_a(String)
+      expect(output).not_to be_empty
+    end
+
+    it 'accepts OCR option' do
+      path = test_document_path('pdfs/100_g_networking_technology_overview_slides_toronto_august_2016.pdf')
+      output = described_class.extract(path, ocr: false)
+
+      expect(output).to be_a(String)
+      expect(output).not_to be_empty
+    end
+  end
+
+  describe '.detect' do
+    it 'detects MIME type' do
+      path = test_document_path('documents/simple.odt')
+      mime_type = described_class.detect(path)
+
+      expect(mime_type).to be_a(String)
+      expect(mime_type).not_to be_empty
+    end
+  end
+
+  describe '.version' do
+    it 'returns version string' do
+      version = described_class.version
+      expect(version).to be_a(String)
+      expect(version).to match(/\d+\.\d+/)
+    end
+  end
+
+  describe '.help' do
+    it 'returns help text' do
+      help_text = described_class.help
+      expect(help_text).to be_a(String)
+      expect(help_text).to include('kreuzberg')
+    end
+  end
+end

data/spec/binding/config_spec.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+RSpec.describe Kreuzberg::Config do
+  describe Kreuzberg::Config::OCR do
+    it 'creates with default values' do
+      ocr = described_class.new
+
+      expect(ocr.backend).to eq('tesseract')
+      expect(ocr.language).to eq('eng')
+      expect(ocr.tesseract_config).to be_nil
+    end
+
+    it 'creates with custom values' do
+      ocr = described_class.new(
+        backend: 'easyocr',
+        language: 'deu'
+      )
+
+      expect(ocr.backend).to eq('easyocr')
+      expect(ocr.language).to eq('deu')
+    end
+
+    it 'converts to hash' do
+      ocr = described_class.new(backend: 'tesseract', language: 'fra')
+      hash = ocr.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:backend]).to eq('tesseract')
+      expect(hash[:language]).to eq('fra')
+    end
+  end
+
+  describe Kreuzberg::Config::Chunking do
+    it 'creates with default values' do
+      chunking = described_class.new
+
+      expect(chunking.max_chars).to eq(1000)
+      expect(chunking.max_overlap).to eq(200)
+      expect(chunking.preset).to be_nil
+      expect(chunking.embedding).to be_nil
+    end
+
+    it 'creates with custom values' do
+      chunking = described_class.new(
+        max_chars: 500,
+        max_overlap: 100,
+        preset: 'fast'
+      )
+
+      expect(chunking.max_chars).to eq(500)
+      expect(chunking.max_overlap).to eq(100)
+      expect(chunking.preset).to eq('fast')
+    end
+
+    it 'converts to hash' do
+      chunking = described_class.new(max_chars: 750)
+      hash = chunking.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:max_chars]).to eq(750)
+    end
+  end
+
+  describe Kreuzberg::Config::LanguageDetection do
+    it 'creates with default values' do
+      lang = described_class.new
+
+      expect(lang.enabled).to be false
+      expect(lang.min_confidence).to eq(0.5)
+    end
+
+    it 'creates with custom values' do
+      lang = described_class.new(enabled: true, min_confidence: 0.9)
+
+      expect(lang.enabled).to be true
+      expect(lang.min_confidence).to eq(0.9)
+    end
+
+    it 'converts to hash' do
+      lang = described_class.new(enabled: true, min_confidence: 0.75)
+      hash = lang.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:enabled]).to be true
+      expect(hash[:min_confidence]).to eq(0.75)
+    end
+  end
+
+  describe Kreuzberg::Config::PDF do
+    it 'creates with default values' do
+      pdf = described_class.new
+
+      expect(pdf.extract_images).to be false
+      expect(pdf.passwords).to be_nil
+      expect(pdf.extract_metadata).to be true
+    end
+
+    it 'creates with custom values' do
+      pdf = described_class.new(
+        extract_images: true,
+        passwords: %w[secret backup]
+      )
+
+      expect(pdf.extract_images).to be true
+      expect(pdf.passwords).to eq(%w[secret backup])
+    end
+
+    it 'converts to hash' do
+      pdf = described_class.new(extract_images: true, passwords: ['test'])
+      hash = pdf.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:extract_images]).to be true
+      expect(hash[:passwords]).to eq(['test'])
+    end
+  end
+
+  describe Kreuzberg::Config::Extraction do
+    describe '.from_file' do
+      it 'loads configuration from TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        config = described_class.from_file(config_path)
+
+        expect(config.use_cache).to be false
+        expect(config.enable_quality_processing).to be true
+        expect(config.force_ocr).to be true
+      end
+
+      it 'loads OCR config from TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        config = described_class.from_file(config_path)
+
+        expect(config.ocr).to be_a(Kreuzberg::Config::OCR)
+        expect(config.ocr.backend).to eq('tesseract')
+        expect(config.ocr.language).to eq('deu')
+      end
+
+      it 'loads chunking config from TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        config = described_class.from_file(config_path)
+
+        expect(config.chunking).to be_a(Kreuzberg::Config::Chunking)
+        expect(config.chunking.max_chars).to eq(500)
+        expect(config.chunking.max_overlap).to eq(100)
+        expect(config.chunking.preset).to eq('fast')
+      end
+
+      it 'loads language detection config from TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        config = described_class.from_file(config_path)
+
+        expect(config.language_detection).to be_a(Kreuzberg::Config::LanguageDetection)
+        expect(config.language_detection.enabled).to be true
+        expect(config.language_detection.min_confidence).to eq(0.9)
+      end
+
+      it 'loads PDF options from TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        config = described_class.from_file(config_path)
+
+        expect(config.pdf_options).to be_a(Kreuzberg::Config::PDF)
+        expect(config.pdf_options.extract_images).to be true
+        expect(config.pdf_options.passwords).to eq(%w[secret backup])
+        expect(config.pdf_options.extract_metadata).to be true
+      end
+
+      it 'loads configuration from YAML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.yaml')
+        config = described_class.from_file(config_path)
+
+        expect(config.use_cache).to be false
+        expect(config.enable_quality_processing).to be true
+        expect(config.force_ocr).to be true
+      end
+
+      it 'loads OCR config from YAML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.yaml')
+        config = described_class.from_file(config_path)
+
+        expect(config.ocr).to be_a(Kreuzberg::Config::OCR)
+        expect(config.ocr.backend).to eq('tesseract')
+        expect(config.ocr.language).to eq('fra')
+      end
+
+      it 'loads chunking config from YAML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.yaml')
+        config = described_class.from_file(config_path)
+
+        expect(config.chunking).to be_a(Kreuzberg::Config::Chunking)
+        expect(config.chunking.max_chars).to eq(750)
+        expect(config.chunking.max_overlap).to eq(150)
+        expect(config.chunking.preset).to eq('balanced')
+      end
+
+      it 'works with absolute paths' do
+        config_path = File.expand_path('../fixtures/config.toml', __dir__)
+        config = described_class.from_file(config_path)
+
+        expect(config.use_cache).to be false
+      end
+
+      it 'works with relative paths' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'config.yaml')
+        config = described_class.from_file(config_path)
+
+        expect(config.use_cache).to be false
+      end
+
+      it 'raises error for non-existent file' do
+        expect do
+          described_class.from_file('/path/to/nonexistent/config.toml')
+        end.to raise_error(Kreuzberg::Errors::ValidationError, /Failed to read config file/)
+      end
+
+      it 'raises error for invalid TOML file' do
+        config_path = File.join(__dir__, '..', 'fixtures', 'invalid_config.toml')
+        expect do
+          described_class.from_file(config_path)
+        end.to raise_error(Kreuzberg::Errors::ValidationError, /Invalid TOML/)
+      end
+
+      it 'detects file format from extension' do
+        toml_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+        yaml_path = File.join(__dir__, '..', 'fixtures', 'config.yaml')
+
+        toml_config = described_class.from_file(toml_path)
+        yaml_config = described_class.from_file(yaml_path)
+
+        expect(toml_config.ocr.language).to eq('deu')
+        expect(yaml_config.ocr.language).to eq('fra')
+      end
+    end
+
+    it 'creates with default values' do
+      config = described_class.new
+
+      expect(config.use_cache).to be true
+      expect(config.enable_quality_processing).to be false
+      expect(config.force_ocr).to be false
+      expect(config.ocr).to be_nil
+      expect(config.chunking).to be_nil
+      expect(config.language_detection).to be_nil
+      expect(config.pdf_options).to be_nil
+    end
+
+    it 'creates with custom values' do
+      ocr = Kreuzberg::Config::OCR.new(backend: 'easyocr')
+      chunking = Kreuzberg::Config::Chunking.new(max_chars: 500)
+      lang = Kreuzberg::Config::LanguageDetection.new(enabled: true)
+      pdf = Kreuzberg::Config::PDF.new(extract_images: true)
+
+      config = described_class.new(
+        use_cache: false,
+        enable_quality_processing: true,
+        force_ocr: true,
+        ocr: ocr,
+        chunking: chunking,
+        language_detection: lang,
+        pdf_options: pdf
+      )
+
+      expect(config.use_cache).to be false
+      expect(config.enable_quality_processing).to be true
+      expect(config.force_ocr).to be true
+      expect(config.ocr).to eq(ocr)
+      expect(config.chunking).to eq(chunking)
+      expect(config.language_detection).to eq(lang)
+      expect(config.pdf_options).to eq(pdf)
+    end
+
+    it 'accepts hash for nested configs' do
+      config = described_class.new(
+        ocr: { backend: 'tesseract', language: 'eng' },
+        chunking: { max_chars: 500 }
+      )
+
+      expect(config.ocr).to be_a(Kreuzberg::Config::OCR)
+      expect(config.ocr.backend).to eq('tesseract')
+      expect(config.chunking).to be_a(Kreuzberg::Config::Chunking)
+      expect(config.chunking.max_chars).to eq(500)
+    end
+
+    it 'converts to hash' do
+      config = described_class.new(
+        use_cache: false,
+        ocr: { backend: 'tesseract' }
+      )
+      hash = config.to_h
+
+      expect(hash).to be_a(Hash)
+      expect(hash[:use_cache]).to be false
+      expect(hash[:ocr]).to be_a(Hash)
+      expect(hash[:ocr][:backend]).to eq('tesseract')
+    end
+
+    it 'raises error for invalid config type' do
+      expect do
+        described_class.new(ocr: 'invalid')
+      end.to raise_error(ArgumentError, /Expected.*OCR/)
+    end
+  end
+
+  describe 'ExtractionConfig alias' do
+    it 'exists at module level' do
+      expect(Kreuzberg.const_defined?(:ExtractionConfig)).to be true
+    end
+
+    it 'is the same class as Config::Extraction' do
+      expect(Kreuzberg::ExtractionConfig).to eq(Kreuzberg::Config::Extraction)
+    end
+
+    it 'can be instantiated using the alias' do
+      config = Kreuzberg::ExtractionConfig.new(use_cache: false)
+
+      expect(config).to be_a(Kreuzberg::Config::Extraction)
+      expect(config.use_cache).to be false
+    end
+
+    it 'supports all methods through the alias' do
+      config = Kreuzberg::ExtractionConfig.new(
+        use_cache: false,
+        force_ocr: true,
+        ocr: { backend: 'tesseract', language: 'eng' }
+      )
+
+      expect(config.use_cache).to be false
+      expect(config.force_ocr).to be true
+      expect(config.ocr).to be_a(Kreuzberg::Config::OCR)
+      expect(config.ocr.backend).to eq('tesseract')
+
+      hash = config.to_h
+      expect(hash[:use_cache]).to be false
+      expect(hash[:force_ocr]).to be true
+    end
+
+    it 'supports from_file through the alias' do
+      config_path = File.join(__dir__, '..', 'fixtures', 'config.toml')
+      config = Kreuzberg::ExtractionConfig.from_file(config_path)
+
+      expect(config).to be_a(Kreuzberg::Config::Extraction)
+      expect(config.use_cache).to be false
+      expect(config.enable_quality_processing).to be true
+    end
+  end
+end