kreuzberg 4.0.0.rc1

data/sig/kreuzberg.rbs

@@ -0,0 +1,468 @@
+# Type signatures for Kreuzberg document intelligence framework
+
+module Kreuzberg
+  VERSION: String
+
+  # Config namespace (defined in lib/kreuzberg/config.rb)
+  module Config
+    class OCR
+      attr_reader backend: String
+      attr_reader language: String
+      attr_reader tesseract_config: Tesseract?
+
+      def initialize: (?backend: String, ?language: String, ?tesseract_config: (Tesseract | Hash[Symbol, untyped])?) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class Tesseract
+      def initialize: (**untyped options) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class Chunking
+      attr_reader max_chars: Integer
+      attr_reader max_overlap: Integer
+      attr_reader preset: String?
+      attr_reader embedding: Embedding?
+      attr_reader enabled: bool?
+
+      def initialize: (
+        ?max_chars: Integer?,
+        ?max_overlap: Integer?,
+        ?preset: String?,
+        ?embedding: (Embedding | Hash[Symbol, untyped])?,
+        ?chunk_size: Integer?,
+        ?chunk_overlap: Integer?,
+        ?enabled: bool
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class Embedding
+      attr_reader model: Hash[Symbol, untyped]
+      attr_reader normalize: bool?
+      attr_reader batch_size: Integer?
+      attr_reader show_download_progress: bool?
+      attr_reader cache_dir: String?
+
+      def initialize: (
+        ?model: Hash[Symbol, untyped],
+        ?normalize: bool?,
+        ?batch_size: Integer?,
+        ?show_download_progress: bool?,
+        ?cache_dir: String?
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class LanguageDetection
+      attr_reader enabled: bool
+      attr_reader min_confidence: Float
+      attr_reader detect_multiple: bool
+
+      def initialize: (?enabled: bool, ?min_confidence: Float, ?detect_multiple: bool) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class PDF
+      attr_reader extract_images: bool
+      attr_reader passwords: Array[String]?
+      attr_reader extract_metadata: bool
+
+      def initialize: (?extract_images: bool, ?passwords: (Array[String] | String)?, ?extract_metadata: bool) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class ImageExtraction
+      attr_reader extract_images: bool
+      attr_reader target_dpi: Integer
+      attr_reader max_image_dimension: Integer
+      attr_reader auto_adjust_dpi: bool
+      attr_reader min_dpi: Integer
+      attr_reader max_dpi: Integer
+
+      def initialize: (
+        ?extract_images: bool,
+        ?target_dpi: Integer,
+        ?max_image_dimension: Integer,
+        ?auto_adjust_dpi: bool,
+        ?min_dpi: Integer,
+        ?max_dpi: Integer
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class ImagePreprocessing
+      attr_reader target_dpi: Integer
+      attr_reader auto_rotate: bool
+      attr_reader deskew: bool
+      attr_reader denoise: bool
+      attr_reader contrast_enhance: bool
+      attr_reader binarization_method: String
+      attr_reader invert_colors: bool
+
+      def initialize: (
+        ?target_dpi: Integer,
+        ?auto_rotate: bool,
+        ?deskew: bool,
+        ?denoise: bool,
+        ?contrast_enhance: bool,
+        ?binarization_method: String,
+        ?invert_colors: bool
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class TokenReduction
+      attr_reader mode: String
+      attr_reader preserve_important_words: bool
+
+      def initialize: (?mode: String, ?preserve_important_words: bool) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class PostProcessor
+      attr_reader enabled: bool
+      attr_reader enabled_processors: Array[String]?
+      attr_reader disabled_processors: Array[String]?
+
+      def initialize: (?enabled: bool, ?enabled_processors: Array[String]?, ?disabled_processors: Array[String]?) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class HtmlPreprocessing
+      attr_reader enabled: bool?
+      attr_reader preset: Symbol?
+      attr_reader remove_navigation: bool?
+      attr_reader remove_forms: bool?
+
+      def initialize: (?enabled: bool?, ?preset: Symbol?, ?remove_navigation: bool?, ?remove_forms: bool?) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class HtmlOptions
+      def initialize: (**untyped options) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class Keywords
+      def initialize: (
+        ?algorithm: Symbol?,
+        ?max_keywords: Integer?,
+        ?min_score: Float?,
+        ?ngram_range: Array[Integer]?,
+        ?language: Symbol?,
+        ?yake_params: Hash[Symbol, untyped]?,
+        ?rake_params: Hash[Symbol, untyped]?
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+    end
+
+    class Extraction
+      attr_reader use_cache: bool
+      attr_reader enable_quality_processing: bool
+      attr_reader force_ocr: bool
+      attr_reader ocr: OCR?
+      attr_reader chunking: Chunking?
+      attr_reader language_detection: LanguageDetection?
+      attr_reader pdf_options: PDF?
+      attr_reader image_extraction: ImageExtraction?
+      attr_reader image_preprocessing: ImagePreprocessing?
+      attr_reader postprocessor: PostProcessor?
+      attr_reader token_reduction: TokenReduction?
+      attr_reader keywords: Keywords?
+      attr_reader html_options: HtmlOptions?
+      attr_reader max_concurrent_extractions: Integer?
+
+      def self.from_file: (String path) -> Extraction
+      def initialize: (
+        ?use_cache: bool,
+        ?enable_quality_processing: bool,
+        ?force_ocr: bool,
+        ?ocr: (OCR | Hash[Symbol, untyped])?,
+        ?chunking: (Chunking | Hash[Symbol, untyped])?,
+        ?language_detection: (LanguageDetection | Hash[Symbol, untyped])?,
+        ?pdf_options: (PDF | Hash[Symbol, untyped])?,
+        ?image_extraction: (ImageExtraction | Hash[Symbol, untyped])?,
+        ?image_preprocessing: (ImagePreprocessing | Hash[Symbol, untyped])?,
+        ?postprocessor: (PostProcessor | Hash[Symbol, untyped])?,
+        ?token_reduction: (TokenReduction | Hash[Symbol, untyped])?,
+        ?keywords: (Keywords | Hash[Symbol, untyped])?,
+        ?html_options: (HtmlOptions | Hash[Symbol, untyped])?,
+        ?max_concurrent_extractions: Integer?
+      ) -> void
+      def to_h: () -> Hash[Symbol, untyped]
+
+      private
+
+      def normalize_config: [T] (T | Hash[Symbol, untyped] | nil value, Class klass) -> T?
+    end
+
+    # Backwards compatibility alias
+    Ocr: singleton(OCR)
+  end
+
+  # Alias for Config::Extraction (for API consistency with other language bindings)
+  ExtractionConfig: singleton(Config::Extraction)
+
+  # Extraction result type
+  type extraction_result_hash = {
+    content: String,
+    mime_type: String,
+    metadata_json: String,
+    tables: Array[table_hash]?,
+    detected_languages: Array[String]?,
+    chunks: Array[chunk_hash]?,
+    images: Array[image_hash]?
+  }
+
+  type table_hash = {
+    cells: Array[Array[String]],
+    markdown: String,
+    page_number: Integer
+  }
+
+  type chunk_hash = {
+    content: String,
+    char_start: Integer,
+    char_end: Integer,
+    token_count: Integer?,
+    chunk_index: Integer?,
+    total_chunks: Integer?,
+    embedding: Array[Float]?
+  }
+
+  type image_hash = {
+    data: String,
+    format: String,
+    image_index: Integer,
+    page_number: Integer?,
+    width: Integer?,
+    height: Integer?,
+    colorspace: String?,
+    bits_per_component: Integer?,
+    is_mask: bool?,
+    description: String?,
+    ocr_result: extraction_result_hash?
+  }
+
+  type config_hash = Hash[Symbol, untyped]
+  type config_input = config_hash | _ToH
+
+  interface _ToH
+    def to_h: () -> config_hash
+  end
+
+  # Extraction result wrapper
+  class Result
+    # Table structure
+    class Table
+      attr_reader cells: Array[Array[String]]
+      attr_reader markdown: String
+      attr_reader page_number: Integer
+
+      def initialize: (cells: Array[Array[String]], markdown: String, page_number: Integer) -> void
+      def to_h: () -> table_hash
+    end
+
+    # Text chunk
+    class Chunk
+      attr_reader content: String
+      attr_reader char_start: Integer
+      attr_reader char_end: Integer
+      attr_reader token_count: Integer?
+      attr_reader chunk_index: Integer?
+      attr_reader total_chunks: Integer?
+      attr_reader embedding: Array[Float]?
+
+      def initialize: (
+        content: String,
+        char_start: Integer,
+        char_end: Integer,
+        token_count: Integer?,
+        chunk_index: Integer?,
+        total_chunks: Integer?,
+        embedding: Array[Float]?
+      ) -> void
+      def to_h: () -> chunk_hash
+    end
+
+    class Image
+      attr_reader data: String
+      attr_reader format: String
+      attr_reader image_index: Integer
+      attr_reader page_number: Integer?
+      attr_reader width: Integer?
+      attr_reader height: Integer?
+      attr_reader colorspace: String?
+      attr_reader bits_per_component: Integer?
+      attr_reader is_mask: bool?
+      attr_reader description: String?
+      attr_reader ocr_result: Result?
+
+      def initialize: (
+        data: String,
+        format: String,
+        image_index: Integer,
+        page_number: Integer?,
+        width: Integer?,
+        height: Integer?,
+        colorspace: String?,
+        bits_per_component: Integer?,
+        is_mask: bool?,
+        description: String?,
+        ocr_result: Result?
+      ) -> void
+      def to_h: () -> image_hash
+    end
+
+    attr_reader content: String
+    attr_reader mime_type: String
+    attr_reader metadata: Hash[untyped, untyped]
+    attr_reader metadata_json: String
+    attr_reader tables: Array[Table]
+    attr_reader detected_languages: Array[String]?
+    attr_reader chunks: Array[Chunk]?
+    attr_reader images: Array[Image]?
+
+    def initialize: (extraction_result_hash hash) -> void
+    def to_h: () -> Hash[Symbol, untyped]
+    def to_json: (*untyped) -> String
+
+    private
+
+    def parse_metadata: (String metadata_json) -> Hash[untyped, untyped]
+    def parse_tables: (Array[table_hash]? tables_data) -> Array[Table]
+    def parse_detected_languages: (Array[String]? langs_data) -> Array[String]?
+    def parse_chunks: (Array[chunk_hash]? chunks_data) -> Array[Chunk]?
+  end
+
+  # Module methods (extraction API)
+  def self.extract_file_sync: (
+    String | Pathname path,
+    ?mime_type: String?,
+    ?config: config_input?
+  ) -> Result
+
+  def self.extract_bytes_sync: (
+    String data,
+    String mime_type,
+    ?config: config_input?
+  ) -> Result
+
+  def self.batch_extract_files_sync: (
+    Array[String | Pathname] paths,
+    ?config: config_input?
+  ) -> Array[Result]
+
+  def self.batch_extract_bytes_sync: (
+    Array[String] data_array,
+    Array[String] mime_types,
+    ?config: config_input?
+  ) -> Array[Result]
+
+  def self.extract_file: (
+    String | Pathname path,
+    ?mime_type: String?,
+    ?config: config_input?
+  ) -> Result
+
+  def self.extract_bytes: (
+    String data,
+    String mime_type,
+    ?config: config_input?
+  ) -> Result
+
+  def self.batch_extract_files: (
+    Array[String | Pathname] paths,
+    ?config: config_input?
+  ) -> Array[Result]
+
+  def self.batch_extract_bytes: (
+    Array[String] data_array,
+    Array[String] mime_types,
+    ?config: config_input?
+  ) -> Array[Result]
+
+  # Cache API
+  def self.clear_cache: () -> void
+  def self.cache_stats: () -> Hash[Symbol | String, Integer]
+
+  # Config loading (native method)
+  def self._config_from_file_native: (String path) -> Hash[Symbol, untyped]
+
+  # Plugin registration
+  def self.register_post_processor: (String name, _PostProcessor processor, ?stage: Symbol?) -> void
+  def self.unregister_post_processor: (String name) -> void
+  def self.clear_post_processors: () -> void
+  def self.register_validator: (String name, _Validator validator, ?priority: Integer?) -> void
+  def self.unregister_validator: (String name) -> void
+  def self.clear_validators: () -> void
+  def self.register_ocr_backend: (_OcrBackend backend) -> void
+
+  interface _PostProcessor
+    def call: (extraction_result_hash result) -> extraction_result_hash
+  end
+
+  interface _Validator
+    def call: (extraction_result_hash result) -> void
+  end
+
+  interface _OcrBackend
+    def name: () -> String
+    def extract_text: (String file_path_or_bytes, Hash[Symbol, untyped] config) -> String
+  end
+
+  module Errors
+    class Error < StandardError
+    end
+
+    class ValidationError < Error
+    end
+
+    class ParsingError < Error
+      def initialize: (String message, ?context: Hash[untyped, untyped]?) -> void
+    end
+
+    class OCRError < Error
+      def initialize: (String message, ?context: Hash[untyped, untyped]?) -> void
+    end
+
+    class MissingDependencyError < Error
+      attr_reader dependency: String?
+
+      def initialize: (String message, ?dependency: String?) -> void
+    end
+
+    class IOError < Error
+    end
+
+    class PluginError < Error
+    end
+
+    class UnsupportedFormatError < Error
+    end
+  end
+
+  # Internal modules (prepended to Kreuzberg singleton)
+  # These are not checked by steep - see Steepfile
+  module CacheAPI : Object
+  end
+
+  module ExtractionAPI : Object
+  end
+
+  module PostProcessorProtocol
+    def call: (extraction_result_hash result) -> extraction_result_hash
+  end
+
+  module ValidatorProtocol
+    def call: (extraction_result_hash result) -> void
+  end
+
+  module OcrBackendProtocol
+    def name: () -> String
+    def extract_text: (String file_path_or_bytes, Hash[Symbol, untyped] config) -> String
+    def process_image: (String file_path_or_bytes, Hash[Symbol, untyped] config) -> String
+  end
+end

data/spec/binding/cache_spec.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'Cache Management' do
+  let(:test_pdf) do
+    test_document_path('pdfs/5_level_paging_and_5_level_ept_intel_revision_1_1_may_2017.pdf')
+  end
+  let(:test_text) { test_document_path('text/contract_test.txt') }
+  let(:test_docx) { test_document_path('documents/contract.docx') }
+
+  before do
+    Kreuzberg.clear_cache
+  end
+
+  after do
+    Kreuzberg.clear_cache
+  end
+
+  describe 'clear_cache' do
+    it 'removes all cached results' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.extract_file_sync(test_text)
+
+      stats_before = Kreuzberg.cache_stats
+      expect(stats_before['total_entries']).to be_positive
+
+      Kreuzberg.clear_cache
+
+      stats_after = Kreuzberg.cache_stats
+      expect(stats_after['total_entries']).to eq(0)
+      expect(stats_after['total_size_bytes']).to eq(0)
+    end
+
+    it 'returns nil (void return)' do
+      result = Kreuzberg.clear_cache
+      expect(result).to be_nil
+    end
+
+    it 'can be called multiple times safely' do
+      Kreuzberg.clear_cache
+      Kreuzberg.clear_cache
+      Kreuzberg.clear_cache
+
+      stats = Kreuzberg.cache_stats
+      expect(stats['total_entries']).to eq(0)
+    end
+
+    it 'does not affect future extractions' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.clear_cache
+
+      result = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result).to be_a(Kreuzberg::Result)
+      expect(result.content).not_to be_empty
+    end
+  end
+
+  describe 'cache_stats' do
+    it 'returns hash with correct structure' do
+      stats = Kreuzberg.cache_stats
+
+      expect(stats).to be_a(Hash)
+      expect(stats).to have_key('total_entries')
+      expect(stats).to have_key('total_size_bytes')
+    end
+
+    it 'returns zero stats when cache is empty' do
+      Kreuzberg.clear_cache
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to eq(0)
+      expect(stats['total_size_bytes']).to eq(0)
+    end
+
+    it 'shows entries after extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to be_positive
+    end
+
+    it 'shows total size in bytes' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_size_bytes']).to be_positive
+    end
+
+    it 'increases stats with multiple extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_one = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats_after_two = Kreuzberg.cache_stats
+
+      expect(stats_after_two['total_entries']).to be >= stats_after_one['total_entries']
+    end
+  end
+
+  describe 'cache behavior across extractions' do
+    it 'caches extraction results' do
+      Kreuzberg.clear_cache
+      stats_initial = Kreuzberg.cache_stats
+      expect(stats_initial['total_entries']).to eq(0)
+
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_first = Kreuzberg.cache_stats
+      expect(stats_after_first['total_entries']).to be_positive
+
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_second = Kreuzberg.cache_stats
+
+      expect(result1.content).to eq(result2.content)
+      expect(stats_after_second['total_entries']).to eq(stats_after_first['total_entries'] + 1)
+    end
+
+    it 'tracks different files separately' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      stats_after_pdf = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats_after_text = Kreuzberg.cache_stats
+
+      expect(stats_after_text['total_entries']).to be >= stats_after_pdf['total_entries']
+    end
+
+    it 'second extraction of same file may use cache' do
+      Kreuzberg.clear_cache
+
+      Time.now
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+      Time.now
+
+      Time.now
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+      Time.now
+
+      expect(result1.content).to eq(result2.content)
+      expect(result1.mime_type).to eq(result2.mime_type)
+    end
+
+    it 'clears cache between extractions when requested' do
+      result1 = Kreuzberg.extract_file_sync(test_pdf)
+
+      Kreuzberg.clear_cache
+
+      result2 = Kreuzberg.extract_file_sync(test_pdf)
+
+      expect(result1.content).to eq(result2.content)
+    end
+  end
+
+  describe 'cache with different configurations' do
+    it 'respects use_cache flag in configs' do
+      Kreuzberg.clear_cache
+
+      config1 = Kreuzberg::Config::Extraction.new(use_cache: true)
+      config2 = Kreuzberg::Config::Extraction.new(use_cache: false)
+
+      Kreuzberg.extract_file_sync(test_pdf, config: config1)
+      stats_after_first = Kreuzberg.cache_stats
+
+      Kreuzberg.extract_file_sync(test_pdf, config: config2)
+      stats_after_second = Kreuzberg.cache_stats
+
+      expect(stats_after_second['total_entries']).to eq(stats_after_first['total_entries'])
+    end
+  end
+
+  describe 'cache stats consistency' do
+    it 'stats remain consistent after clear' do
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.extract_file_sync(test_text)
+
+      Kreuzberg.clear_cache
+      stats = Kreuzberg.cache_stats
+
+      expect(stats['total_entries']).to eq(0)
+      expect(stats['total_size_bytes']).to eq(0)
+    end
+
+    it 'stats update correctly after new extractions' do
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_pdf)
+      Kreuzberg.cache_stats
+
+      Kreuzberg.clear_cache
+
+      Kreuzberg.extract_file_sync(test_text)
+      stats2 = Kreuzberg.cache_stats
+
+      expect(stats2['total_entries']).to be_positive
+    end
+  end
+
+  describe 'integration with batch operations' do
+    it 'caches batch extraction results' do
+      Kreuzberg.clear_cache
+
+      results = Kreuzberg.batch_extract_files_sync([test_pdf, test_text])
+      stats = Kreuzberg.cache_stats
+
+      expect(results.length).to eq(2)
+      expect(stats['total_entries']).to be_positive
+    end
+
+    it 'clear_cache affects batch extractions' do
+      Kreuzberg.batch_extract_files_sync([test_pdf, test_text])
+
+      Kreuzberg.clear_cache
+
+      stats = Kreuzberg.cache_stats
+      expect(stats['total_entries']).to eq(0)
+    end
+  end
+end