kreuzberg 4.3.5-aarch64-linux

data/lib/kreuzberg/setup_lib_path.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'rbconfig'
+require 'open3'
+
+module Kreuzberg
+  module SetupLibPath
+    module_function
+
+    def configure
+      lib_dir = File.expand_path('..', __dir__ || '.')
+      host_os = RbConfig::CONFIG['host_os']
+
+      case host_os
+      when /darwin/
+        prepend_env('DYLD_LIBRARY_PATH', lib_dir)
+        prepend_env('DYLD_FALLBACK_LIBRARY_PATH', "#{lib_dir}:/usr/local/lib:/usr/lib")
+        fix_macos_install_name(lib_dir)
+      when /linux/
+        prepend_env('LD_LIBRARY_PATH', lib_dir)
+      when /mswin|mingw|cygwin/
+        # Windows uses PATH to locate DLLs
+        prepend_env('PATH', lib_dir, separator: ';')
+        # Also check common locations for PDFium on Windows
+        setup_windows_library_paths(lib_dir)
+      end
+    end
+
+    def setup_windows_library_paths(lib_dir)
+      # Add target/release to PATH for DLL lookup during development
+      target_release = File.expand_path('../../target/release', lib_dir)
+      prepend_env('PATH', target_release, separator: ';') if Dir.exist?(target_release)
+
+      # Check for short path CARGO_TARGET_DIR (CI uses C:\t)
+      cargo_target_dir = ENV.fetch('CARGO_TARGET_DIR', nil)
+      return unless cargo_target_dir
+
+      target_release_alt = File.join(cargo_target_dir, 'release')
+      prepend_env('PATH', target_release_alt, separator: ';') if Dir.exist?(target_release_alt)
+
+      # Also check for target-specific subdirectory (Windows GNU builds)
+      gnu_release = File.join(cargo_target_dir, 'x86_64-pc-windows-gnu', 'release')
+      prepend_env('PATH', gnu_release, separator: ';') if Dir.exist?(gnu_release)
+    end
+    private_class_method :setup_windows_library_paths
+
+    def prepend_env(key, value, separator: ':')
+      current = ENV.fetch(key, nil)
+      return if current&.split(separator)&.include?(value)
+
+      ENV[key] = current.nil? || current.empty? ? value : "#{value}#{separator}#{current}"
+    end
+    private_class_method :prepend_env
+
+    def fix_macos_install_name(lib_dir)
+      bundle = macos_bundle(lib_dir)
+      return unless bundle
+
+      ensure_install_name(bundle)
+      ensure_loader_rpath(bundle)
+    rescue Errno::ENOENT, IOError # rubocop:disable Lint/SuppressedException
+    end
+    private_class_method :fix_macos_install_name
+
+    def macos_bundle(lib_dir)
+      bundle = File.join(lib_dir, 'kreuzberg_rb.bundle')
+      pdfium = File.join(lib_dir, 'libpdfium.dylib')
+      return unless File.exist?(bundle) && File.exist?(pdfium)
+
+      bundle
+    end
+    private_class_method :macos_bundle
+
+    def ensure_install_name(bundle)
+      output, status = Open3.capture2('otool', '-L', bundle)
+      return unless status.success?
+
+      replacements = {
+        './libpdfium.dylib' => '@loader_path/libpdfium.dylib',
+        '@rpath/libpdfium.dylib' => '@loader_path/libpdfium.dylib'
+      }
+
+      replacements.each do |current, desired|
+        next unless output.include?(current)
+
+        Open3.capture2('install_name_tool', '-change', current, desired, bundle)
+      end
+    end
+    private_class_method :ensure_install_name
+
+    def ensure_loader_rpath(bundle)
+      rpath_output, rpath_status = Open3.capture2('otool', '-l', bundle)
+      return unless rpath_status.success? && !rpath_output.include?('@loader_path')
+
+      Open3.capture2('install_name_tool', '-add_rpath', '@loader_path', bundle)
+    end
+    private_class_method :ensure_loader_rpath
+  end
+end

data/lib/kreuzberg/types.rb

@@ -0,0 +1,414 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module Kreuzberg
+  # Semantic element type classification.
+  #
+  # Categorizes text content into semantic units for downstream processing.
+  # Supports the element types commonly found in Unstructured documents.
+  #
+  # @example
+  #   type = Kreuzberg::ElementType::TITLE
+  #
+  ElementType = T.type_alias do
+    T.any(
+      'title',
+      'narrative_text',
+      'heading',
+      'list_item',
+      'table',
+      'image',
+      'page_break',
+      'code_block',
+      'block_quote',
+      'footer',
+      'header'
+    )
+  end
+
+  # Bounding box coordinates for element positioning.
+  #
+  # Represents rectangular coordinates for an element within a page.
+  #
+  # @example
+  #   bbox = Kreuzberg::BoundingBox.new(
+  #     x0: 10.0,
+  #     y0: 20.0,
+  #     x1: 100.0,
+  #     y1: 50.0
+  #   )
+  #   puts "Width: #{bbox.x1 - bbox.x0}"
+  #
+  class BoundingBox < T::Struct
+    extend T::Sig
+
+    const :x0, Float
+
+    const :y0, Float
+
+    const :x1, Float
+
+    const :y1, Float
+  end
+
+  # Metadata for a semantic element.
+  #
+  # Provides contextual information about an extracted element including
+  # its position within the document and custom metadata fields.
+  #
+  # @example
+  #   metadata = Kreuzberg::ElementMetadata.new(
+  #     page_number: 1,
+  #     filename: "document.pdf",
+  #     coordinates: bbox,
+  #     element_index: 5,
+  #     additional: { "style" => "bold" }
+  #   )
+  #
+  class ElementMetadata < T::Struct
+    extend T::Sig
+
+    const :page_number, T.nilable(Integer)
+
+    const :filename, T.nilable(String)
+
+    const :coordinates, T.nilable(BoundingBox)
+
+    const :element_index, T.nilable(Integer)
+
+    const :additional, T::Hash[String, String]
+  end
+
+  # Semantic element extracted from document.
+  #
+  # Represents a logical unit of content with semantic classification,
+  # unique identifier, and metadata for tracking origin and position.
+  # Compatible with Unstructured.io element format when output_format='element_based'.
+  #
+  # @example
+  #   element = Kreuzberg::Element.new(
+  #     element_id: "elem-abc123",
+  #     element_type: "narrative_text",
+  #     text: "This is the main content.",
+  #     metadata: metadata
+  #   )
+  #   puts "#{element.element_type}: #{element.text}"
+  #
+  class Element < T::Struct
+    extend T::Sig
+
+    const :element_id, String
+
+    const :element_type, String
+
+    const :text, String
+
+    const :metadata, ElementMetadata
+  end
+
+  # Header/Heading metadata
+  #
+  # Represents a heading element found in the HTML document
+  #
+  # @example
+  #   header = Kreuzberg::HeaderMetadata.new(
+  #     level: 1,
+  #     text: "Main Title",
+  #     id: "main-title",
+  #     depth: 0,
+  #     html_offset: 245
+  #   )
+  #   puts "#{header.text} (H#{header.level})"
+  #
+  class HeaderMetadata < T::Struct
+    extend T::Sig
+
+    const :level, Integer
+
+    const :text, String
+
+    const :id, T.nilable(String)
+
+    const :depth, Integer
+
+    const :html_offset, Integer
+  end
+
+  # Link metadata
+  #
+  # Represents a link element found in the HTML document
+  #
+  # @example
+  #   link = Kreuzberg::LinkMetadata.new(
+  #     href: "https://example.com",
+  #     text: "Example",
+  #     title: "Example Site",
+  #     link_type: "external",
+  #     rel: ["noopener", "noreferrer"],
+  #     attributes: { "data-id" => "123" }
+  #   )
+  #   puts "#{link.text} -> #{link.href}"
+  #
+  class LinkMetadata < T::Struct
+    extend T::Sig
+
+    const :href, String
+
+    const :text, String
+
+    const :title, T.nilable(String)
+
+    const :link_type, String
+
+    const :rel, T::Array[String]
+
+    const :attributes, T::Hash[String, String]
+  end
+
+  # Image metadata
+  #
+  # Represents an image element found in the HTML document
+  #
+  # @example
+  #   image = Kreuzberg::ImageMetadata.new(
+  #     src: "images/logo.png",
+  #     alt: "Company Logo",
+  #     title: nil,
+  #     dimensions: [200, 100],
+  #     image_type: "png",
+  #     attributes: { "loading" => "lazy" }
+  #   )
+  #   if image.dimensions
+  #     width, height = image.dimensions
+  #     puts "#{width}x#{height}"
+  #   end
+  #
+  class ImageMetadata < T::Struct
+    extend T::Sig
+
+    const :src, String
+
+    const :alt, T.nilable(String)
+
+    const :title, T.nilable(String)
+
+    const :dimensions, T.nilable(T::Array[Integer])
+
+    const :image_type, String
+
+    const :attributes, T::Hash[String, String]
+  end
+
+  # Structured data metadata
+  #
+  # Represents structured data (JSON-LD, microdata, etc.) found in the HTML document
+  #
+  # @example
+  #   structured = Kreuzberg::StructuredData.new(
+  #     data_type: "json-ld",
+  #     raw_json: '{"@context":"https://schema.org","@type":"Article",...}',
+  #     schema_type: "Article"
+  #   )
+  #   data = JSON.parse(structured.raw_json)
+  #   puts data['@type']
+  #
+  class StructuredData < T::Struct
+    extend T::Sig
+
+    const :data_type, String
+
+    const :raw_json, String
+
+    const :schema_type, T.nilable(String)
+  end
+
+  # @example
+  class HtmlMetadata < T::Struct
+    extend T::Sig
+
+    const :title, T.nilable(String)
+
+    const :description, T.nilable(String)
+
+    const :author, T.nilable(String)
+
+    const :copyright, T.nilable(String)
+
+    const :keywords, T::Array[String]
+
+    const :canonical_url, T.nilable(String)
+
+    const :language, T.nilable(String)
+
+    const :text_direction, T.nilable(String)
+
+    const :mime_type, T.nilable(String)
+
+    const :charset, T.nilable(String)
+
+    const :generator, T.nilable(String)
+
+    const :viewport, T.nilable(String)
+
+    const :theme_color, T.nilable(String)
+
+    const :application_name, T.nilable(String)
+
+    const :robots, T.nilable(String)
+
+    const :open_graph, T::Hash[String, String]
+
+    const :twitter_card, T::Hash[String, String]
+
+    const :meta_tags, T::Hash[String, String]
+
+    const :headers, T::Array[HeaderMetadata]
+
+    const :links, T::Array[LinkMetadata]
+
+    const :images, T::Array[ImageMetadata]
+
+    const :structured_data, T::Array[StructuredData]
+  end
+
+  # Extracted keyword with relevance metadata.
+  #
+  # Represents a single keyword extracted from text along with its relevance score,
+  # the algorithm that extracted it, and optional position information.
+  #
+  # @example
+  #   keyword = Kreuzberg::ExtractedKeyword.new(
+  #     text: "machine learning",
+  #     score: 0.95,
+  #     algorithm: "yake",
+  #     positions: [42, 128]
+  #   )
+  #   puts "#{keyword.text}: #{keyword.score}"
+  #
+  class ExtractedKeyword < T::Struct
+    extend T::Sig
+
+    const :text, String
+
+    const :score, Float
+
+    const :algorithm, String
+
+    const :positions, T.nilable(T::Array[Integer])
+  end
+
+  # Processing warning from a pipeline stage.
+  #
+  # Represents a non-fatal warning generated during document processing.
+  #
+  # @example
+  #   warning = Kreuzberg::ProcessingWarning.new(
+  #     source: "ocr",
+  #     message: "Low confidence on page 3"
+  #   )
+  #   puts "[#{warning.source}] #{warning.message}"
+  #
+  class ProcessingWarning < T::Struct
+    extend T::Sig
+
+    const :source, String
+
+    const :message, String
+  end
+
+  # Bounding box for document node positioning.
+  #
+  # Represents rectangular coordinates for a node within the document.
+  #
+  # @example
+  #   bbox = Kreuzberg::DocumentBoundingBox.new(
+  #     x0: 10.0,
+  #     y0: 20.0,
+  #     x1: 100.0,
+  #     y1: 50.0
+  #   )
+  #
+  class DocumentBoundingBox < T::Struct
+    extend T::Sig
+
+    const :x0, Float
+
+    const :y0, Float
+
+    const :x1, Float
+
+    const :y1, Float
+  end
+
+  # Annotation for a document node.
+  #
+  # Provides additional metadata about document node content.
+  #
+  class DocumentAnnotation < T::Struct
+    extend T::Sig
+
+    const :key, String
+
+    const :value, String
+  end
+
+  # Single node in the document structure tree.
+  #
+  # Represents a logical unit of content with deterministic ID, content,
+  # tree structure information, and metadata.
+  #
+  # @example
+  #   node = Kreuzberg::DocumentNode.new(
+  #     id: "node-abc123",
+  #     content: "This is the content",
+  #     parent: nil,
+  #     children: [],
+  #     content_layer: "body",
+  #     page: 1,
+  #     page_end: 1,
+  #     bbox: bbox,
+  #     annotations: []
+  #   )
+  #
+  class DocumentNode < T::Struct
+    extend T::Sig
+
+    const :id, String
+
+    const :content, String
+
+    const :parent, T.nilable(Integer)
+
+    const :children, T::Array[Integer]
+
+    const :content_layer, String
+
+    const :page, T.nilable(Integer)
+
+    const :page_end, T.nilable(Integer)
+
+    const :bbox, T.nilable(DocumentBoundingBox)
+
+    const :annotations, T::Array[DocumentAnnotation]
+  end
+
+  # Structured document representation.
+  #
+  # Provides a hierarchical, tree-based representation of document content
+  # using a flat array of nodes with index-based parent/child references.
+  #
+  # @example
+  #   structure = Kreuzberg::DocumentStructure.new(
+  #     nodes: [node1, node2, node3]
+  #   )
+  #   structure.nodes.each do |node|
+  #     puts "#{node.id}: #{node.content}"
+  #   end
+  #
+  class DocumentStructure < T::Struct
+    extend T::Sig
+
+    const :nodes, T::Array[DocumentNode]
+  end
+end

data/lib/kreuzberg/validator_protocol.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  # @example Implementing a minimum length validator
+  # @example Implementing a content quality validator
+  # @example Using a Proc as a validator
+  module ValidatorProtocol
+    # @param result [Hash] Extraction result to validate with the following structure:
+    # @return [void]
+    # @raise [Kreuzberg::Errors::ValidationError] if validation fails
+    # @example
+    def call(result)
+      raise NotImplementedError, "#{self.class} must implement #call(result)"
+    end
+  end
+end

data/lib/kreuzberg/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Kreuzberg
+  VERSION = '4.3.5'
+end

data/lib/kreuzberg.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative 'kreuzberg/setup_lib_path'
+Kreuzberg::SetupLibPath.configure
+
+require_relative 'kreuzberg/version'
+require 'kreuzberg_rb'
+
+# Kreuzberg is a Ruby binding for the Rust core library providing document extraction,
+# text extraction, and OCR capabilities.
+module Kreuzberg
+  autoload :Config, 'kreuzberg/config'
+  autoload :Result, 'kreuzberg/result'
+  autoload :CLI, 'kreuzberg/cli'
+  autoload :CLIProxy, 'kreuzberg/cli_proxy'
+  autoload :APIProxy, 'kreuzberg/api_proxy'
+  autoload :MCPProxy, 'kreuzberg/mcp_proxy'
+  autoload :Errors, 'kreuzberg/errors'
+  autoload :ErrorContext, 'kreuzberg/error_context'
+  autoload :PostProcessorProtocol, 'kreuzberg/post_processor_protocol'
+  autoload :ValidatorProtocol, 'kreuzberg/validator_protocol'
+  autoload :OcrBackendProtocol, 'kreuzberg/ocr_backend_protocol'
+
+  autoload :BoundingBox, 'kreuzberg/types'
+  autoload :ElementMetadata, 'kreuzberg/types'
+  autoload :Element, 'kreuzberg/types'
+  autoload :HtmlMetadata, 'kreuzberg/types'
+  autoload :HeaderMetadata, 'kreuzberg/types'
+  autoload :LinkMetadata, 'kreuzberg/types'
+  autoload :ImageMetadata, 'kreuzberg/types'
+  autoload :StructuredData, 'kreuzberg/types'
+  autoload :ExtractedKeyword, 'kreuzberg/types'
+  autoload :ProcessingWarning, 'kreuzberg/types'
+  autoload :DocumentBoundingBox, 'kreuzberg/types'
+  autoload :DocumentAnnotation, 'kreuzberg/types'
+  autoload :DocumentNode, 'kreuzberg/types'
+  autoload :DocumentStructure, 'kreuzberg/types'
+
+  ExtractionConfig = Config::Extraction
+  PageConfig = Config::PageConfig
+
+  module KeywordAlgorithm
+    YAKE = :yake
+    RAKE = :rake
+  end
+
+  @__cache_tracker = { entries: 0, bytes: 0 }
+
+  class << self
+    alias native_extract_file_sync extract_file_sync
+    alias native_extract_bytes_sync extract_bytes_sync
+    alias native_batch_extract_files_sync batch_extract_files_sync
+    alias native_extract_file extract_file
+    alias native_extract_bytes extract_bytes
+    alias native_batch_extract_files batch_extract_files
+    alias native_batch_extract_bytes_sync batch_extract_bytes_sync
+    alias native_batch_extract_bytes batch_extract_bytes
+    alias native_clear_cache clear_cache
+    alias native_cache_stats cache_stats
+
+    private :native_extract_file_sync, :native_extract_bytes_sync, :native_batch_extract_files_sync
+    private :native_extract_file, :native_extract_bytes, :native_batch_extract_files
+    private :native_batch_extract_bytes_sync, :native_batch_extract_bytes
+  end
+
+  module_function :register_post_processor
+
+  module_function :unregister_post_processor
+
+  module_function :clear_post_processors
+
+  module_function :register_validator
+
+  module_function :unregister_validator
+
+  module_function :clear_validators
+
+  module_function :list_validators
+
+  module_function :list_post_processors
+
+  module_function :register_ocr_backend
+
+  module_function :unregister_ocr_backend
+
+  module_function :list_ocr_backends
+
+  module_function :detect_mime_type
+
+  module_function :detect_mime_type_from_path
+
+  module_function :validate_mime_type
+
+  module_function :get_extensions_for_mime
+end
+
+require_relative 'kreuzberg/cache_api'
+require_relative 'kreuzberg/extraction_api'
+require_relative 'kreuzberg/djot_content'
+
+Kreuzberg.singleton_class.prepend(Kreuzberg::CacheAPI)
+Kreuzberg.singleton_class.prepend(Kreuzberg::ExtractionAPI)