langextract 0.1.0

data/lib/langextract/core/resolver.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+require_relative "data"
+require_relative "tokenizer"
+
+module LangExtract
+  module Core
+    class Resolver
+      DEFAULT_FUZZY_THRESHOLD = 0.78
+      MAX_FUZZY_CANDIDATE_STARTS = 4_000
+      Match = Data.define(:left_start, :right_start, :span_length)
+
+      def initialize(text:, tokenizer: UnicodeTokenizer.new, fuzzy_threshold: DEFAULT_FUZZY_THRESHOLD,
+                     allow_overlaps: false, suppress_alignment_errors: true)
+        @text = text
+        @tokenizer = tokenizer
+        @fuzzy_threshold = fuzzy_threshold
+        @allow_overlaps = allow_overlaps
+        @suppress_alignment_errors = suppress_alignment_errors
+        @tokens = tokenizer.tokenize(text)
+      end
+
+      def resolve(items, document_id: nil, preferred_interval: nil)
+        occupied = []
+
+        items.map.with_index do |item, index|
+          resolve_one(item, index, document_id, preferred_interval, occupied)
+        end
+      end
+
+      private
+
+      attr_reader :text, :tokens, :fuzzy_threshold, :allow_overlaps, :suppress_alignment_errors
+
+      def resolve_one(item, index, document_id, preferred_interval, occupied)
+        hash = HashCoercion.stringify_keys(item)
+        extraction_text = hash.fetch("text").to_s
+
+        interval, status = find_interval(extraction_text, preferred_interval, occupied)
+        occupied << interval if interval && !overlap_status?(status)
+
+        build_extraction(hash, extraction_text, interval, status, index, document_id)
+      rescue AlignmentError
+        raise unless suppress_alignment_errors
+
+        build_extraction(hash || {}, extraction_text || "", nil, AlignmentStatus::ERROR, index, document_id)
+      end
+
+      def find_interval(extraction_text, preferred_interval, occupied)
+        return [nil, AlignmentStatus::UNGROUNDED] if extraction_text.strip.empty?
+
+        exact = find_exact(extraction_text, preferred_interval, occupied)
+        return exact if exact
+
+        fuzzy = find_fuzzy(extraction_text, preferred_interval, occupied)
+        return fuzzy if fuzzy
+
+        raise AlignmentError, "could not align extraction: #{extraction_text}" unless suppress_alignment_errors
+
+        [nil, AlignmentStatus::UNGROUNDED]
+      end
+
+      def find_exact(extraction_text, preferred_interval, occupied)
+        intervals = candidate_search_ranges(preferred_interval).flat_map do |range|
+          exact_intervals_in_range(extraction_text, range)
+        end
+        intervals = intervals.uniq
+        return nil if intervals.empty?
+
+        first_non_overlap = intervals.find { |interval| allow_overlaps || !overlaps_any?(interval, occupied) }
+        return [first_non_overlap, AlignmentStatus::EXACT] if first_non_overlap
+
+        [intervals.first, AlignmentStatus::OVERLAP]
+      end
+
+      def exact_intervals_in_range(extraction_text, range)
+        intervals = []
+        cursor = range.begin
+
+        while cursor < range.end
+          match_pos = text.index(extraction_text, cursor)
+          break unless match_pos && match_pos < range.end
+
+          end_pos = match_pos + extraction_text.length
+          intervals << CharInterval.new(start_pos: match_pos, end_pos: end_pos) if end_pos <= range.end
+          cursor = match_pos + 1
+        end
+
+        if intervals.empty?
+          downcase_text = text[range].downcase
+          downcase_target = extraction_text.downcase
+          local_pos = downcase_text.index(downcase_target)
+          if local_pos
+            intervals << CharInterval.new(
+              start_pos: range.begin + local_pos,
+              end_pos: range.begin + local_pos + extraction_text.length
+            )
+          end
+        end
+
+        intervals
+      end
+
+      def find_fuzzy(extraction_text, preferred_interval, occupied)
+        target = normalize_for_match(extraction_text)
+        return nil if target.empty?
+
+        candidates = candidate_search_ranges(preferred_interval).flat_map do |range|
+          fuzzy_candidates_in_range(extraction_text, target, range)
+        end
+        candidates = ranked_unique_candidates(candidates)
+        return nil if candidates.empty?
+
+        non_overlap = candidates.find { |interval, _score| allow_overlaps || !overlaps_any?(interval, occupied) }
+        return [non_overlap.first, AlignmentStatus::FUZZY] if non_overlap
+
+        [candidates.first.first, AlignmentStatus::OVERLAP]
+      end
+
+      def fuzzy_candidates_in_range(extraction_text, normalized_target, range)
+        target_length = extraction_text.length
+        min_length = [1, (target_length * 0.65).floor].max
+        max_length = [(target_length * 1.35).ceil, min_length].max
+        candidates = []
+
+        candidate_start_positions(range).each do |start_pos|
+          (min_length..max_length).each do |length|
+            end_pos = start_pos + length
+            next if end_pos > range.end
+
+            candidate = text[start_pos...end_pos]
+            score = similarity(normalized_target, normalize_for_match(candidate))
+            next if score < fuzzy_threshold
+
+            candidates << [
+              CharInterval.new(start_pos: start_pos, end_pos: end_pos),
+              score
+            ]
+          end
+        end
+
+        candidates
+      end
+
+      def candidate_start_positions(range)
+        starts = tokens.filter_map do |token|
+          start_pos = token.char_interval.start_pos
+          start_pos if start_pos >= range.begin && start_pos < range.end
+        end.uniq
+
+        return starts.first(MAX_FUZZY_CANDIDATE_STARTS) unless starts.empty?
+
+        (range.begin...range.end).reject { |position| text[position].match?(/\s/) }.first(MAX_FUZZY_CANDIDATE_STARTS)
+      end
+
+      def ranked_unique_candidates(candidates)
+        best_by_interval = candidates.each_with_object({}) do |(interval, score), result|
+          key = [interval.start_pos, interval.end_pos]
+          result[key] = [interval, score] if result[key].nil? || score > result[key].last
+        end
+
+        best_by_interval.values.sort_by { |interval, score| [-score, interval.start_pos, interval.end_pos] }
+      end
+
+      def candidate_search_ranges(preferred_interval)
+        ranges = []
+        ranges << (preferred_interval.start_pos...preferred_interval.end_pos) if preferred_interval
+        ranges << (0...text.length)
+        ranges
+      end
+
+      def build_extraction(hash, extraction_text, interval, status, index, document_id)
+        Extraction.new(
+          extraction_class: hash["extraction_class"],
+          text: extraction_text,
+          description: hash["description"],
+          attributes: hash["attributes"] || {},
+          char_interval: interval,
+          token_interval: interval ? token_interval_for(interval) : nil,
+          alignment_status: status,
+          extraction_index: index,
+          group_id: hash["group_id"],
+          document_id: document_id
+        )
+      end
+
+      def token_interval_for(char_interval)
+        matching = tokens.select { |token| token.char_interval.overlaps?(char_interval) }
+        return nil if matching.empty?
+
+        TokenInterval.new(start_pos: matching.first.index, end_pos: matching.last.index + 1)
+      end
+
+      def overlaps_any?(interval, occupied)
+        occupied.any? { |other| interval.overlaps?(other) }
+      end
+
+      def overlap_status?(status)
+        status == AlignmentStatus::OVERLAP
+      end
+
+      def normalize_for_match(value)
+        value.to_s.unicode_normalize(:nfc).downcase.gsub(/\s+/, " ").strip
+      end
+
+      def similarity(left, right)
+        return 1.0 if left == right
+        return 0.0 if left.empty? || right.empty?
+
+        matches = sequence_match_count(left.each_char.to_a, right.each_char.to_a)
+        (2.0 * matches) / (left.length + right.length)
+      end
+
+      def sequence_match_count(left_chars, right_chars, left_range = 0...left_chars.length,
+                               right_range = 0...right_chars.length)
+        match = longest_common_substring(left_chars, right_chars, left_range, right_range)
+        return 0 unless match.span_length.positive?
+
+        left_count = sequence_match_count(
+          left_chars,
+          right_chars,
+          left_range.begin...match.left_start,
+          right_range.begin...match.right_start
+        )
+        right_count = sequence_match_count(
+          left_chars,
+          right_chars,
+          (match.left_start + match.span_length)...left_range.end,
+          (match.right_start + match.span_length)...right_range.end
+        )
+
+        match.span_length + left_count + right_count
+      end
+
+      def longest_common_substring(left_chars, right_chars, left_range, right_range)
+        best = Match.new(left_start: left_range.begin, right_start: right_range.begin, span_length: 0)
+        previous_lengths = Array.new(right_range.size + 1, 0)
+
+        left_range.each do |left_index|
+          current_lengths = Array.new(right_range.size + 1, 0)
+          right_range.each_with_index do |right_index, offset|
+            next unless left_chars[left_index] == right_chars[right_index]
+
+            length = previous_lengths[offset] + 1
+            current_lengths[offset + 1] = length
+            next unless length > best.span_length
+
+            best = Match.new(
+              left_start: left_index - length + 1,
+              right_start: right_index - length + 1,
+              span_length: length
+            )
+          end
+          previous_lengths = current_lengths
+        end
+
+        best
+      end
+    end
+  end
+end

data/lib/langextract/core/tokenizer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative "data"
+
+module LangExtract
+  module Core
+    Token = Data.define(:text, :char_interval, :index) do
+      def to_h
+        {
+          "text" => text,
+          "char_interval" => char_interval.to_h,
+          "index" => index
+        }
+      end
+    end
+
+    class RegexTokenizer
+      DEFAULT_PATTERN = /\S+/
+
+      def initialize(pattern: DEFAULT_PATTERN)
+        @pattern = pattern
+      end
+
+      def tokenize(text)
+        tokens = []
+        text.to_enum(:scan, @pattern).each do
+          match = Regexp.last_match
+          tokens << Token.new(
+            text: match[0],
+            char_interval: CharInterval.new(start_pos: match.begin(0), end_pos: match.end(0)),
+            index: tokens.length
+          )
+        end
+        tokens.freeze
+      end
+    end
+
+    class UnicodeTokenizer
+      TOKEN_PATTERN = /
+        \p{L}[\p{L}\p{M}\p{N}_'-]* |
+        \p{N}+(?:[.,]\p{N}+)* |
+        [^\s]
+      /ux
+
+      def tokenize(text)
+        RegexTokenizer.new(pattern: TOKEN_PATTERN).tokenize(text)
+      end
+    end
+  end
+end

data/lib/langextract/core/types.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "errors"
+
+module LangExtract
+  module Core
+    module AlignmentStatus
+      EXACT = "exact"
+      FUZZY = "fuzzy"
+      UNGROUNDED = "ungrounded"
+      OVERLAP = "overlap"
+      ERROR = "error"
+
+      ALL = [EXACT, FUZZY, UNGROUNDED, OVERLAP, ERROR].freeze
+
+      def self.valid?(status)
+        ALL.include?(status)
+      end
+    end
+  end
+end

data/lib/langextract/errors.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "core/errors"
+
+module LangExtract
+  Error = Core::Error
+  InvalidModelConfigError = Core::InvalidModelConfigError
+  ProviderConfigError = Core::ProviderConfigError
+  FormatParsingError = Core::FormatParsingError
+  PromptValidationError = Core::PromptValidationError
+  AlignmentError = Core::AlignmentError
+  IOFailure = Core::IOFailure
+end

data/lib/langextract/extractor.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative "core/annotation"
+require_relative "core/chunking"
+require_relative "core/format_handler"
+require_relative "core/prompting"
+
+module LangExtract
+  class Extractor
+    DEFAULT_MAX_CHAR_BUFFER = Core::SentenceAwareChunker::DEFAULT_MAX_CHAR_BUFFER
+
+    def initialize(model:, prompt_description:, examples: [], additional_context: nil,
+                   max_char_buffer: DEFAULT_MAX_CHAR_BUFFER, context_window_chars: 0,
+                   extraction_passes: 1, format: :auto, strict: true,
+                   prompt_validation: :warning, suppress_parse_errors: false,
+                   suppress_alignment_errors: true, allow_overlaps: false,
+                   fuzzy_threshold: Core::Resolver::DEFAULT_FUZZY_THRESHOLD,
+                   tokenizer: Core::UnicodeTokenizer.new)
+      @model = model
+      @prompt_description = prompt_description
+      @examples = examples
+      @additional_context = additional_context
+      @max_char_buffer = max_char_buffer
+      @context_window_chars = context_window_chars
+      @extraction_passes = extraction_passes
+      @format = format
+      @strict = strict
+      @prompt_validation = prompt_validation
+      @suppress_parse_errors = suppress_parse_errors
+      @suppress_alignment_errors = suppress_alignment_errors
+      @allow_overlaps = allow_overlaps
+      @fuzzy_threshold = fuzzy_threshold
+      @tokenizer = tokenizer
+      validate_model!
+    end
+
+    def extract(text: nil, documents: nil)
+      coerced_documents = coerce_documents(text, documents)
+      annotated = coerced_documents.map { |document| extract_document(document) }
+
+      documents.nil? && !text.nil? ? annotated.first : annotated
+    end
+
+    private
+
+    attr_reader :model, :prompt_description, :examples, :additional_context, :max_char_buffer,
+                :context_window_chars, :extraction_passes, :format, :strict, :prompt_validation,
+                :suppress_parse_errors, :suppress_alignment_errors, :allow_overlaps, :fuzzy_threshold,
+                :tokenizer
+
+    def extract_document(document)
+      chunker = Core::SentenceAwareChunker.new(max_char_buffer: max_char_buffer, tokenizer: tokenizer)
+      prompt_builder = Core::PromptBuilder.new(validation_mode: prompt_validation)
+      format_handler = Core::FormatHandler.new
+      extractions = []
+
+      chunker.chunks(document).each do |chunk|
+        extraction_passes.times do |pass_index|
+          raw_output = infer(prompt_builder, document, chunk, pass_index)
+          parsed = parse(format_handler, raw_output)
+          resolver = Core::Resolver.new(
+            text: document.text,
+            tokenizer: tokenizer,
+            fuzzy_threshold: fuzzy_threshold,
+            allow_overlaps: allow_overlaps,
+            suppress_alignment_errors: suppress_alignment_errors
+          )
+          extractions.concat(
+            resolver.resolve(parsed, document_id: document.id, preferred_interval: chunk.char_interval)
+          )
+        end
+      end
+
+      Core::AnnotatedDocument.new(document: document, extractions: merge_extractions(extractions))
+    end
+
+    def infer(prompt_builder, document, chunk, pass_index)
+      prompt = prompt_builder.build(
+        document: document,
+        chunk: chunk,
+        prompt_description: prompt_description,
+        examples: examples,
+        additional_context: additional_context,
+        context_window_chars: context_window_chars,
+        pass_index: pass_index,
+        total_passes: extraction_passes
+      )
+      call_model(prompt)
+    end
+
+    def call_model(prompt)
+      result = model.infer(prompt: prompt)
+
+      result.respond_to?(:text) ? result.text.to_s : result.to_s
+    end
+
+    def parse(format_handler, raw_output)
+      format_handler.parse(raw_output, format: format, strict: strict)
+    rescue Core::FormatParsingError
+      raise unless suppress_parse_errors
+
+      []
+    end
+
+    def merge_extractions(extractions)
+      occupied = []
+      seen = {}
+
+      extractions.filter_map do |extraction|
+        key = merge_key(extraction)
+        next if seen[key]
+
+        seen[key] = true
+        if extraction.grounded?
+          next if !allow_overlaps && occupied.any? { |interval| interval.overlaps?(extraction.char_interval) }
+
+          occupied << extraction.char_interval
+        end
+        extraction
+      end.freeze
+    end
+
+    def merge_key(extraction)
+      interval = extraction.char_interval
+      [
+        extraction.document_id,
+        interval&.start_pos,
+        interval&.end_pos,
+        extraction.text,
+        extraction.description,
+        extraction.extraction_class
+      ]
+    end
+
+    def coerce_documents(text, documents)
+      if documents
+        Array(documents).map.with_index { |document, index| coerce_document(document, index) }
+      elsif text
+        [Core::Document.new(text: text, id: "document_0")]
+      else
+        raise ArgumentError, "text or documents is required"
+      end
+    end
+
+    def coerce_document(document, index)
+      return document if document.is_a?(Core::Document)
+      return Core::Document.from_h(document) if document.is_a?(Hash)
+
+      Core::Document.new(text: document.to_s, id: "document_#{index}")
+    end
+
+    def validate_model!
+      return if model.respond_to?(:infer)
+
+      raise Core::InvalidModelConfigError, "model must respond to #infer(prompt:)"
+    end
+  end
+end

data/lib/langextract/factory.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require_relative "config"
+require_relative "core/errors"
+require_relative "providers/router"
+require_relative "providers/ruby_llm"
+
+module LangExtract
+  class ModelConfig
+    attr_reader :adapter, :provider, :model, :options
+
+    def initialize(adapter: "ruby_llm", provider: nil, model: nil, **options)
+      @adapter = adapter.to_s
+      @provider = provider&.to_s
+      @model = model || LangExtract.config.default_model
+      @options = options.freeze
+
+      validate!
+      freeze
+    end
+
+    def to_h
+      {
+        "adapter" => adapter,
+        "provider" => provider,
+        "model" => model,
+        "options" => options
+      }
+    end
+
+    private
+
+    def validate!
+      raise Core::InvalidModelConfigError, "adapter is required" if adapter.empty?
+      raise Core::InvalidModelConfigError, "provider cannot be blank" if provider == ""
+      raise Core::InvalidModelConfigError, "model cannot be blank" if model == ""
+    end
+  end
+
+  module Factory
+    module_function
+
+    def create_model(config = ModelConfig.new)
+      router.create(config)
+    end
+
+    def router
+      @router ||= default_router
+    end
+
+    def reset_router!
+      @router = default_router
+    end
+
+    def default_router
+      Providers::Router.new.tap do |router|
+        router.register("ruby_llm", Providers::RubyLLMProvider)
+      end
+    end
+  end
+end

data/lib/langextract/io.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "core/data"
+require_relative "core/types"
+
+module LangExtract
+  module IO
+    module_function
+
+    def save_annotated_documents(path, documents)
+      normalized = Array(documents)
+      ::File.open(path, "w") do |file|
+        normalized.each do |document|
+          annotated = document.is_a?(Core::AnnotatedDocument) ? document : Core::AnnotatedDocument.from_h(document)
+          file.puts(JSON.generate(annotated.to_h))
+        end
+      end
+      path
+    rescue SystemCallError, JSON::GeneratorError => e
+      raise Core::IOFailure, "failed to save annotated documents: #{e.message}"
+    end
+
+    def load_annotated_documents_jsonl(path)
+      ::File.readlines(path, chomp: true).filter_map do |line|
+        next if line.strip.empty?
+
+        Core::AnnotatedDocument.from_h(JSON.parse(line))
+      end
+    rescue SystemCallError, JSON::ParserError => e
+      raise Core::IOFailure, "failed to load annotated documents: #{e.message}"
+    end
+  end
+end

data/lib/langextract/providers/base.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module LangExtract
+  module Providers
+    InferenceResult = Data.define(:text, :raw)
+
+    class Base
+      def initialize(config)
+        @config = config
+      end
+
+      def infer(prompt:)
+        raise NotImplementedError, "#{self.class} must implement #infer"
+      end
+
+      private
+
+      attr_reader :config
+    end
+  end
+end

data/lib/langextract/providers/router.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "../core/errors"
+
+module LangExtract
+  module Providers
+    class Router
+      def initialize
+        @providers = {}
+      end
+
+      def register(name, provider_class)
+        providers[name.to_s] = provider_class
+      end
+
+      def fetch(name)
+        providers.fetch(name.to_s) do
+          raise Core::InvalidModelConfigError, "unknown provider adapter: #{name}"
+        end
+      end
+
+      def create(config)
+        fetch(config.adapter).new(config)
+      end
+
+      def names
+        providers.keys.sort
+      end
+
+      private
+
+      attr_reader :providers
+    end
+  end
+end

data/lib/langextract/providers/ruby_llm.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "base"
+require_relative "../core/types"
+
+module LangExtract
+  module Providers
+    class RubyLLMProvider < Base
+      def infer(prompt:)
+        require "ruby_llm"
+
+        response = RubyLLM.chat(**chat_options).ask(prompt)
+        InferenceResult.new(text: extract_text(response), raw: response)
+      rescue LoadError => e
+        raise Core::ProviderConfigError, "ruby_llm is required for live provider inference: #{e.message}"
+      rescue StandardError => e
+        raise Core::ProviderConfigError, "provider inference failed: #{e.message}"
+      end
+
+      private
+
+      def chat_options
+        options = config.options.dup
+        options[:model] = config.model if config.model
+        options[:provider] = config.provider if config.provider
+        options
+      end
+
+      def extract_text(response)
+        return response.content.to_s if response.respond_to?(:content)
+        return response.text.to_s if response.respond_to?(:text)
+
+        response.to_s
+      end
+    end
+  end
+end