langextract 0.3.0 → 0.4.0

data/lib/langextract/core/fuzzy_token_stream.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require_relative "data"
+require_relative "tokenizer"
+
+module LangExtract
+  module Core
+    # Builds the temporary source-token stream used only by fuzzy alignment.
+    # Canonical tokenizer output remains unchanged for exact and token offsets.
+    class FuzzyTokenStream
+      DASH_PUNCTUATION = /\p{Pd}/u
+      # Commas are the grouping variant covered by the fuzzy contract.  Keep
+      # periods intact so sentence-boundary barriers remain visible.
+      NUMERIC_SEPARATOR = /,/u
+      APOSTROPHE = /[\u0027\u2019]/u
+      SEPARATOR_PUNCTUATION = /\A(?:\p{Pd}|,)\z/u
+
+      def initialize(tokens)
+        @tokens = tokens
+      end
+
+      def tokens_in(range)
+        in_range = tokens.filter do |token|
+          token.char_interval.start_pos >= range.begin && token.char_interval.end_pos <= range.end
+        end
+        pieces = in_range.flat_map { |token| split_boundary_punctuation(token) }
+        merge_apostrophe_tokens(pieces).reject { |token| separator_punctuation?(token.text) }
+      end
+
+      private
+
+      attr_reader :tokens
+
+      def split_boundary_punctuation(token)
+        pieces = []
+        cursor = 0
+        token.text.to_enum(:scan, /#{DASH_PUNCTUATION}|(?<=\p{N})#{NUMERIC_SEPARATOR}(?=\p{N})/u).each do
+          match = Regexp.last_match
+          append_token_piece(token, pieces, cursor, match.begin(0))
+          append_token_piece(token, pieces, match.begin(0), match.end(0))
+          cursor = match.end(0)
+        end
+        append_token_piece(token, pieces, cursor, token.text.length)
+        pieces
+      end
+
+      def merge_apostrophe_tokens(pieces)
+        merged = []
+        index = 0
+        while index < pieces.length
+          current = pieces[index]
+          if apostrophe?(current.text) && contiguous_text_pieces?(merged.last, pieces[index + 1], current)
+            merged << merged_apostrophe_token(merged.pop, current, pieces[index + 1], merged.length)
+            index += 2
+          else
+            merged << current
+            index += 1
+          end
+        end
+        reindex(merged)
+      end
+
+      def merged_apostrophe_token(previous, apostrophe, following, index)
+        Token.new(
+          text: previous.text + apostrophe.text + following.text,
+          char_interval: CharInterval.new(
+            start_pos: previous.char_interval.start_pos,
+            end_pos: following.char_interval.end_pos
+          ),
+          index: index
+        )
+      end
+
+      def reindex(tokens)
+        tokens.each_with_index.map do |token, index|
+          Token.new(text: token.text, char_interval: token.char_interval, index: index)
+        end
+      end
+
+      def contiguous_text_pieces?(previous, following, apostrophe)
+        previous && following &&
+          previous.char_interval.end_pos == apostrophe.char_interval.start_pos &&
+          apostrophe.char_interval.end_pos == following.char_interval.start_pos &&
+          previous.text.match?(text_piece_pattern) && following.text.match?(text_piece_pattern)
+      end
+
+      def text_piece_pattern
+        /\A[\p{L}\p{N}_-]+\z/u
+      end
+
+      def apostrophe?(text)
+        text.match?(APOSTROPHE)
+      end
+
+      def separator_punctuation?(text)
+        text.match?(SEPARATOR_PUNCTUATION)
+      end
+
+      def append_token_piece(token, pieces, start_pos, end_pos)
+        return if start_pos >= end_pos
+
+        pieces << Token.new(
+          text: token.text[start_pos...end_pos],
+          char_interval: CharInterval.new(
+            start_pos: token.char_interval.start_pos + start_pos,
+            end_pos: token.char_interval.start_pos + end_pos
+          ),
+          index: pieces.length
+        )
+      end
+    end
+  end
+end

data/lib/langextract/core/resolver.rb

@@ -1,37 +1,34 @@
 # frozen_string_literal: true
 
 require_relative "data"
+require_relative "fuzzy_aligner"
+require_relative "fuzzy_token_stream"
+require_relative "token_similarity"
 require_relative "tokenizer"
 
 module LangExtract
   module Core
-    module SimilarityUpperBound
-      module_function
-
-      def passes?(target, candidate, target_counts, threshold)
-        denominator = target.length + candidate.length
-        length_matches = [target.length, candidate.length].min
-        return false if (2.0 * length_matches / denominator) < threshold
-
-        candidate_counts = candidate.each_char.tally
-        character_matches = target_counts.sum { |character, count| [count, candidate_counts[character].to_i].min }
-        (2.0 * character_matches / denominator) >= threshold
-      end
-    end
-
+    # Resolves model extraction text to exact or fuzzy source intervals.
     class Resolver
       DEFAULT_FUZZY_THRESHOLD = 0.78
-      MAX_FUZZY_CANDIDATE_STARTS = 4_000
-      Match = Data.define(:left_start, :right_start, :span_length)
+      DEFAULT_MIN_COVERAGE = 0.70
+      DEFAULT_MIN_DENSITY = 0.50
+      MAX_FUZZY_RANGE_TOKENS = 20_000
+      FuzzySourceContext = Data.define(:tokens, :index)
 
       def initialize(text:, tokenizer: UnicodeTokenizer.new, fuzzy_threshold: DEFAULT_FUZZY_THRESHOLD,
-                     allow_overlaps: false, suppress_alignment_errors: true)
+                     allow_overlaps: false, suppress_alignment_errors: true,
+                     min_coverage: DEFAULT_MIN_COVERAGE, min_density: DEFAULT_MIN_DENSITY)
         @text = text
         @tokenizer = tokenizer
         @fuzzy_threshold = fuzzy_threshold
         @allow_overlaps = allow_overlaps
         @suppress_alignment_errors = suppress_alignment_errors
+        @min_coverage = min_coverage
+        @min_density = min_density
         @tokens = tokenizer.tokenize(text)
+        @fuzzy_token_stream = FuzzyTokenStream.new(tokens)
+        @fuzzy_contexts = {}
       end
 
       def resolve(items, document_id: nil, preferred_interval: nil)
@@ -44,7 +41,8 @@ module LangExtract
 
       private
 
-      attr_reader :text, :tokens, :fuzzy_threshold, :allow_overlaps, :suppress_alignment_errors
+      attr_reader :text, :tokens, :tokenizer, :fuzzy_threshold, :allow_overlaps, :suppress_alignment_errors,
+                  :min_coverage, :min_density, :fuzzy_token_stream, :fuzzy_contexts
 
       def resolve_one(item, index, document_id, preferred_interval, occupied)
         hash = HashCoercion.stringify_keys(item)
@@ -88,39 +86,44 @@ module LangExtract
       end
 
       def exact_intervals_in_range(extraction_text, range)
+        intervals = case_sensitive_intervals(extraction_text, range)
+        return intervals unless intervals.empty?
+
+        case_insensitive_intervals(extraction_text, range)
+      end
+
+      def case_sensitive_intervals(extraction_text, range)
         intervals = []
         cursor = range.begin
-        range_end = range.end
-        while cursor < range_end
+        while cursor < range.end
           match_pos = text.index(extraction_text, cursor)
-          break unless match_pos && match_pos < range_end
+          break unless match_pos && match_pos < range.end
 
           end_pos = match_pos + extraction_text.length
-          intervals << CharInterval.new(start_pos: match_pos, end_pos: end_pos) if end_pos <= range_end
+          intervals << CharInterval.new(start_pos: match_pos, end_pos: end_pos) if end_pos <= range.end
           cursor = match_pos + 1
         end
-        if intervals.empty?
-          pattern = Regexp.new(Regexp.escape(extraction_text), Regexp::IGNORECASE)
-          cursor = range.begin
-          while (match = pattern.match(text, cursor)) && match.begin(0) < range_end
-            intervals << CharInterval.new(start_pos: match.begin(0), end_pos: match.end(0)) if match.end(0) <= range_end
-            cursor = match.begin(0) + 1
-          end
-        end
+        intervals
+      end
 
+      def case_insensitive_intervals(extraction_text, range)
+        intervals = []
+        pattern = Regexp.new(Regexp.escape(extraction_text), Regexp::IGNORECASE)
+        cursor = range.begin
+        while (match = pattern.match(text, cursor)) && match.begin(0) < range.end
+          intervals << CharInterval.new(start_pos: match.begin(0), end_pos: match.end(0)) if match.end(0) <= range.end
+          cursor = match.begin(0) + 1
+        end
         intervals
       end
 
       def find_fuzzy(extraction_text, preferred_interval, occupied)
-        target = normalize_for_match(extraction_text)
-        return nil if target.empty?
+        target_tokens = tokenize_extraction(extraction_text)
+        return nil if target_tokens.empty?
 
-        target_counts = target.each_char.tally
         overlap_fallback = nil
-
         candidate_search_ranges(preferred_interval).each do |range|
-          candidates = fuzzy_candidates_in_range(extraction_text, target, target_counts, range, occupied)
-          candidates = ranked_unique_candidates(candidates)
+          candidates = fuzzy_candidates(target_tokens, range, occupied)
           overlap_fallback ||= candidates.first&.first
           non_overlap = candidates.find { |interval, _score| allow_overlaps || !overlaps_any?(interval, occupied) }
           return [non_overlap.first, AlignmentStatus::FUZZY] if non_overlap
@@ -129,54 +132,49 @@ module LangExtract
         overlap_fallback && [overlap_fallback, AlignmentStatus::OVERLAP]
       end
 
-      def fuzzy_candidates_in_range(extraction_text, normalized_target, target_counts, range, occupied)
-        target_length = extraction_text.length
-        min_length = [1, (target_length * 0.65).floor].max
-        max_length = [(target_length * 1.35).ceil, min_length].max
-        candidates = []
+      def fuzzy_candidates(target_tokens, range, occupied)
+        context = fuzzy_context(range)
+        return [] unless context
 
-        candidate_start_positions(range).each do |start_pos|
-          (min_length..max_length).each do |length|
-            end_pos = start_pos + length
-            next if end_pos > range.end
-
-            candidate = text[start_pos...end_pos]
-            normalized_candidate = normalize_for_match(candidate)
-            next unless SimilarityUpperBound.passes?(normalized_target, normalized_candidate, target_counts,
-                                                     fuzzy_threshold)
+        FuzzyAligner.new(
+          source_tokens: context.tokens,
+          fuzzy_threshold: fuzzy_threshold,
+          min_coverage: min_coverage,
+          min_density: min_density,
+          allow_overlaps: allow_overlaps,
+          alignment_index: context.index
+        ).candidates(target_tokens, range, occupied)
+      end
 
-            score = similarity(normalized_target, normalized_candidate)
-            next if score < fuzzy_threshold
+      def fuzzy_context(range)
+        key = [range.begin, range.end]
+        return fuzzy_contexts[key] if fuzzy_contexts.key?(key)
 
-            match = [CharInterval.new(start_pos: start_pos, end_pos: end_pos), score]
-            interval = match.first
-            return [match] if score >= 1.0 && (allow_overlaps || !overlaps_any?(interval, occupied))
+        # Skip the entire oversized range instead of truncating candidate
+        # starts; preferred chunk ranges and exact alignment remain available.
+        return fuzzy_contexts[key] = nil if canonical_token_count(range) > MAX_FUZZY_RANGE_TOKENS
 
-            candidates << match
-          end
-        end
+        source_tokens = fuzzy_token_stream.tokens_in(range)
+        return fuzzy_contexts[key] = nil if source_tokens.empty? || source_tokens.length > MAX_FUZZY_RANGE_TOKENS
 
-        candidates
+        fuzzy_contexts[key] = FuzzySourceContext.new(
+          tokens: source_tokens,
+          index: FuzzyAlignmentIndex.new(source_tokens)
+        )
       end
 
-      def candidate_start_positions(range)
-        starts = tokens.filter_map do |token|
-          start_pos = token.char_interval.start_pos
-          start_pos if start_pos >= range.begin && start_pos < range.end
-        end.uniq
-
-        return starts.first(MAX_FUZZY_CANDIDATE_STARTS) unless starts.empty?
-
-        (range.begin...range.end).reject { |position| text[position].match?(/\s/) }.first(MAX_FUZZY_CANDIDATE_STARTS)
+      def canonical_token_count(range)
+        tokens.count do |token|
+          token.char_interval.start_pos >= range.begin && token.char_interval.end_pos <= range.end
+        end
       end
 
-      def ranked_unique_candidates(candidates)
-        best_by_interval = candidates.each_with_object({}) do |(interval, score), result|
-          key = [interval.start_pos, interval.end_pos]
-          result[key] = [interval, score] if result[key].nil? || score > result[key].last
+      def tokenize_extraction(extraction_text)
+        target_tokens = tokenizer.tokenize(extraction_text.unicode_normalize(:nfc))
+        FuzzyTokenStream.new(target_tokens).tokens_in(0...extraction_text.length).filter_map do |token|
+          normalized = TokenSimilarity.normalize(token.text)
+          normalized unless normalized.empty?
         end
-
-        best_by_interval.values.sort_by { |interval, score| [-score, interval.start_pos, interval.end_pos] }
       end
 
       def candidate_search_ranges(preferred_interval)
@@ -215,64 +213,6 @@ module LangExtract
       def overlap_status?(status)
         status == AlignmentStatus::OVERLAP
       end
-
-      def normalize_for_match(value)
-        value.to_s.unicode_normalize(:nfc).downcase.gsub(/\s+/, " ").strip
-      end
-
-      def similarity(left, right)
-        return 1.0 if left == right
-        return 0.0 if left.empty? || right.empty?
-
-        matches = sequence_match_count(left.each_char.to_a, right.each_char.to_a)
-        (2.0 * matches) / (left.length + right.length)
-      end
-
-      def sequence_match_count(left_chars, right_chars, left_range = 0...left_chars.length,
-                               right_range = 0...right_chars.length)
-        match = longest_common_substring(left_chars, right_chars, left_range, right_range)
-        return 0 unless match.span_length.positive?
-
-        left_count = sequence_match_count(
-          left_chars,
-          right_chars,
-          left_range.begin...match.left_start,
-          right_range.begin...match.right_start
-        )
-        right_count = sequence_match_count(
-          left_chars,
-          right_chars,
-          (match.left_start + match.span_length)...left_range.end,
-          (match.right_start + match.span_length)...right_range.end
-        )
-
-        match.span_length + left_count + right_count
-      end
-
-      def longest_common_substring(left_chars, right_chars, left_range, right_range)
-        best = Match.new(left_start: left_range.begin, right_start: right_range.begin, span_length: 0)
-        previous_lengths = Array.new(right_range.size + 1, 0)
-
-        left_range.each do |left_index|
-          current_lengths = Array.new(right_range.size + 1, 0)
-          right_range.each_with_index do |right_index, offset|
-            next unless left_chars[left_index] == right_chars[right_index]
-
-            length = previous_lengths[offset] + 1
-            current_lengths[offset + 1] = length
-            next unless length > best.span_length
-
-            best = Match.new(
-              left_start: left_index - length + 1,
-              right_start: right_index - length + 1,
-              span_length: length
-            )
-          end
-          previous_lengths = current_lengths
-        end
-
-        best
-      end
     end
   end
 end

data/lib/langextract/core/token_similarity.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module LangExtract
+  module Core
+    # Per-token similarity gate used by the token-level fuzzy aligner.
+    # Prevents near-word substitutions (humane→human, chart→cart) by
+    # requiring both a length-ratio floor and a character-similarity floor.
+    module TokenSimilarity
+      MIN_LENGTH_RATIO = 0.85
+      MIN_CHAR_SIMILARITY = 0.78
+
+      module_function
+
+      def similar?(target_token, source_token)
+        target = normalize(target_token)
+        source = normalize(source_token)
+        return true if target == source
+        return false if target.empty? || source.empty?
+
+        # Short one-edit substitutions are useful when they are anchored by
+        # another exact token in the same extraction.  The alignment policy enforces
+        # that contextual anchor; keeping the broader lexical gate here lets
+        # it consider those candidates without weakening single-token
+        # grounding.
+        return true if short_typo?(target, source)
+
+        ratio = [target.length, source.length].min.to_f / [target.length, source.length].max
+        return false if ratio < MIN_LENGTH_RATIO
+
+        char_similarity(target, source) >= MIN_CHAR_SIMILARITY
+      end
+
+      def normalize(value)
+        normalized = value.to_s.unicode_normalize(:nfc).downcase
+        normalized.gsub(/[\u0027\u2019]/u, "").gsub(/\s+/, " ").strip
+      end
+
+      # A one-character edit is intentionally limited to short words.  Long
+      # near-words (for example `human`/`humane`) remain subject to the
+      # stricter ratio and character-similarity gates above.
+      def short_typo?(target_token, source_token)
+        target = normalize(target_token)
+        source = normalize(source_token)
+        return false if target.empty? || source.empty?
+        return false if target == source
+        return false if target.length < 3 || source.length < 3
+        return false if [target.length, source.length].max > 5
+
+        edit_distance_at_most_one?(target, source)
+      end
+
+      def edit_distance_at_most_one?(left, right)
+        return true if left == right
+        return false if (left.length - right.length).abs > 1
+
+        return equal_length_one_mismatch?(left, right) if left.length == right.length
+
+        one_insertion_or_deletion?(left, right)
+      end
+
+      def equal_length_one_mismatch?(left, right)
+        left.each_char.zip(right.each_char).count { |a, b| a != b } <= 1
+      end
+
+      def one_insertion_or_deletion?(left, right)
+        shorter, longer = left.length < right.length ? [left, right] : [right, left]
+        shorter_chars = shorter.each_char.to_a
+        longer_chars = longer.each_char.to_a
+        short_index = 0
+        long_index = 0
+        edits = 0
+
+        # Keep this bounded to one edit.  The early exits make the routine
+        # linear in the short token length and avoid allocating a matrix.
+        while short_index < shorter.length && long_index < longer.length
+          if shorter_chars[short_index] == longer_chars[long_index]
+            short_index += 1
+          else
+            edits += 1
+            return false if edits > 1
+          end
+          long_index += 1
+        end
+
+        edits + (longer.length - long_index) <= 1
+      end
+
+      def char_similarity(left, right)
+        return 1.0 if left == right
+        return 0.0 if left.empty? || right.empty?
+
+        matches = SequenceMatcher.match_count(left.each_char.to_a, right.each_char.to_a)
+        (2.0 * matches) / (left.length + right.length)
+      end
+    end
+
+    # Longest-common-substring sequence matcher used for per-token character
+    # similarity scoring (equivalent to Python's SequenceMatcher.ratio()).
+    module SequenceMatcher
+      Match = Data.define(:left_start, :right_start, :span_length)
+
+      module_function
+
+      def match_count(left_chars, right_chars)
+        recursive_match_count(left_chars, right_chars, 0...left_chars.length, 0...right_chars.length)
+      end
+
+      def recursive_match_count(left_chars, right_chars, left_range, right_range)
+        match = longest_common_substring(left_chars, right_chars, left_range, right_range)
+        return 0 unless match.span_length.positive?
+
+        left_count = recursive_match_count(
+          left_chars, right_chars,
+          left_range.begin...match.left_start,
+          right_range.begin...match.right_start
+        )
+        right_count = recursive_match_count(
+          left_chars, right_chars,
+          (match.left_start + match.span_length)...left_range.end,
+          (match.right_start + match.span_length)...right_range.end
+        )
+
+        match.span_length + left_count + right_count
+      end
+
+      def longest_common_substring(left_chars, right_chars, left_range, right_range)
+        best = Match.new(left_start: left_range.begin, right_start: right_range.begin, span_length: 0)
+        previous_lengths = Array.new(right_range.size + 1, 0)
+
+        left_range.each do |left_index|
+          current_lengths = Array.new(right_range.size + 1, 0)
+          right_range.each_with_index do |right_index, offset|
+            next unless left_chars[left_index] == right_chars[right_index]
+
+            length = previous_lengths[offset] + 1
+            current_lengths[offset + 1] = length
+            next unless length > best.span_length
+
+            best = Match.new(
+              left_start: left_index - length + 1,
+              right_start: right_index - length + 1,
+              span_length: length
+            )
+          end
+          previous_lengths = current_lengths
+        end
+
+        best
+      end
+    end
+  end
+end

data/lib/langextract/factory.rb

@@ -7,12 +7,13 @@ require_relative "providers/ruby_llm"
 
 module LangExtract
   class ModelConfig
-    attr_reader :adapter, :provider, :model, :options
+    attr_reader :adapter, :provider, :model, :options, :structured_output
 
-    def initialize(adapter: "ruby_llm", provider: nil, model: nil, **options)
+    def initialize(adapter: "ruby_llm", provider: nil, model: nil, structured_output: false, **options)
       @adapter = adapter.to_s
       @provider = provider&.to_s
       @model = model || LangExtract.config.default_model
+      @structured_output = structured_output
       @options = options.freeze
 
       validate!
@@ -24,6 +25,7 @@ module LangExtract
         "adapter" => adapter,
         "provider" => provider,
         "model" => model,
+        "structured_output" => structured_output,
         "options" => options
       }
     end
@@ -34,6 +36,10 @@ module LangExtract
       raise Core::InvalidModelConfigError, "adapter is required" if adapter.empty?
       raise Core::InvalidModelConfigError, "provider cannot be blank" if provider == ""
       raise Core::InvalidModelConfigError, "model cannot be blank" if model == ""
+      return if [true, false].include?(structured_output)
+
+      raise Core::InvalidModelConfigError,
+            "structured_output must be a boolean, got: #{structured_output.inspect}"
     end
   end
 

data/lib/langextract/providers/ruby_llm.rb

@@ -2,15 +2,37 @@
 
 require_relative "base"
 require_relative "../core/types"
+require "json"
 require "timeout"
 
 module LangExtract
   module Providers
     class RubyLLMProvider < Base
+      INTERNAL_EXTRACTION_SCHEMA = {
+        "type" => "object",
+        "properties" => {
+          "extractions" => {
+            "type" => "array",
+            "items" => {
+              "type" => "object",
+              "properties" => {
+                "text" => { "type" => "string" },
+                "extraction_class" => { "type" => "string" },
+                "description" => { "type" => "string" },
+                "attributes" => { "type" => "object" },
+                "group_id" => { "type" => "string" }
+              },
+              "required" => ["text"]
+            }
+          }
+        },
+        "required" => ["extractions"]
+      }.freeze
+
       def infer(prompt:)
         require "ruby_llm"
 
-        response = RubyLLM.chat(**chat_options).ask(prompt)
+        response = build_chat.ask(prompt)
         InferenceResult.new(text: extract_text(response), raw: response)
       rescue LoadError => e
         raise Core::ProviderConfigError, "ruby_llm is required for live provider inference: #{e.message}"
@@ -23,6 +45,12 @@ module LangExtract
 
       private
 
+      def build_chat
+        chat = RubyLLM.chat(**chat_options)
+        chat.with_schema(INTERNAL_EXTRACTION_SCHEMA) if config.structured_output
+        chat
+      end
+
       def provider_error_class(error)
         return Core::ProviderTimeoutError if timeout_error?(error)
         return Core::ProviderAuthError if ruby_llm_error?(error, :UnauthorizedError, :ForbiddenError,
@@ -57,7 +85,9 @@ module LangExtract
       end
 
       def extract_text(response)
-        return response.content.to_s if response.respond_to?(:content)
+        content = response.content
+        return JSON.generate(content) if content.is_a?(Hash) || content.is_a?(Array)
+        return content.to_s if response.respond_to?(:content)
         return response.text.to_s if response.respond_to?(:text)
 
         response.to_s

data/lib/langextract/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LangExtract
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end