langextract 0.1.0

data/lib/langextract/core/data.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "types"
+
+module LangExtract
+  module Core
+    module HashCoercion
+      module_function
+
+      def stringify_keys(hash)
+        (hash || {}).each_with_object({}) do |(key, value), result|
+          result[key.to_s] = coerce_value(value)
+        end
+      end
+
+      def coerce_value(value)
+        case value
+        when Hash
+          stringify_keys(value)
+        when Array
+          value.map { |entry| coerce_value(entry) }.freeze
+        else
+          value
+        end
+      end
+
+      def read(hash, *keys)
+        keys.each do |key|
+          return hash[key] if hash.key?(key)
+          return hash[key.to_s] if hash.key?(key.to_s)
+        end
+        nil
+      end
+    end
+
+    class CharInterval
+      attr_reader :start_pos, :end_pos
+
+      def initialize(start_pos:, end_pos:)
+        raise ArgumentError, "start_pos must be non-negative" if start_pos.negative?
+        raise ArgumentError, "end_pos must be >= start_pos" if end_pos < start_pos
+
+        @start_pos = start_pos
+        @end_pos = end_pos
+        freeze
+      end
+
+      def length
+        end_pos - start_pos
+      end
+
+      def overlaps?(other)
+        start_pos < other.end_pos && other.start_pos < end_pos
+      end
+
+      def contains?(other)
+        start_pos <= other.start_pos && end_pos >= other.end_pos
+      end
+
+      def shift(offset)
+        self.class.new(start_pos: start_pos + offset, end_pos: end_pos + offset)
+      end
+
+      def to_h
+        { "start_pos" => start_pos, "end_pos" => end_pos }
+      end
+
+      def self.from_h(hash)
+        start_pos = HashCoercion.read(hash, :start_pos)
+        end_pos = HashCoercion.read(hash, :end_pos)
+        raise ArgumentError, "start_pos is required" if start_pos.nil?
+        raise ArgumentError, "end_pos is required" if end_pos.nil?
+
+        new(
+          start_pos: start_pos.to_i,
+          end_pos: end_pos.to_i
+        )
+      end
+
+      def ==(other)
+        other.is_a?(self.class) && start_pos == other.start_pos && end_pos == other.end_pos
+      end
+      alias eql? ==
+
+      def hash
+        [start_pos, end_pos].hash
+      end
+
+      def to_s
+        "[#{start_pos}, #{end_pos})"
+      end
+    end
+
+    class TokenInterval < CharInterval; end
+
+    class Extraction
+      attr_reader :extraction_class, :text, :description, :attributes, :char_interval,
+                  :token_interval, :alignment_status, :extraction_index, :group_id,
+                  :document_id
+
+      def initialize(text:, extraction_class: nil, description: nil, attributes: {},
+                     char_interval: nil, token_interval: nil,
+                     alignment_status: AlignmentStatus::UNGROUNDED,
+                     extraction_index: nil, group_id: nil, document_id: nil)
+        raise ArgumentError, "text is required" if text.nil?
+        raise ArgumentError, "invalid alignment status" unless AlignmentStatus.valid?(alignment_status)
+
+        @extraction_class = extraction_class
+        @text = text
+        @description = description
+        @attributes = HashCoercion.stringify_keys(attributes).freeze
+        @char_interval = char_interval
+        @token_interval = token_interval
+        @alignment_status = alignment_status
+        @extraction_index = extraction_index
+        @group_id = group_id
+        @document_id = document_id
+        freeze
+      end
+
+      def grounded?
+        char_interval && ![AlignmentStatus::UNGROUNDED, AlignmentStatus::ERROR].include?(alignment_status)
+      end
+
+      def to_h
+        {
+          "extraction_class" => extraction_class,
+          "text" => text,
+          "description" => description,
+          "attributes" => attributes,
+          "char_interval" => char_interval&.to_h,
+          "token_interval" => token_interval&.to_h,
+          "alignment_status" => alignment_status,
+          "extraction_index" => extraction_index,
+          "group_id" => group_id,
+          "document_id" => document_id
+        }
+      end
+
+      def self.from_h(hash)
+        new(
+          extraction_class: HashCoercion.read(hash, :extraction_class, :class, :type),
+          text: HashCoercion.read(hash, :text, :extraction_text).to_s,
+          description: HashCoercion.read(hash, :description),
+          attributes: HashCoercion.read(hash, :attributes) || {},
+          char_interval: interval_from_hash(HashCoercion.read(hash, :char_interval), CharInterval),
+          token_interval: interval_from_hash(HashCoercion.read(hash, :token_interval), TokenInterval),
+          alignment_status: HashCoercion.read(hash, :alignment_status) || AlignmentStatus::UNGROUNDED,
+          extraction_index: HashCoercion.read(hash, :extraction_index),
+          group_id: HashCoercion.read(hash, :group_id),
+          document_id: HashCoercion.read(hash, :document_id)
+        )
+      end
+
+      def self.interval_from_hash(value, interval_class)
+        return value if value.is_a?(interval_class)
+        return nil if value.nil?
+
+        interval_class.from_h(value)
+      end
+    end
+
+    class Document
+      attr_reader :text, :id, :metadata
+
+      def initialize(text:, id: nil, metadata: {})
+        raise ArgumentError, "text is required" if text.nil?
+
+        @text = text
+        @id = id
+        @metadata = HashCoercion.stringify_keys(metadata).freeze
+        freeze
+      end
+
+      def to_h
+        { "id" => id, "text" => text, "metadata" => metadata }
+      end
+
+      def self.from_h(hash)
+        new(
+          id: HashCoercion.read(hash, :id),
+          text: HashCoercion.read(hash, :text).to_s,
+          metadata: HashCoercion.read(hash, :metadata) || {}
+        )
+      end
+    end
+
+    class AnnotatedDocument
+      attr_reader :document, :extractions
+
+      def initialize(document:, extractions: [])
+        @document = document
+        @extractions = extractions.map { |item| item.is_a?(Extraction) ? item : Extraction.from_h(item) }.freeze
+        freeze
+      end
+
+      def id
+        document.id
+      end
+
+      def text
+        document.text
+      end
+
+      def metadata
+        document.metadata
+      end
+
+      def to_h
+        {
+          "document" => document.to_h,
+          "extractions" => extractions.map(&:to_h)
+        }
+      end
+
+      def self.from_h(hash)
+        document_hash = HashCoercion.read(hash, :document)
+        document = Document.from_h(document_hash || hash)
+        extractions = Array(HashCoercion.read(hash, :extractions)).map { |item| Extraction.from_h(item) }
+
+        new(document: document, extractions: extractions)
+      end
+    end
+
+    class ExampleData
+      attr_reader :text, :extractions
+
+      def initialize(text:, extractions:)
+        raise ArgumentError, "text is required" if text.nil?
+
+        @text = text
+        @extractions = extractions.map { |item| normalize_extraction(item) }.freeze
+        freeze
+      end
+
+      def to_h
+        {
+          "text" => text,
+          "extractions" => extractions
+        }
+      end
+
+      def self.from_h(hash)
+        new(
+          text: HashCoercion.read(hash, :text).to_s,
+          extractions: Array(HashCoercion.read(hash, :extractions))
+        )
+      end
+
+      private
+
+      def normalize_extraction(item)
+        return example_hash_from_extraction(item) if item.is_a?(Extraction)
+
+        HashCoercion.stringify_keys(item)
+      end
+
+      def example_hash_from_extraction(extraction)
+        {
+          "extraction_class" => extraction.extraction_class,
+          "text" => extraction.text,
+          "description" => extraction.description,
+          "attributes" => extraction.attributes,
+          "group_id" => extraction.group_id
+        }
+      end
+    end
+  end
+end

data/lib/langextract/core/errors.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module LangExtract
+  module Core
+    class Error < StandardError; end
+    class InvalidModelConfigError < Error; end
+    class ProviderConfigError < Error; end
+    class FormatParsingError < Error; end
+    class PromptValidationError < Error; end
+    class AlignmentError < Error; end
+    class IOFailure < Error; end
+  end
+end

data/lib/langextract/core/format_handler.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require "json"
+require "yaml"
+
+require_relative "data"
+
+module LangExtract
+  module Core
+    class FormatHandler
+      FENCE_PATTERN = /```(?:json|yaml|yml)?\s*(.*?)```/im
+
+      def parse(output, format: :auto, strict: true, schema: nil)
+        candidates = fenced_payloads(output)
+        candidates << output.to_s if candidates.empty?
+
+        parsed = parse_first(candidates, format)
+        normalize_extractions(parsed).tap do |extractions|
+          validate_schema!(extractions, schema) if schema
+        end
+      rescue FormatParsingError
+        raise if strict
+
+        []
+      end
+
+      private
+
+      def fenced_payloads(output)
+        output.to_s.scan(FENCE_PATTERN).flatten.map(&:strip).reject(&:empty?)
+      end
+
+      def parse_first(candidates, format)
+        errors = []
+
+        candidates.each do |candidate|
+          parse_formats(format).each do |parser_format|
+            parsed = parse_payload(candidate, parser_format)
+            unless parsed.nil? || parsed == false || parsed.is_a?(Hash) || parsed.is_a?(Array)
+              raise FormatParsingError, "parsed payload must be an object or array"
+            end
+
+            return parsed
+          rescue JSON::ParserError, Psych::Exception, TypeError, FormatParsingError => e
+            errors << "#{parser_format}: #{e.message}"
+          end
+        end
+
+        raise FormatParsingError, "unable to parse provider output (#{errors.join('; ')})"
+      end
+
+      def parse_formats(format)
+        case format.to_sym
+        when :auto
+          %i[json yaml]
+        when :json, :yaml
+          [format.to_sym]
+        else
+          raise FormatParsingError, "unsupported format: #{format}"
+        end
+      end
+
+      def parse_payload(payload, format)
+        case format
+        when :json
+          JSON.parse(payload)
+        when :yaml
+          YAML.safe_load(payload, permitted_classes: [Symbol], aliases: false)
+        end
+      end
+
+      def normalize_extractions(parsed)
+        payload = unwrap(parsed)
+        extraction_items = payload.is_a?(Array) ? payload : [payload]
+
+        extraction_items.compact.map do |item|
+          normalize_extraction(item)
+        end
+      end
+
+      def unwrap(parsed)
+        return [] if parsed.nil? || parsed == false
+        return parsed unless parsed.is_a?(Hash)
+
+        hash = HashCoercion.stringify_keys(parsed)
+        hash["extractions"] || hash["data"] || hash["items"] || hash
+      end
+
+      def normalize_extraction(item)
+        raise FormatParsingError, "extraction must be an object" unless item.is_a?(Hash)
+
+        hash = HashCoercion.stringify_keys(item)
+        text = hash["text"] || hash["extraction_text"]
+        raise FormatParsingError, "extraction text is required" if text.nil?
+
+        {
+          "extraction_class" => hash["extraction_class"] || hash["class"] || hash["type"],
+          "text" => text.to_s,
+          "description" => hash["description"],
+          "attributes" => hash["attributes"] || {},
+          "group_id" => hash["group_id"]
+        }
+      end
+
+      def validate_schema!(extractions, schema)
+        normalized_schema = HashCoercion.stringify_keys(schema)
+        required = Array(normalized_schema["required"])
+        properties = normalized_schema["properties"] || {}
+
+        extractions.each_with_index do |extraction, index|
+          required.each do |field|
+            value = extraction[field.to_s]
+            raise FormatParsingError, "schema validation failed: extraction #{index} missing #{field}" if blank?(value)
+          end
+
+          validate_property_types!(extraction, properties, index)
+        end
+      end
+
+      def validate_property_types!(extraction, properties, index)
+        properties.each do |field, rule|
+          value = extraction[field]
+          next if value.nil?
+
+          expected_type = rule["type"]
+          next if expected_type.nil? || matches_type?(value, expected_type)
+
+          raise FormatParsingError, "schema validation failed: extraction #{index} #{field} must be #{expected_type}"
+        end
+      end
+
+      def matches_type?(value, expected_type)
+        case expected_type
+        when "string"
+          value.is_a?(String)
+        when "object"
+          value.is_a?(Hash)
+        when "array"
+          value.is_a?(Array)
+        when "number"
+          value.is_a?(Numeric)
+        when "boolean"
+          [true, false].include?(value)
+        else
+          true
+        end
+      end
+
+      def blank?(value)
+        value.nil? || (value.respond_to?(:empty?) && value.empty?)
+      end
+    end
+  end
+end

data/lib/langextract/core/prompt_validation.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative "data"
+require_relative "types"
+
+module LangExtract
+  module Core
+    class PromptValidation
+      MODES = %i[off warning error].freeze
+
+      def initialize(mode: :warning, warning_io: $stderr)
+        @mode = mode.to_sym
+        @warning_io = warning_io
+        raise ArgumentError, "invalid prompt validation mode" unless MODES.include?(@mode)
+      end
+
+      def validate!(examples)
+        return if mode == :off
+
+        issues = examples.flat_map.with_index { |example, index| issues_for(example, index) }
+        return if issues.empty?
+
+        message = "Prompt examples failed validation: #{issues.join('; ')}"
+        raise PromptValidationError, message if mode == :error
+
+        warning_io.puts(message)
+      end
+
+      private
+
+      attr_reader :mode, :warning_io
+
+      def issues_for(example, index)
+        normalized = example.is_a?(ExampleData) ? example : ExampleData.from_h(example)
+        issues = []
+        issues << "example #{index} text is empty" if normalized.text.strip.empty?
+        issues << "example #{index} has no extractions" if normalized.extractions.empty?
+
+        normalized.extractions.each_with_index do |extraction, extraction_index|
+          extraction_text = extraction["text"] || extraction["extraction_text"]
+          if extraction_text.to_s.strip.empty?
+            issues << "example #{index} extraction #{extraction_index} text is empty"
+          elsif !normalized.text.include?(extraction_text.to_s)
+            issues << "example #{index} extraction #{extraction_index} text is not present in example text"
+          end
+        end
+
+        issues
+      end
+    end
+  end
+end

data/lib/langextract/core/prompting.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "json"
+
+require_relative "chunking"
+require_relative "prompt_validation"
+
+module LangExtract
+  module Core
+    class PromptBuilder
+      def initialize(validation_mode: :warning, warning_io: $stderr)
+        @validator = PromptValidation.new(mode: validation_mode, warning_io: warning_io)
+      end
+
+      def build(document:, chunk:, prompt_description:, examples: [], additional_context: nil,
+                context_window_chars: 0, pass_index: nil, total_passes: 1)
+        normalized_examples = examples.map { |example| normalize_example(example) }
+        validator.validate!(normalized_examples)
+
+        sections = []
+        sections << "Task: #{prompt_description}"
+        sections << "Additional context: #{additional_context}" if additional_context
+        sections << pass_section(pass_index, total_passes) if total_passes.to_i > 1
+        sections << context_section(document, chunk, context_window_chars) if context_window_chars.positive?
+        sections << examples_section(normalized_examples) unless normalized_examples.empty?
+        sections << "Text:\n#{chunk.text}"
+        sections << output_contract
+        sections.join("\n\n")
+      end
+
+      private
+
+      attr_reader :validator
+
+      def normalize_example(example)
+        example.is_a?(ExampleData) ? example : ExampleData.from_h(example)
+      end
+
+      def context_section(document, chunk, context_window_chars)
+        before_start = [chunk.char_interval.start_pos - context_window_chars, 0].max
+        after_end = [chunk.char_interval.end_pos + context_window_chars, document.text.length].min
+        before = document.text[before_start...chunk.char_interval.start_pos]
+        after = document.text[chunk.char_interval.end_pos...after_end]
+
+        "Context before:\n#{before}\n\nContext after:\n#{after}"
+      end
+
+      def examples_section(examples)
+        rendered = examples.map.with_index do |example, index|
+          [
+            "Example #{index + 1} text:",
+            example.text,
+            "Example #{index + 1} extractions:",
+            JSON.pretty_generate(example.extractions)
+          ].join("\n")
+        end
+
+        rendered.join("\n\n")
+      end
+
+      def pass_section(pass_index, total_passes)
+        "Extraction pass: #{pass_index + 1} of #{total_passes}. Return grounded extractions for this pass."
+      end
+
+      def output_contract
+        <<~TEXT.strip
+          Return JSON with an "extractions" array. Each extraction must include "text", and may include "extraction_class", "description", "attributes", and "group_id".
+        TEXT
+      end
+    end
+  end
+end