json_completer 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2fe03f14437a3cfd88193b2cfc7a7f156e116632b1937a42ea6c4a1aefffa7c2
-  data.tar.gz: b20c23c7843a3ff8f18f0110ed824b54d1306eb440309186efc00c71317d33ba
+  metadata.gz: acf870fc5a65bf4f4f1b586cfa61e012137fc96c0c48e3345d26a38da7d765bf
+  data.tar.gz: fe454f3e2485ae789840bb9c55c7fe72f3e8c9afffc9eccd0d13cc53ede8c05f
 SHA512:
-  metadata.gz: 9815201cb51addf45defae03cb710502ed93091208ec13c404865fdbbd58be2b20773334e3528beef4d82bd93cb3de81d2de22e5ecad996aebdded6e3a138b87
-  data.tar.gz: 261db1237466e85281eb969d90b7f6d90555c72955df728d001122db6f0d0cfd1546e399d3f7185d869d0e689108d49cb63347fffd5f5484dfe24c311e22d193
+  metadata.gz: 5cb2ad4d01e5f204cafc9b7fb4df996e319ba2c0dccdce6a16f23eacd2493cbdf1168c068c15586b7e4643863935ca70fe8e2628ed8e0994439fefa2151c60b7
+  data.tar.gz: 7dcd0cee8613e5a45f9cedbee90386234f31fe36ce832bb5abe160d4993b25a06d891340b9f17c0559026dd8e4b5bb4754a9a350ca480baed8791496f2837437

data/README.md

@@ -1,6 +1,6 @@
 # JsonCompleter
 
-A Ruby gem that converts partial JSON strings into valid JSON with high-performance incremental parsing. Efficiently processes streaming JSON with O(n) complexity for new data by maintaining parsing state between chunks. Handles truncated primitives, missing values, and unclosed structures without reprocessing previously parsed data.
+A Ruby gem for incremental parsing of partial and incomplete JSON streams. It is built for streaming output from LLM providers such as OpenAI and Anthropic, and processes each new chunk in O(n) time by maintaining parser state between calls. Use `.parse` for parsed Ruby values and `.complete` when you specifically need completed JSON text.
 
 ## Installation
 
@@ -26,44 +26,58 @@ gem install json_completer
 
 ### Basic Usage
 
-Complete partial JSON strings in one call:
+Use `.parse` when you want the current parsed Ruby value directly from a partial stream:
 
 ```ruby
 require 'json_completer'
 
-# Complete truncated JSON
-JsonCompleter.complete('{"name": "John", "age":')
-# => '{"name": "John", "age": null}'
+# Parse partial JSON into Ruby objects
+JsonCompleter.parse('{"name": "John", "age":')
+# => {"name" => "John", "age" => nil}
 
 # Handle incomplete strings
-JsonCompleter.complete('{"message": "Hello wo')
-# => '{"message": "Hello wo"}'
+JsonCompleter.parse('{"message": "Hello wo')
+# => {"message" => "Hello wo"}
 
-# Fix unclosed structures
-JsonCompleter.complete('[1, 2, {"key": "value"')
-# => '[1, 2, {"key": "value"}]'
+# Close unclosed structures
+JsonCompleter.parse('[1, 2, {"key": "value"')
+# => [1, 2, {"key" => "value"}]
 ```
 
 ### Incremental Processing
 
-For streaming scenarios where JSON arrives in chunks. Each call processes only new data (O(n) complexity) by maintaining parsing state, making it highly efficient for large streaming responses:
+For streaming scenarios where JSON arrives in chunks. Each call processes only new data (O(n) complexity) by maintaining parsing state:
 
 ```ruby
 completer = JsonCompleter.new
 
 # Process first chunk
-result1 = completer.complete('{"users": [{"name": "')
-# => '{"users": [{"name": ""}]}'
+result1 = completer.parse('{"users": [{"name": "')
+# => {"users" => [{"name" => ""}]}
 
 # Process additional data
-result2 = completer.complete('{"users": [{"name": "Alice"}')
-# => '{"users": [{"name": "Alice"}]}'
+result2 = completer.parse('{"users": [{"name": "Alice"}')
+# => {"users" => [{"name" => "Alice"}]}
 
-# Final complete JSON
-result3 = completer.complete('{"users": [{"name": "Alice"}, {"name": "Bob"}]}')
-# => '{"users": [{"name": "Alice"}, {"name": "Bob"}]}'
+# Final parsed value
+result3 = completer.parse('{"users": [{"name": "Alice"}, {"name": "Bob"}]}')
+# => {"users" => [{"name" => "Alice"}, {"name" => "Bob"}]}
 ```
 
+### String Output with `.complete`
+
+Use `.complete` when you specifically need completed JSON text instead of parsed Ruby objects:
+
+```ruby
+JsonCompleter.complete('{"name": "John", "age":')
+# => '{"name": "John", "age": null}'
+
+JsonCompleter.complete('[1, 2, {"key": "value"')
+# => '[1, 2, {"key": "value"}]'
+```
+
+This is the second-tier option when another layer expects JSON text and you want `json_completer` to materialize the current partial state as valid JSON.
+
 #### Performance Characteristics
 
 - **Zero reprocessing**: Maintains parsing state to avoid reparsing previously processed data
@@ -73,9 +87,9 @@ result3 = completer.complete('{"users": [{"name": "Alice"}, {"name": "Bob"}]}')
 
 ### Common Use Cases
 
-- **High-performance streaming JSON**: Process large JSON responses efficiently as data arrives over network connections
-- **Truncated API responses**: Complete JSON that was cut off due to size limits
-- **Log parsing**: Handle incomplete JSON entries in log files
+- **LLM streaming output**: Parse partial JSON emitted token-by-token from providers such as OpenAI and Anthropic
+- **Incremental structured output parsing**: Keep a live Ruby object while more JSON arrives
+- **JSON text completion**: Produce valid JSON text snapshots for downstream consumers that require a string
 
 ## Contributing
 

data/lib/json_completer/completion_engine.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+class JsonCompleter
+  module CompletionEngine
+    def complete(partial_json)
+      input = partial_json
+
+      if @state.nil? || @state.input_length > input.length
+        @state = ParsingState.new
+      end
+
+      return input if input.empty?
+      return input if valid_json_primitive_or_document?(input)
+
+      if @state.input_length == input.length && !@state.output_tokens.empty?
+        return finalize_completion(@state.output_tokens.dup, @state.context_stack.dup, @state.incomplete_string_token)
+      end
+
+      output_tokens = @state.output_tokens.dup
+      context_stack = @state.context_stack.dup
+      index = @state.last_index
+      length = input.length
+      incomplete_string_token = @state.incomplete_string_token
+
+      if incomplete_string_token && output_tokens.last&.start_with?('"') && output_tokens.last.end_with?('"')
+        output_tokens.pop
+      end
+
+      while index < length
+        if incomplete_string_token && index == @state.last_index
+          index, status = Scanners.scan_string(input, index, incomplete_string_token)
+
+          break unless %i[terminated invalid_unicode].include?(status)
+
+          output_tokens << incomplete_string_token.buffer.string
+          incomplete_string_token = nil
+
+          next
+        end
+
+        char = input[index]
+        last_significant_char_in_output = get_last_significant_char(output_tokens)
+
+        case char
+        when '{'
+          ensure_comma_before_new_item(output_tokens, context_stack, last_significant_char_in_output)
+          ensure_colon_if_value_expected(output_tokens, context_stack, last_significant_char_in_output)
+          output_tokens << char
+          context_stack << '{'
+          index += 1
+        when '['
+          ensure_comma_before_new_item(output_tokens, context_stack, last_significant_char_in_output)
+          ensure_colon_if_value_expected(output_tokens, context_stack, last_significant_char_in_output)
+          output_tokens << char
+          context_stack << '['
+          index += 1
+        when '}'
+          remove_trailing_comma(output_tokens)
+          output_tokens << char
+          context_stack.pop if !context_stack.empty? && context_stack.last == '{'
+          index += 1
+        when ']'
+          output_tokens << char
+          context_stack.pop if !context_stack.empty? && context_stack.last == '['
+          index += 1
+        when '"'
+          ensure_comma_before_new_item(output_tokens, context_stack, last_significant_char_in_output)
+          ensure_colon_if_value_expected(output_tokens, context_stack, last_significant_char_in_output)
+
+          string_token = Scanners::CompletionStringToken.new
+          index, status = Scanners.scan_string(input, index + 1, string_token)
+
+          if %i[terminated invalid_unicode].include?(status)
+            output_tokens << string_token.buffer.string
+          else
+            incomplete_string_token = string_token
+          end
+        when ':'
+          remove_trailing_comma(output_tokens) if last_significant_char_in_output == ','
+          output_tokens << char
+          index += 1
+        when ','
+          remove_trailing_comma(output_tokens)
+          output_tokens << char
+          index += 1
+        when 't', 'f', 'n'
+          ensure_comma_before_new_item(output_tokens, context_stack, last_significant_char_in_output)
+          ensure_colon_if_value_expected(output_tokens, context_stack, last_significant_char_in_output)
+
+          keyword_val, consumed = Scanners.scan_keyword_literal(input, index, KEYWORD_MAP[char.downcase])
+          output_tokens << keyword_val
+          index += consumed
+        when '-', '0'..'9'
+          ensure_comma_before_new_item(output_tokens, context_stack, last_significant_char_in_output)
+          ensure_colon_if_value_expected(output_tokens, context_stack, last_significant_char_in_output)
+
+          num_str, consumed = Scanners.scan_number_literal(input, index)
+          output_tokens << num_str
+          index += consumed
+        when /\s/
+          output_tokens << char
+          index += 1
+        else
+          index += 1
+        end
+      end
+
+      @state = ParsingState.new(
+        output_tokens: output_tokens,
+        context_stack: context_stack,
+        last_index: index,
+        input_length: length,
+        incomplete_string_token: incomplete_string_token
+      )
+
+      finalize_completion(output_tokens.dup, context_stack.dup, incomplete_string_token)
+    end
+
+    private
+
+    def finalize_completion(output_tokens, context_stack, incomplete_string_token = nil)
+      output_tokens << incomplete_string_token.finalized_incomplete_value if incomplete_string_token
+
+      last_sig_char_final = get_last_significant_char(output_tokens)
+
+      unless context_stack.empty?
+        current_ctx = context_stack.last
+        if current_ctx == '{'
+          if last_sig_char_final == '"'
+            prev_sig_char = get_previous_significant_char(output_tokens)
+            output_tokens << ':' << 'null' if ['{', ','].include?(prev_sig_char)
+          elsif last_sig_char_final == ':'
+            output_tokens << 'null'
+          end
+        elsif current_ctx == '['
+          output_tokens << 'null' if last_sig_char_final == ','
+        end
+      end
+
+      until context_stack.empty?
+        opener = context_stack.pop
+        remove_trailing_comma(output_tokens)
+        output_tokens << (opener == '{' ? '}' : ']')
+      end
+
+      reassembled_json = output_tokens.join
+      return 'null' if reassembled_json.match?(/\A\s*[,:]\s*\z/)
+
+      reassembled_json
+    end
+
+    def get_last_significant_char(output_tokens)
+      (output_tokens.length - 1).downto(0) do |index|
+        stripped_token = output_tokens[index].strip
+        return stripped_token[-1] unless stripped_token.empty?
+      end
+
+      nil
+    end
+
+    def get_previous_significant_char(output_tokens)
+      significant_chars = []
+
+      (output_tokens.length - 1).downto(0) do |index|
+        stripped_token = output_tokens[index].strip
+        next if stripped_token.empty?
+
+        significant_chars << stripped_token[-1]
+        return significant_chars[1] if significant_chars.length >= 2
+      end
+
+      nil
+    end
+
+    def ensure_comma_before_new_item(output_tokens, context_stack, last_sig_char)
+      return if output_tokens.empty? || context_stack.empty? || last_sig_char.nil?
+      return if STRUCTURE_CHARS.include?(last_sig_char)
+      return unless context_stack.last == '[' || (context_stack.last == '{' && last_sig_char != ':')
+
+      output_tokens << ','
+    end
+
+    def ensure_colon_if_value_expected(output_tokens, context_stack, last_sig_char)
+      return if output_tokens.empty? || context_stack.empty? || last_sig_char.nil?
+      return unless context_stack.last == '{' && last_sig_char == '"'
+
+      output_tokens << ':'
+    end
+
+    def remove_trailing_comma(output_tokens)
+      last_token_idx = -1
+
+      (output_tokens.length - 1).downto(0) do |index|
+        next if output_tokens[index].strip.empty?
+
+        last_token_idx = index
+        break
+      end
+
+      return unless last_token_idx != -1 && output_tokens[last_token_idx].strip == ','
+
+      output_tokens.slice!(last_token_idx)
+
+      while last_token_idx.positive? && output_tokens[last_token_idx - 1].strip.empty?
+        output_tokens.slice!(last_token_idx - 1)
+        last_token_idx -= 1
+      end
+    end
+
+    def valid_json_primitive_or_document?(str)
+      return true if VALID_PRIMITIVES.include?(str)
+
+      if str.match?(/\A-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?\z/) &&
+         !str.end_with?('.') && !str.match?(/[eE][+-]?$/)
+        return true
+      end
+
+      str.match?(/\A"(?:[^"\\]|\\.)*"\z/)
+    end
+  end
+
+  include CompletionEngine
+end

data/lib/json_completer/parser_engine.rb

@@ -0,0 +1,345 @@
+# frozen_string_literal: true
+
+class JsonCompleter
+  module ParserEngine
+    def parse(partial_json)
+      input = partial_json
+
+      if @parse_state.nil? ||
+         @parse_state.input_length > input.length ||
+         (@parse_state.input_snapshot && !input.start_with?(@parse_state.input_snapshot))
+        @parse_state = self.class.new_parse_state
+      end
+
+      return nil if input.empty?
+
+      begin
+        if @parse_state.input_length == input.length
+          finalize_parse_result
+          return @parse_state.root
+        end
+
+        prepare_parse_state_for_incremental_input
+
+        index = @parse_state.last_index
+        while index < input.length
+          if @parse_state.token_state
+            index = continue_parse_token(input, index)
+            next
+          end
+
+          char = input[index]
+          if top_level_value_complete? && char !~ /\s/
+            raise ParseError, 'unexpected token after top-level value'
+          end
+
+          case char
+          when /\s/
+            index += 1
+          when '{'
+            start_parse_container({})
+            index += 1
+          when '['
+            start_parse_container([])
+            index += 1
+          when '}'
+            close_parse_object!
+            index += 1
+          when ']'
+            close_parse_array!
+            index += 1
+          when '"'
+            start_parse_string_token
+            index += 1
+          when ':'
+            parse_colon!
+            index += 1
+          when ','
+            parse_comma!
+            index += 1
+          when 't', 'f', 'n'
+            start_parse_keyword_token(char)
+            index += 1
+          when '-', '0'..'9'
+            start_parse_number_token(char)
+            index += 1
+          else
+            raise ParseError, "unexpected token #{char.inspect}"
+          end
+        end
+
+        @parse_state.last_index = index
+        @parse_state.input_length = input.length
+        @parse_state.input_snapshot = input
+        finalize_parse_result
+        @parse_state.root
+      rescue ParseError
+        @parse_state = self.class.new_parse_state
+        raise
+      end
+    end
+
+    private
+
+    def prepare_parse_state_for_incremental_input
+      token = @parse_state.token_state
+      return unless token.is_a?(Scanners::ParsedStringToken) && token.role == :key && token.visible_key
+
+      restore_visible_key_placeholder(token)
+    end
+
+    def continue_parse_token(input, index)
+      token = @parse_state.token_state
+
+      case token
+      when Scanners::ParsedStringToken
+        continue_parse_string_token(input, index)
+      when Scanners::NumberToken
+        continue_parse_number_token(input, index)
+      when Scanners::KeywordToken
+        continue_parse_keyword_token(input, index)
+      else
+        raise ParseError, "unsupported token state: #{token.class}"
+      end
+    end
+
+    def start_parse_container(container)
+      slot = parse_value_slot!
+      assign_parse_slot(slot, container)
+      transition_after_parse_value(slot)
+
+      @parse_state.context_stack << if container.is_a?(Hash)
+                                      ObjectContext.new(container: container)
+                                    else
+                                      ArrayContext.new(container: container)
+                                    end
+    end
+
+    def close_parse_object!
+      context = @parse_state.context_stack.last
+      raise ParseError, 'unexpected object close' unless context.is_a?(ObjectContext)
+      raise ParseError, 'cannot close object while a key is incomplete' if context.mode == :key_in_progress
+      raise ParseError, 'cannot close object before a colon' if context.mode == :after_key
+      raise ParseError, 'cannot close object while a value is missing' if context.mode == :value
+
+      @parse_state.context_stack.pop
+    end
+
+    def close_parse_array!
+      context = @parse_state.context_stack.last
+      raise ParseError, 'unexpected array close' unless context.is_a?(ArrayContext)
+      raise ParseError, 'cannot close array while a value is missing' if context.provisional_index
+
+      @parse_state.context_stack.pop
+    end
+
+    def start_parse_string_token
+      context = @parse_state.context_stack.last
+
+      if context.is_a?(ObjectContext) && context.mode == :key_or_end
+        context.mode = :key_in_progress
+        @parse_state.token_state = Scanners::ParsedStringToken.new(role: :key, context: context)
+        return
+      end
+
+      slot = parse_value_slot!
+      token = Scanners::ParsedStringToken.new(role: :value, slot: slot)
+      assign_parse_slot(slot, token.buffer)
+      transition_after_parse_value(slot)
+      @parse_state.token_state = token
+    end
+
+    def continue_parse_string_token(input, index)
+      token = @parse_state.token_state
+      index, status = Scanners.scan_string(input, index, token)
+      raise ParseError, 'invalid string escape sequence' if status == :invalid_escape
+      raise ParseError, 'invalid unicode escape sequence' if status == :invalid_unicode
+      raise ParseError, 'invalid control character in string literal' if status == :invalid_control_character
+
+      finish_parse_string_token! if status == :terminated
+      index
+    end
+
+    def finish_parse_string_token!
+      token = @parse_state.token_state
+      return unless token
+
+      if token.role == :key
+        token.context.current_key = token.buffer.dup
+        token.context.mode = :after_key
+      end
+
+      @parse_state.token_state = nil
+    end
+
+    def start_parse_number_token(first_char)
+      slot = parse_value_slot!
+      token = Scanners::NumberToken.new(slot: slot)
+      token.append(first_char)
+      assign_parse_slot(slot, token.parsed_value)
+      transition_after_parse_value(slot)
+      @parse_state.token_state = token
+    end
+
+    def continue_parse_number_token(input, index)
+      token = @parse_state.token_state
+
+      while index < input.length && token.append(input[index])
+        assign_parse_slot(token.slot, token.parsed_value)
+        index += 1
+      end
+
+      raise ParseError, 'invalid number literal' if token.invalid?
+
+      @parse_state.token_state = nil if index < input.length
+      index
+    end
+
+    def start_parse_keyword_token(first_char)
+      slot = parse_value_slot!
+      token = Scanners::KeywordToken.new(slot: slot, target: KEYWORD_MAP[first_char], matched: 1)
+      assign_parse_slot(slot, token.parsed_value)
+      transition_after_parse_value(slot)
+      @parse_state.token_state = token
+    end
+
+    def continue_parse_keyword_token(input, index)
+      token = @parse_state.token_state
+
+      while index < input.length && token.matched < token.target.length && token.append(input[index])
+        index += 1
+      end
+
+      raise ParseError, 'invalid keyword literal' if token.matched < token.target.length && index < input.length
+
+      @parse_state.token_state = nil if index < input.length || token.matched == token.target.length
+      index
+    end
+
+    def parse_colon!
+      context = @parse_state.context_stack.last
+      raise ParseError, 'unexpected colon' unless context.is_a?(ObjectContext) && context.mode == :after_key
+
+      context.mode = :value
+    end
+
+    def parse_comma!
+      context = @parse_state.context_stack.last
+      raise ParseError, 'unexpected comma' unless context
+
+      case context
+      when ArrayContext
+        raise ParseError, 'cannot add a comma while an array value is missing' unless context.mode == :after_value
+
+        context.mode = :value_or_end
+        context.provisional_index = context.container.length
+      when ObjectContext
+        raise ParseError, 'cannot add a comma while an object entry is incomplete' unless context.mode == :after_value
+
+        context.mode = :key_or_end
+        context.current_key = nil
+
+      end
+    end
+
+    def parse_value_slot!
+      context = @parse_state.context_stack.last
+
+      unless context
+        raise ParseError, 'unexpected token after top-level value' if @parse_state.root_assigned
+
+        return ParseSlot.new(root: true)
+      end
+
+      case context
+      when ArrayContext
+        raise ParseError, 'expected comma before next array value' if context.mode == :after_value
+        raise ParseError, 'cannot parse array value here' unless context.mode == :value_or_end
+
+        index = context.provisional_index || context.container.length
+        context.provisional_index = nil
+        ParseSlot.new(container: context.container, key: index, root: false)
+      when ObjectContext
+        raise ParseError, 'expected colon before object value' if context.mode == :after_key
+        raise ParseError, 'expected comma before next object entry' if context.mode == :after_value
+        raise ParseError, 'expected object key' unless context.mode == :value
+
+        ParseSlot.new(container: context.container, key: context.current_key, root: false)
+      end
+    end
+
+    def top_level_value_complete?
+      @parse_state.root_assigned &&
+        @parse_state.context_stack.empty? &&
+        @parse_state.token_state.nil?
+    end
+
+    def assign_parse_slot(slot, value)
+      if slot.root
+        @parse_state.root = value
+        @parse_state.root_assigned = true
+      else
+        slot.container[slot.key] = value
+      end
+    end
+
+    def transition_after_parse_value(slot)
+      context = @parse_state.context_stack.last
+
+      case context
+      when ArrayContext
+        context.mode = :after_value
+      when ObjectContext
+        context.mode = :after_value if slot.root || !context.current_key.nil?
+      end
+    end
+
+    def finalize_parse_result
+      token = @parse_state.token_state
+
+      if token.is_a?(Scanners::ParsedStringToken) && token.role == :key
+        update_visible_key_placeholder(token)
+        return
+      end
+
+      @parse_state.context_stack.each do |context|
+        case context
+        when ObjectContext
+          next unless %i[after_key value].include?(context.mode) && context.current_key
+
+          context.container[context.current_key] = nil
+        when ArrayContext
+          next unless context.provisional_index
+
+          context.container[context.provisional_index] = nil
+        end
+      end
+    end
+
+    def restore_visible_key_placeholder(token)
+      if token.visible_key_replaced_present
+        token.context.container[token.visible_key] = token.visible_key_replaced_value
+      else
+        token.context.container.delete(token.visible_key)
+      end
+
+      token.visible_key = nil
+      token.visible_key_replaced_value = nil
+      token.visible_key_replaced_present = false
+    end
+
+    def update_visible_key_placeholder(token)
+      current_key = token.buffer.dup
+      return if token.visible_key == current_key
+
+      restore_visible_key_placeholder(token) if token.visible_key
+
+      token.visible_key = current_key
+      token.visible_key_replaced_present = token.context.container.key?(current_key)
+      token.visible_key_replaced_value = token.context.container[current_key]
+      token.context.container[current_key] = nil
+    end
+  end
+
+  include ParserEngine
+end