llm-shell 0.9.2 → 0.10.0

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/handler.rb

@@ -0,0 +1,268 @@
+# frozen-string-literal: true
+
+module Tomlrb
+  class Handler
+    attr_reader :output, :symbolize_keys
+
+    def initialize(**options)
+      @output = {}
+      @current = @output
+      @stack = []
+      @array_names = []
+      @current_table = []
+      @keys = Keys.new
+      @symbolize_keys = options[:symbolize_keys]
+    end
+
+    def set_context(identifiers, is_array_of_tables: false)
+      if identifiers.empty?
+        raise ParseError, 'Array needs a name'
+      end
+
+      @current_table = identifiers.dup
+      @keys.add_table_key identifiers, is_array_of_tables
+      @current = @output
+
+      deal_with_array_of_tables(identifiers, is_array_of_tables) do |identifierz|
+        identifierz.each do |k|
+          k = k.to_sym if @symbolize_keys
+          if @current[k].is_a?(Array)
+            @current = @current[k].last
+          else
+            @current[k] ||= {}
+            @current = @current[k]
+          end
+        end
+      end
+    end
+
+    def deal_with_array_of_tables(identifiers, is_array_of_tables)
+      stringified_identifier = identifiers.join('.')
+
+      if is_array_of_tables
+        @array_names << stringified_identifier
+        last_identifier = identifiers.pop
+      elsif @array_names.include?(stringified_identifier)
+        raise ParseError, 'Cannot define a normal table with the same name as an already established array'
+      end
+
+      yield(identifiers)
+
+      if is_array_of_tables
+        last_identifier = last_identifier.to_sym if @symbolize_keys
+        @current[last_identifier] ||= []
+        raise ParseError, "Cannot use key #{last_identifier} for both table and array at once" unless @current[last_identifier].respond_to?(:<<)
+        @current[last_identifier] << {}
+        @current = @current[last_identifier].last
+      end
+    end
+
+    def assign(k)
+      @keys.add_pair_key k, @current_table
+      current = @current
+      while (key = k.shift)
+        key = key.to_sym if @symbolize_keys
+        current = assign_key_path(current, key, k.empty?)
+      end
+    end
+
+    def push(o)
+      @stack << o
+    end
+
+    def push_inline(inline_arrays)
+      merged_inline = {}
+
+      inline_arrays.each do |inline_array|
+        current = merged_inline
+        value = inline_array.pop
+        inline_array.each_with_index do |inline_key, inline_index|
+          inline_key = inline_key.to_sym if @symbolize_keys
+          last_key = inline_index == inline_array.size - 1
+
+          if last_key
+            if current[inline_key].nil?
+              current[inline_key] = value
+            else
+              raise Key::KeyConflict, "Inline key #{inline_key} is already used"
+            end
+          else
+            current[inline_key] ||= {}
+            current = current[inline_key]
+          end
+        end
+      end
+
+      push(merged_inline)
+    end
+
+    def start_(type)
+      push([type])
+    end
+
+    def end_(type)
+      array = []
+      while (value = @stack.pop) != [type]
+        raise ParseError, 'Unclosed table' if @stack.empty?
+
+        array.unshift(value)
+      end
+      array
+    end
+
+    def validate_value(value)
+      if value.nil?
+        raise ParseError, 'Value must be present'
+      end
+    end
+
+    private
+
+    def assign_key_path(current, key, key_emptied)
+      if key_emptied
+        raise ParseError, "Cannot overwrite value with key #{key}" unless current.is_a?(Hash)
+
+        current[key] = @stack.pop
+        return current
+      end
+      current[key] ||= {}
+      current[key]
+    end
+  end
+
+  class Keys
+    def initialize
+      @keys = {}
+    end
+
+    def add_table_key(keys, is_array_of_tables = false)
+      self << [keys, [], is_array_of_tables]
+    end
+
+    def add_pair_key(keys, context)
+      self << [context, keys, false]
+    end
+
+    def <<(keys)
+      table_keys, pair_keys, is_array_of_tables = keys
+      current = @keys
+      current = append_table_keys(current, table_keys, pair_keys.empty?, is_array_of_tables)
+      append_pair_keys(current, pair_keys, table_keys.empty?, is_array_of_tables)
+    end
+
+    private
+
+    def append_table_keys(current, table_keys, pair_keys_empty, is_array_of_tables)
+      table_keys.each_with_index do |key, index|
+        declared = (index == table_keys.length - 1) && pair_keys_empty
+        if index.zero?
+          current = find_or_create_first_table_key(current, key, declared, is_array_of_tables)
+        else
+          current <<= [key, :table, declared, is_array_of_tables]
+        end
+      end
+
+      current.clear_children if is_array_of_tables
+      current
+    end
+
+    def find_or_create_first_table_key(current, key, declared, is_array_of_tables)
+      existed = current[key]
+      if existed && existed.type == :pair
+        raise Key::KeyConflict, "Key #{key} is already used as #{existed.type} key"
+      end
+      if existed && existed.declared? && declared && ! is_array_of_tables
+        raise Key::KeyConflict, "Key #{key} is already used"
+      end
+      k = existed || Key.new(key, :table, declared)
+      current[key] = k
+      k
+    end
+
+    def append_pair_keys(current, pair_keys, table_keys_empty, is_array_of_tables)
+      pair_keys.each_with_index do |key, index|
+        declared = index == pair_keys.length - 1
+        if index.zero? && table_keys_empty
+          current = find_or_create_first_pair_key(current, key, declared, table_keys_empty)
+        else
+          key = current << [key, :pair, declared, is_array_of_tables]
+          current = key
+        end
+      end
+    end
+
+    def find_or_create_first_pair_key(current, key, declared, table_keys_empty)
+      existed = current[key]
+      if existed && (existed.type == :pair) && declared && table_keys_empty
+        raise Key::KeyConflict, "Key #{key} is already used"
+      end
+      k = Key.new(key, :pair, declared)
+      current[key] = k
+      k
+    end
+  end
+
+  class Key
+    class KeyConflict < ParseError; end
+
+    attr_reader :key, :type
+
+    def initialize(key, type, declared = false)
+      @key = key
+      @type = type
+      @declared = declared
+      @children = {}
+    end
+
+    def declared?
+      @declared
+    end
+
+    def <<(key_type_declared)
+      key, type, declared, is_array_of_tables = key_type_declared
+      existed = @children[key]
+      validate_already_declared_as_different_key(type, declared, existed)
+      validate_already_declared_as_non_array_table(type, is_array_of_tables, declared, existed)
+      validate_path_already_created_as_different_type(type, declared, existed)
+      validate_path_already_declared_as_different_type(type, declared, existed)
+      validate_already_declared_as_same_key(declared, existed)
+      @children[key] = existed || self.class.new(key, type, declared)
+    end
+
+    def clear_children
+      @children.clear
+    end
+
+    private
+
+    def validate_already_declared_as_different_key(type, _declared, existed)
+      if existed && existed.declared? && existed.type != type
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
+
+    def validate_already_declared_as_non_array_table(type, is_array_of_tables, declared, existed)
+      if declared && type == :table && existed && existed.declared? && !is_array_of_tables
+        raise KeyConflict, "Key #{existed.key} is already used"
+      end
+    end
+
+    def validate_path_already_created_as_different_type(type, declared, existed)
+      if declared && (type == :table) && existed && (existed.type == :pair) && !existed.declared?
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
+
+    def validate_path_already_declared_as_different_type(type, declared, existed)
+      if !declared && (type == :pair) && existed && (existed.type == :pair) && existed.declared?
+        raise KeyConflict, "Key #{key} is already used as #{type} key"
+      end
+    end
+
+    def validate_already_declared_as_same_key(declared, existed)
+      if existed && !existed.declared? && declared
+        raise KeyConflict, "Key #{existed.key} is already used as #{existed.type} key"
+      end
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/local_date.rb

@@ -0,0 +1,35 @@
+# frozen-string-literal: true
+
+require 'forwardable'
+
+module Tomlrb
+  class LocalDate
+    extend Forwardable
+
+    def_delegators :@time, :year, :month, :day
+
+    def initialize(year, month, day)
+      @time = Time.new(year, month, day, 0, 0, 0, '-00:00')
+    end
+
+    # @param offset see {LocalDateTime#to_time}
+    # @return [Time] 00:00:00 of the date
+    def to_time(offset = '-00:00')
+      return @time if offset == '-00:00'
+      Time.new(year, month, day, 0, 0, 0, offset)
+    end
+
+    def to_s
+      @time.strftime('%F')
+    end
+
+    def ==(other)
+      other.is_a?(self.class) &&
+        to_time == other.to_time
+    end
+
+    def inspect
+      "#<#{self.class}: #{self}>"
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/local_date_time.rb

@@ -0,0 +1,42 @@
+# frozen-string-literal: true
+
+require 'forwardable'
+
+module Tomlrb
+  class LocalDateTime
+    extend Forwardable
+
+    def_delegators :@time, :year, :month, :day, :hour, :min, :sec, :usec, :nsec
+
+    def initialize(year, month, day, hour, min, sec) # rubocop:disable Metrics/ParameterLists
+      @time = Time.new(year, month, day, hour, min, sec, '-00:00')
+      @sec = sec
+    end
+
+    # @param offset [String, Symbol, Numeric, nil] time zone offset.
+    #   * when +String+, must be '+HH:MM' format, '-HH:MM' format, 'UTC', 'A'..'I' or 'K'..'Z'. Arguments excluding '+-HH:MM' are supporeted at Ruby >= 2.7.0
+    #   * when +Symbol+, must be +:dst+(for summar time for local) or +:std+(for standard time).
+    #   * when +Numeric+, it is time zone offset in second.
+    #   * when +nil+, local time zone offset is used.
+    # @return [Time]
+    def to_time(offset = '-00:00')
+      return @time if offset == '-00:00'
+      Time.new(year, month, day, hour, min, @sec, offset)
+    end
+
+    def to_s
+      frac = (@sec - sec)
+      frac_str = frac.zero? ? '' : frac.to_s[1..-1]
+      @time.strftime('%FT%T') << frac_str
+    end
+
+    def ==(other)
+      other.is_a?(self.class) &&
+        to_time == other.to_time
+    end
+
+    def inspect
+      "#<#{self.class}: #{self}>"
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/local_time.rb

@@ -0,0 +1,40 @@
+# frozen-string-literal: true
+
+require 'forwardable'
+
+module Tomlrb
+  class LocalTime
+    extend Forwardable
+
+    def_delegators :@time, :hour, :min, :sec, :usec, :nsec
+
+    def initialize(hour, min, sec)
+      @time = Time.new(0, 1, 1, hour, min, sec, '-00:00')
+      @sec = sec
+    end
+
+    # @param year [Integer]
+    # @param month [Integer]
+    # @param day [Integer]
+    # @param offset see {LocalDateTime#to_time}
+    # @return [Time] the time of the date specified by params
+    def to_time(year, month, day, offset = '-00:00')
+      Time.new(year, month, day, hour, min, @sec, offset)
+    end
+
+    def to_s
+      frac = (@sec - sec)
+      frac_str = frac.zero? ? '' : frac.to_s[1..-1]
+      @time.strftime('%T') << frac_str
+    end
+
+    def ==(other)
+      other.is_a?(self.class) &&
+        @time == other.to_time(0, 1, 1)
+    end
+
+    def inspect
+      "#<#{self.class}: #{self}>"
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/parser.rb

@@ -0,0 +1,21 @@
+# frozen-string-literal: true
+
+require 'tomlrb/generated_parser'
+
+class Tomlrb::Parser < Tomlrb::GeneratedParser
+
+  def initialize(tokenizer, **options)
+    @tokenizer = tokenizer
+    @handler   = Tomlrb::Handler.new(**options)
+    super()
+  end
+
+  def next_token
+    @tokenizer.next_token
+  end
+
+  def parse
+    do_parse
+    @handler
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/scanner.rb

@@ -0,0 +1,92 @@
+# frozen-string-literal: true
+
+require 'strscan'
+
+module Tomlrb
+  class Scanner
+    COMMENT =
+      /#[^\u0000-\u0008\u000A-\u001F\u007F]*/.freeze
+    IDENTIFIER =
+      /[A-Za-z0-9_-]+/.freeze
+    SPACE =
+      /[ \t]/.freeze
+    NEWLINE =
+      /(?:[ \t]*(?:\r?\n)[ \t]*)+/.freeze
+    STRING_BASIC =
+      /(")(?:\\?[^\u0000-\u0008\u000A-\u001F\u007F])*?\1/.freeze
+    STRING_MULTI =
+      /"{3}([^\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]*?(?<!\\)"{3,5})/m.freeze
+    STRING_LITERAL =
+      /(')(?:\\?[^\u0000-\u0008\u000A-\u001F\u007F])*?\1/.freeze
+    STRING_LITERAL_MULTI =
+      /'{3}([^\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]*?'{3,5})/m.freeze
+    DATETIME =
+      /(-?\d{4})-(\d{2})-(\d{2})(?:(?:t|\s)(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?))?(z|[-+]\d{2}:\d{2})?/i.freeze
+    LOCAL_TIME =
+      /(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?)/.freeze
+    FLOAT =
+      /[+-]?(?:(?:\d|[1-9](?:_?\d)*)\.\d(?:_?\d)*|\d+(?=[eE]))(?:[eE][+-]?[0-9]+(_[0-9])*[0-9]*)?(?!\w)/.freeze
+    FLOAT_KEYWORD =
+      /[+-]?(?:inf|nan)\b/.freeze
+    INTEGER =
+      /[+-]?([1-9](_?\d)*|0)(?![A-Za-z0-9_-]+)/.freeze
+    NON_DEC_INTEGER =
+      /0(?:x[0-9A-Fa-f](_?[0-9A-Fa-f])*|o[0-7](_?[0-7])*|b[01](_?[01])*)/.freeze
+    BOOLEAN =
+      /true|false/.freeze
+    SPACED_ARRAY_OF_TABLES_START =
+      /^\[[ \t]+\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\]\]$/.freeze
+    SPACED_ARRAY_OF_TABLES_END =
+      /^\[\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\][ \t]+\]$/.freeze
+    SPACED_ARRAY_OF_TABLES_BOTH =
+      /^\[[ \t]+\[(#{IDENTIFIER}|#{STRING_BASIC}|#{STRING_LITERAL}|#{INTEGER}|#{NON_DEC_INTEGER}|#{FLOAT_KEYWORD}|#{BOOLEAN})\][ \t]+\]$/.freeze
+
+    def initialize(io)
+      @ss = StringScanner.new(io.read)
+      @eos = false
+    end
+
+    def next_token
+      case
+      when @ss.scan(NEWLINE) then [:NEWLINE, nil]
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_START) then raise ParseError.new('Array of tables has spaces in starting brackets')
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_END) then raise ParseError.new('Array of tables has spaces in ending brackets')
+      when @ss.scan(SPACED_ARRAY_OF_TABLES_BOTH) then raise ParseError.new('Array of tables has spaces in starting and ending brackets')
+      when @ss.scan(SPACE) then next_token
+      when @ss.scan(COMMENT) then next_token
+      when @ss.scan(DATETIME) then process_datetime
+      when @ss.scan(LOCAL_TIME) then process_local_time
+      when text = @ss.scan(STRING_MULTI) then [:STRING_MULTI, text[3..-4]]
+      when text = @ss.scan(STRING_BASIC) then [:STRING_BASIC, text[1..-2]]
+      when text = @ss.scan(STRING_LITERAL_MULTI) then [:STRING_LITERAL_MULTI, text[3..-4]]
+      when text = @ss.scan(STRING_LITERAL) then [:STRING_LITERAL, text[1..-2]]
+      when text = @ss.scan(FLOAT) then [:FLOAT, text]
+      when text = @ss.scan(FLOAT_KEYWORD) then [:FLOAT_KEYWORD, text]
+      when text = @ss.scan(INTEGER) then [:INTEGER, text]
+      when text = @ss.scan(NON_DEC_INTEGER) then [:NON_DEC_INTEGER, text]
+      when text = @ss.scan(BOOLEAN) then [:BOOLEAN, text]
+      when text = @ss.scan(IDENTIFIER) then [:IDENTIFIER, text]
+      when @ss.eos? then process_eos
+      else x = @ss.getch; [x, x]
+      end
+    end
+
+    def process_datetime
+      offset = @ss[7].gsub(/[zZ]/, '+00:00') if @ss[7]
+      args = [@ss[1], @ss[2], @ss[3], @ss[4], @ss[5], @ss[6], offset]
+      [:DATETIME, args]
+    end
+
+    def process_local_time
+      args = [@ss[1], @ss[2], @ss[3].to_f]
+      [:LOCAL_TIME, args]
+    end
+
+    def process_eos
+      return if @eos
+
+      @eos = true
+      [:EOS, nil]
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/string_utils.rb

@@ -0,0 +1,40 @@
+# frozen-string-literal: true
+
+module Tomlrb
+  class StringUtils
+    SPECIAL_CHARS = {
+      '\\t'  => "\t",
+      '\\b'  => "\b",
+      '\\f'  => "\f",
+      '\\n'  => "\n",
+      '\\r'  => "\r",
+      '\\"'  => '"',
+      '\\\\' => '\\'
+    }.freeze
+
+    def self.multiline_replacements(str)
+      strip_spaces(str).gsub(/\\+\s*\n\s*/) do |matched|
+        if matched.match(/\\+/)[0].length.odd?
+          matched.gsub(/\\\s*\n\s*/, '')
+        else
+          matched
+        end
+      end
+    end
+
+    def self.replace_escaped_chars(str)
+      str.gsub(/\\(u[\da-fA-F]{4}|U[\da-fA-F]{8}|.)/) do |m|
+        if m.size == 2
+          SPECIAL_CHARS[m] || (raise Tomlrb::ParseError.new "Escape sequence #{m} is reserved")
+        else
+          m[2..-1].to_i(16).chr(Encoding::UTF_8)
+        end
+      end
+    end
+
+    def self.strip_spaces(str)
+      str[0] = '' if str[0] == "\n"
+      str
+    end
+  end
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb/version.rb

@@ -0,0 +1,5 @@
+# frozen-string-literal: true
+
+module Tomlrb
+  VERSION = '2.0.3'
+end

data/lib/llm/shell/internal/tomlrb/lib/tomlrb.rb

@@ -0,0 +1,49 @@
+# frozen-string-literal: true
+
+require 'time'
+require 'stringio'
+require 'tomlrb/version'
+require 'tomlrb/local_date_time'
+require 'tomlrb/local_date'
+require 'tomlrb/local_time'
+require 'tomlrb/string_utils'
+require 'tomlrb/scanner'
+require 'tomlrb/parser'
+require 'tomlrb/handler'
+
+module Tomlrb
+  class ParseError < StandardError; end
+
+  # Parses a valid TOML string into its Ruby data structure
+  #
+  # @param string_or_io [String, StringIO] the content
+  # @param options [Hash] the options hash
+  # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings
+  # @return [Hash] the Ruby data structure represented by the input
+  def self.parse(string_or_io, **options)
+    io = string_or_io.is_a?(String) ? StringIO.new(string_or_io) : string_or_io
+    scanner = Scanner.new(io)
+    parser = Parser.new(scanner, **options)
+    begin
+      handler = parser.parse
+    rescue Racc::ParseError => e
+      raise ParseError, e.message
+    end
+
+    handler.output
+  end
+
+  # Reads a file content and parses it into its Ruby data structure
+  #
+  # @param path [String] the path to the file
+  # @param options [Hash] the options hash
+  # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings
+  # @return [Hash] the Ruby data structure represented by the input
+  def self.load_file(path, **options)
+    # By default Ruby sets the external encoding of an IO object to the
+    # default external encoding. The default external encoding is set by
+    # locale encoding or the interpreter -E option.
+    tmp = File.read(path, encoding: 'utf-8')
+    Tomlrb.parse(tmp, **options)
+  end
+end

data/lib/llm/shell/options.rb

@@ -33,7 +33,7 @@ class LLM::Shell
 
     def custom_prompt
       prompt = @options.delete(:prompt)
-      File.join(LLM::Shell.home, "prompts", prompt)
+      File.join(LLM::Shell.home_data, "prompts", prompt)
     end
   end
 end

data/lib/llm/shell/renderer.rb

@@ -3,7 +3,6 @@
 class LLM::Shell
   class Renderer
     RenderError = Class.new(RuntimeError)
-    include Command::Utils
 
     ##
     # @param [LLM::Message] message
@@ -40,8 +39,8 @@ class LLM::Shell
         body = "Tool call(s) return"
       elsif Array === content
         body = content.map { |c| render_message(c, color) }.join("\n")
-      elsif content =~ file_pattern
-        path = content.match(file_pattern) ? Regexp.last_match[1] : nil
+      elsif content =~ LLM::Shell.pattern
+        path = content.match(LLM::Shell.pattern) ? Regexp.last_match[1] : nil
         body = "<file path=#{path} />"
       else
         body = Markdown.render(content)