llm-shell 0.10.0 → 0.11.0

data/lib/llm/shell/internal/coderay/lib/coderay/scanners/debug.rb

@@ -1,75 +0,0 @@
-require 'set'
-
-module CodeRay
-module Scanners
-  
-  # = Debug Scanner
-  # 
-  # Interprets the output of the Encoders::Debug encoder (basically the inverse function).
-  class Debug < Scanner
-    
-    register_for :debug
-    title 'CodeRay Token Dump Import'
-    
-  protected
-    
-    def setup
-      super
-      @known_token_kinds = TokenKinds.keys.map(&:to_s).to_set
-    end
-    
-    def scan_tokens encoder, options
-      
-      opened_tokens = []
-      
-      until eos?
-        
-        if match = scan(/\s+/)
-          encoder.text_token match, :space
-          
-        elsif match = scan(/ (\w+) \( ( [^\)\\]* ( \\. [^\)\\]* )* ) \)? /x)
-          if @known_token_kinds.include? self[1]
-            encoder.text_token self[2].gsub(/\\(.)/m, '\1'), self[1].to_sym
-          else
-            encoder.text_token matched, :unknown
-          end
-          
-        elsif match = scan(/ (\w+) ([<\[]) /x)
-          if @known_token_kinds.include? self[1]
-            kind = self[1].to_sym
-          else
-            kind = :unknown
-          end
-          
-          opened_tokens << kind
-          case self[2]
-          when '<'
-            encoder.begin_group kind
-          when '['
-            encoder.begin_line kind
-          else
-            raise 'CodeRay bug: This case should not be reached.'
-          end
-          
-        elsif !opened_tokens.empty? && match = scan(/ > /x)
-          encoder.end_group opened_tokens.pop
-          
-        elsif !opened_tokens.empty? && match = scan(/ \] /x)
-          encoder.end_line opened_tokens.pop
-          
-        else
-          encoder.text_token getch, :space
-          
-        end
-        
-      end
-      
-      encoder.end_group opened_tokens.pop until opened_tokens.empty?
-      
-      encoder
-    end
-    
-  end
-  
-end
-end

data/lib/llm/shell/internal/coderay/lib/coderay/scanners/delphi.rb

@@ -1,144 +0,0 @@
-module CodeRay
-module Scanners
-  
-  # Scanner for the Delphi language (Object Pascal).
-  # 
-  # Alias: +pascal+
-  class Delphi < Scanner
-    
-    register_for :delphi
-    file_extension 'pas'
-    
-    KEYWORDS = [
-      'and', 'array', 'as', 'at', 'asm', 'at', 'begin', 'case', 'class',
-      'const', 'constructor', 'destructor', 'dispinterface', 'div', 'do',
-      'downto', 'else', 'end', 'except', 'exports', 'file', 'finalization',
-      'finally', 'for', 'function', 'goto', 'if', 'implementation', 'in',
-      'inherited', 'initialization', 'inline', 'interface', 'is', 'label',
-      'library', 'mod', 'nil', 'not', 'object', 'of', 'or', 'out', 'packed',
-      'procedure', 'program', 'property', 'raise', 'record', 'repeat',
-      'resourcestring', 'set', 'shl', 'shr', 'string', 'then', 'threadvar',
-      'to', 'try', 'type', 'unit', 'until', 'uses', 'var', 'while', 'with',
-      'xor', 'on',
-    ]  # :nodoc:
-    
-    DIRECTIVES = [
-      'absolute', 'abstract', 'assembler', 'at', 'automated', 'cdecl',
-      'contains', 'deprecated', 'dispid', 'dynamic', 'export',
-      'external', 'far', 'forward', 'implements', 'local', 
-      'near', 'nodefault', 'on', 'overload', 'override',
-      'package', 'pascal', 'platform', 'private', 'protected', 'public',
-      'published', 'read', 'readonly', 'register', 'reintroduce',
-      'requires', 'resident', 'safecall', 'stdcall', 'stored', 'varargs',
-      'virtual', 'write', 'writeonly',
-    ]  # :nodoc:
-    
-    IDENT_KIND = WordList::CaseIgnoring.new(:ident).
-      add(KEYWORDS, :keyword).
-      add(DIRECTIVES, :directive)  # :nodoc:
-    
-    NAME_FOLLOWS = WordList::CaseIgnoring.new(false).
-      add(%w(procedure function .))  # :nodoc:
-    
-  protected
-    
-    def scan_tokens encoder, options
-      
-      state = :initial
-      last_token = ''
-      
-      until eos?
-        
-        if state == :initial
-          
-          if match = scan(/ \s+ /x)
-            encoder.text_token match, :space
-            next
-            
-          elsif match = scan(%r! \{ \$ [^}]* \}? | \(\* \$ (?: .*? \*\) | .* ) !mx)
-            encoder.text_token match, :preprocessor
-            next
-            
-          elsif match = scan(%r! // [^\n]* | \{ [^}]* \}? | \(\* (?: .*? \*\) | .* ) !mx)
-            encoder.text_token match, :comment
-            next
-            
-          elsif match = scan(/ <[>=]? | >=? | :=? | [-+=*\/;,@\^|\(\)\[\]] | \.\. /x)
-            encoder.text_token match, :operator
-          
-          elsif match = scan(/\./)
-            encoder.text_token match, :operator
-            next if last_token == 'end'
-            
-          elsif match = scan(/ [A-Za-z_][A-Za-z_0-9]* /x)
-            encoder.text_token match, NAME_FOLLOWS[last_token] ? :ident : IDENT_KIND[match]
-            
-          elsif match = skip(/ ' ( [^\n']|'' ) (?:'|$) /x)
-            encoder.begin_group :char
-            encoder.text_token "'", :delimiter
-            encoder.text_token self[1], :content
-            encoder.text_token "'", :delimiter
-            encoder.end_group :char
-            next
-            
-          elsif match = scan(/ ' /x)
-            encoder.begin_group :string
-            encoder.text_token match, :delimiter
-            state = :string
-            
-          elsif match = scan(/ \# (?: \d+ | \$[0-9A-Fa-f]+ ) /x)
-            encoder.text_token match, :char
-            
-          elsif match = scan(/ \$ [0-9A-Fa-f]+ /x)
-            encoder.text_token match, :hex
-            
-          elsif match = scan(/ (?: \d+ ) (?![eE]|\.[^.]) /x)
-            encoder.text_token match, :integer
-            
-          elsif match = scan(/ \d+ (?: \.\d+ (?: [eE][+-]? \d+ )? | [eE][+-]? \d+ ) /x)
-            encoder.text_token match, :float
-            
-          else
-            encoder.text_token getch, :error
-            next
-            
-          end
-          
-        elsif state == :string
-          if match = scan(/[^\n']+/)
-            encoder.text_token match, :content
-          elsif match = scan(/''/)
-            encoder.text_token match, :char
-          elsif match = scan(/'/)
-            encoder.text_token match, :delimiter
-            encoder.end_group :string
-            state = :initial
-            next
-          elsif match = scan(/\n/)
-            encoder.end_group :string
-            encoder.text_token match, :space
-            state = :initial
-          else
-            raise "else case \' reached; %p not handled." % peek(1), encoder
-          end
-          
-        else
-          raise 'else-case reached', encoder
-          
-        end
-        
-        last_token = match
-        
-      end
-      
-      if state == :string
-        encoder.end_group state
-      end
-      
-      encoder
-    end
-
-  end
-
-end
-end

data/lib/llm/shell/internal/coderay/lib/coderay/scanners/diff.rb

@@ -1,221 +0,0 @@
-module CodeRay
-module Scanners
-  
-  # Scanner for output of the diff command.
-  # 
-  # Alias: +patch+
-  class Diff < Scanner
-    
-    register_for :diff
-    title 'diff output'
-    
-    DEFAULT_OPTIONS = {
-      :highlight_code => true,
-      :inline_diff    => true,
-    }
-    
-  protected
-    
-    def scan_tokens encoder, options
-      
-      line_kind = nil
-      state = :initial
-      deleted_lines_count = 0
-      scanners = Hash.new do |h, lang|
-        h[lang] = Scanners[lang].new '', :keep_tokens => true, :keep_state => true
-      end
-      content_scanner = scanners[:plain]
-      content_scanner_entry_state = nil
-      
-      until eos?
-        
-        if match = scan(/\n/)
-          deleted_lines_count = 0 unless line_kind == :delete
-          if line_kind
-            encoder.end_line line_kind
-            line_kind = nil
-          end
-          encoder.text_token match, :space
-          next
-        end
-        
-        case state
-        
-        when :initial
-          if match = scan(/--- |\+\+\+ |=+|_+/)
-            encoder.begin_line line_kind = :head
-            encoder.text_token match, :head
-            if match = scan(/[^\x00\n]+?(?=$|[\t\n]|  \(revision)/)
-              encoder.text_token match, :filename
-              if options[:highlight_code] && match != '/dev/null'
-                file_type = CodeRay::FileType.fetch(match, :text)
-                file_type = :text if file_type == :diff
-                content_scanner = scanners[file_type]
-                content_scanner_entry_state = nil
-              end
-            end
-            next unless match = scan(/.+/)
-            encoder.text_token match, :plain
-          elsif match = scan(/Index: |Property changes on: /)
-            encoder.begin_line line_kind = :head
-            encoder.text_token match, :head
-            next unless match = scan(/.+/)
-            encoder.text_token match, :plain
-          elsif match = scan(/Added: /)
-            encoder.begin_line line_kind = :head
-            encoder.text_token match, :head
-            next unless match = scan(/.+/)
-            encoder.text_token match, :plain
-            state = :added
-          elsif match = scan(/\\ .*/)
-            encoder.text_token match, :comment
-          elsif match = scan(/@@(?>[^@\n]+)@@/)
-            content_scanner.state = :initial unless match?(/\n\+/)
-            content_scanner_entry_state = nil
-            if check(/\n|$/)
-              encoder.begin_line line_kind = :change
-            else
-              encoder.begin_group :change
-            end
-            encoder.text_token match[0,2], :change
-            encoder.text_token match[2...-2], :plain
-            encoder.text_token match[-2,2], :change
-            encoder.end_group :change unless line_kind
-            next unless match = scan(/.+/)
-            if options[:highlight_code]
-              content_scanner.tokenize match, :tokens => encoder
-            else
-              encoder.text_token match, :plain
-            end
-            next
-          elsif match = scan(/\+/)
-            encoder.begin_line line_kind = :insert
-            encoder.text_token match, :insert
-            next unless match = scan(/.+/)
-            if options[:highlight_code]
-              content_scanner.tokenize match, :tokens => encoder
-            else
-              encoder.text_token match, :plain
-            end
-            next
-          elsif match = scan(/-/)
-            deleted_lines_count += 1
-            if options[:inline_diff] && deleted_lines_count == 1 && (changed_lines_count = 1 + check(/.*(?:\n\-.*)*/).count("\n")) && changed_lines_count <= 100_000 && match?(/(?>.*(?:\n\-.*){#{changed_lines_count - 1}}(?:\n\+.*){#{changed_lines_count}})$(?!\n\+)/)
-              deleted_lines  = Array.new(changed_lines_count) { |i| skip(/\n\-/) if i > 0; scan(/.*/) }
-              inserted_lines = Array.new(changed_lines_count) { |i| skip(/\n\+/)         ; scan(/.*/) }
-              
-              deleted_lines_tokenized  = []
-              inserted_lines_tokenized = []
-              for deleted_line, inserted_line in deleted_lines.zip(inserted_lines)
-                pre, deleted_part, inserted_part, post = diff deleted_line, inserted_line
-                content_scanner_entry_state = content_scanner.state
-                deleted_lines_tokenized << content_scanner.tokenize([pre, deleted_part, post], :tokens => Tokens.new)
-                content_scanner.state = content_scanner_entry_state || :initial
-                inserted_lines_tokenized << content_scanner.tokenize([pre, inserted_part, post], :tokens => Tokens.new)
-              end
-              
-              for pre, deleted_part, post in deleted_lines_tokenized
-                encoder.begin_line :delete
-                encoder.text_token '-', :delete
-                encoder.tokens pre
-                unless deleted_part.empty?
-                  encoder.begin_group :eyecatcher
-                  encoder.tokens deleted_part
-                  encoder.end_group :eyecatcher
-                end
-                encoder.tokens post
-                encoder.end_line :delete
-                encoder.text_token "\n", :space
-              end
-              
-              for pre, inserted_part, post in inserted_lines_tokenized
-                encoder.begin_line :insert
-                encoder.text_token '+', :insert
-                encoder.tokens pre
-                unless inserted_part.empty?
-                  encoder.begin_group :eyecatcher
-                  encoder.tokens inserted_part
-                  encoder.end_group :eyecatcher
-                end
-                encoder.tokens post
-                changed_lines_count -= 1
-                if changed_lines_count > 0
-                  encoder.end_line :insert
-                  encoder.text_token "\n", :space
-                end
-              end
-              
-              line_kind = :insert
-              
-            elsif match = scan(/.*/)
-              encoder.begin_line line_kind = :delete
-              encoder.text_token '-', :delete
-              if options[:highlight_code]
-                if deleted_lines_count == 1
-                  content_scanner_entry_state = content_scanner.state
-                end
-                content_scanner.tokenize match, :tokens => encoder unless match.empty?
-                if !match?(/\n-/)
-                  if match?(/\n\+/)
-                    content_scanner.state = content_scanner_entry_state || :initial
-                  end
-                  content_scanner_entry_state = nil
-                end
-              else
-                encoder.text_token match, :plain
-              end
-            end
-            next
-          elsif match = scan(/ .*/)
-            if options[:highlight_code]
-              content_scanner.tokenize match, :tokens => encoder
-            else
-              encoder.text_token match, :plain
-            end
-            next
-          elsif match = scan(/.+/)
-            encoder.begin_line line_kind = :comment
-            encoder.text_token match, :plain
-          else
-            raise_inspect 'else case rached'
-          end
-        
-        when :added
-          if match = scan(/   \+/)
-            encoder.begin_line line_kind = :insert
-            encoder.text_token match, :insert
-            next unless match = scan(/.+/)
-            encoder.text_token match, :plain
-          else
-            state = :initial
-            next
-          end
-        end
-        
-      end
-      
-      encoder.end_line line_kind if line_kind
-      
-      encoder
-    end
-    
-  private
-    
-    def diff a, b
-      # i will be the index of the leftmost difference from the left.
-      i_max = [a.size, b.size].min
-      i = 0
-      i += 1 while i < i_max && a[i] == b[i]
-      # j_min will be the index of the leftmost difference from the right.
-      j_min = i - i_max
-      # j will be the index of the rightmost difference from the right which
-      # does not precede the leftmost one from the left.
-      j = -1
-      j -= 1 while j >= j_min && a[j] == b[j]
-      return a[0...i], a[i..j], b[i..j], (j < -1) ? a[j + 1..-1] : ''
-    end
-    
-  end
-  
-end
-end

data/lib/llm/shell/internal/coderay/lib/coderay/scanners/erb.rb

@@ -1,81 +0,0 @@
-module CodeRay
-module Scanners
-  
-  load :html
-  load :ruby
-  
-  # Scanner for HTML ERB templates.
-  class ERB < Scanner
-    
-    register_for :erb
-    title 'HTML ERB Template'
-    
-    KINDS_NOT_LOC = HTML::KINDS_NOT_LOC
-    
-    ERB_RUBY_BLOCK = /
-      (<%(?!%)[-=\#]?)
-      ((?>
-        [^\-%]*    # normal*
-        (?>        # special
-          (?: %(?!>) | -(?!%>) )
-          [^\-%]*  # normal*
-        )*
-      ))
-      ((?: -?%> )?)
-    /x  # :nodoc:
-    
-    START_OF_ERB = /
-      <%(?!%)
-    /x  # :nodoc:
-    
-  protected
-    
-    def setup
-      @ruby_scanner = CodeRay.scanner :ruby, :tokens => @tokens, :keep_tokens => true
-      @html_scanner = CodeRay.scanner :html, :tokens => @tokens, :keep_tokens => true, :keep_state => true
-    end
-    
-    def reset_instance
-      super
-      @html_scanner.reset
-    end
-    
-    def scan_tokens encoder, options
-      
-      until eos?
-        
-        if (match = scan_until(/(?=#{START_OF_ERB})/o) || scan_rest) and not match.empty?
-          @html_scanner.tokenize match, :tokens => encoder
-          
-        elsif match = scan(/#{ERB_RUBY_BLOCK}/o)
-          start_tag = self[1]
-          code = self[2]
-          end_tag = self[3]
-          
-          encoder.begin_group :inline
-          encoder.text_token start_tag, :inline_delimiter
-          
-          if start_tag == '<%#'
-            encoder.text_token code, :comment
-          else
-            @ruby_scanner.tokenize code, :tokens => encoder
-          end unless code.empty?
-          
-          encoder.text_token end_tag, :inline_delimiter unless end_tag.empty?
-          encoder.end_group :inline
-          
-        else
-          raise_inspect 'else-case reached!', encoder
-          
-        end
-        
-      end
-      
-      encoder
-      
-    end
-    
-  end
-  
-end
-end