llm-shell 0.9.2 → 0.10.0

data/lib/llm/shell/internal/coderay/lib/coderay/scanners/yaml.rb

@@ -0,0 +1,140 @@
+module CodeRay
+module Scanners
+  
+  # Scanner for YAML.
+  #
+  # Based on the YAML scanner from Syntax by Jamis Buck.
+  class YAML < Scanner
+    
+    register_for :yaml
+    file_extension 'yml'
+    
+    KINDS_NOT_LOC = :all
+    
+  protected
+    
+    def scan_tokens encoder, options
+      
+      state = :initial
+      key_indent = string_indent = 0
+      
+      until eos?
+        
+        key_indent = nil if bol?
+        
+        if match = scan(/ +[\t ]*/)
+          encoder.text_token match, :space
+          
+        elsif match = scan(/\n+/)
+          encoder.text_token match, :space
+          state = :initial if match.index(?\n)
+          
+        elsif match = scan(/#.*/)
+          encoder.text_token match, :comment
+          
+        elsif bol? and case
+          when match = scan(/---|\.\.\./)
+            encoder.begin_group :head
+            encoder.text_token match, :head
+            encoder.end_group :head
+            next
+          when match = scan(/%.*/)
+            encoder.text_token match, :doctype
+            next
+          end
+        
+        elsif state == :value and case
+          when !check(/(?:"[^"]*")(?=: |:$)/) && match = scan(/"/)
+            encoder.begin_group :string
+            encoder.text_token match, :delimiter
+            encoder.text_token match, :content if (match = scan(/ [^"\\]* (?: \\. [^"\\]* )* /mx)) && !match.empty?
+            encoder.text_token match, :delimiter if match = scan(/"/)
+            encoder.end_group :string
+            next
+          when match = scan(/[|>][-+]?/)
+            encoder.begin_group :string
+            encoder.text_token match, :delimiter
+            string_indent = key_indent || column(pos - match.size) - 1
+            encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+            encoder.end_group :string
+            next
+          when match = scan(/(?![!"*&]).+?(?=$|\s+#)/)
+            encoder.begin_group :string
+            encoder.text_token match, :content
+            string_indent = key_indent || column(pos - match.size) - 1
+            encoder.text_token matched, :content if scan(/(?:\n+ {#{string_indent + 1}}.*)+/)
+            encoder.end_group :string
+            next
+          end
+          
+        elsif case
+          when match = scan(/[-:](?= |$)/)
+            state = :value if state == :colon && (match == ':' || match == '-')
+            state = :value if state == :initial && match == '-'
+            encoder.text_token match, :operator
+            next
+          when match = scan(/[,{}\[\]]/)
+            encoder.text_token match, :operator
+            next
+          when state == :initial && match = scan(/[-\w.()\/ ]*\S(?= *:(?: |$))/)
+            encoder.text_token match, :key
+            key_indent = column(pos - match.size) - 1
+            state = :colon
+            next
+          when match = scan(/(?:"[^"\n]*"|'[^'\n]*')(?= *:(?: |$))/)
+            encoder.begin_group :key
+            encoder.text_token match[0,1], :delimiter
+            encoder.text_token match[1..-2], :content if match.size > 2
+            encoder.text_token match[-1,1], :delimiter
+            encoder.end_group :key
+            key_indent = column(pos - match.size) - 1
+            state = :colon
+            next
+          when match = scan(/(![\w\/]+)(:([\w:]+))?/)
+            encoder.text_token self[1], :type
+            if self[2]
+              encoder.text_token ':', :operator
+              encoder.text_token self[3], :class
+            end
+            next
+          when match = scan(/&\S+/)
+            encoder.text_token match, :variable
+            next
+          when match = scan(/\*\w+/)
+            encoder.text_token match, :global_variable
+            next
+          when match = scan(/<</)
+            encoder.text_token match, :class_variable
+            next
+          when match = scan(/\d\d:\d\d:\d\d/)
+            encoder.text_token match, :octal
+            next
+          when match = scan(/\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d(\.\d+)? [-+]\d\d:\d\d/)
+            encoder.text_token match, :octal
+            next
+          when match = scan(/:\w+/)
+            encoder.text_token match, :symbol
+            next
+          when match = scan(/[^:\s]+(:(?! |$)[^:\s]*)* .*/)
+            encoder.text_token match, :error
+            next
+          when match = scan(/[^:\s]+(:(?! |$)[^:\s]*)*/)
+            encoder.text_token match, :error
+            next
+          end
+          
+        else
+          raise if eos?
+          encoder.text_token getch, :error
+          
+        end
+        
+      end
+      
+      encoder
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/scanners.rb

@@ -0,0 +1,27 @@
+require 'strscan'
+
+module CodeRay
+  
+  autoload :WordList, coderay_path('helpers', 'word_list')
+  
+  # = Scanners
+  #
+  # This module holds the Scanner class and its subclasses.
+  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
+  # can be found in coderay/scanners/ruby.
+  #
+  # Scanner also provides methods and constants for the register
+  # mechanism and the [] method that returns the Scanner class
+  # belonging to the given lang.
+  #
+  # See PluginHost.
+  module Scanners
+    
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'scanners'
+    
+    autoload :Scanner, CodeRay.coderay_path('scanners', 'scanner')
+    
+  end
+  
+end

data/lib/llm/shell/internal/coderay/lib/coderay/styles/_map.rb

@@ -0,0 +1,7 @@
+module CodeRay
+module Styles
+  
+  default :alpha
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/styles/alpha.rb

@@ -0,0 +1,153 @@
+module CodeRay
+module Styles
+  
+  # A colorful theme using CSS 3 colors (with alpha channel).
+  class Alpha < Style
+    
+    register_for :alpha
+    
+    code_background = 'hsl(0,0%,95%)'
+    numbers_background = 'hsl(180,65%,90%)'
+    border_color = 'silver'
+    normal_color = 'black'
+    
+    CSS_MAIN_STYLES = <<-MAIN  # :nodoc:
+.CodeRay {
+  background-color: #{code_background};
+  border: 1px solid #{border_color};
+  color: #{normal_color};
+}
+.CodeRay pre {
+  margin: 0px;
+}
+
+span.CodeRay { white-space: pre; border: 0px; padding: 2px; }
+
+table.CodeRay { border-collapse: collapse; width: 100%; padding: 2px; }
+table.CodeRay td { padding: 2px 4px; vertical-align: top; }
+
+.CodeRay .line-numbers {
+  background-color: #{numbers_background};
+  color: gray;
+  text-align: right;
+  -webkit-user-select: none;
+  -moz-user-select: none;
+  user-select: none;
+}
+.CodeRay .line-numbers a {
+  background-color: #{numbers_background} !important;
+  color: gray !important;
+  text-decoration: none !important;
+}
+.CodeRay .line-numbers pre {
+  word-break: normal;
+}
+.CodeRay .line-numbers a:target { color: blue !important; }
+.CodeRay .line-numbers .highlighted { color: red !important; }
+.CodeRay .line-numbers .highlighted a { color: red !important; }
+.CodeRay span.line-numbers { padding: 0px 4px; }
+.CodeRay .line { display: block; float: left; width: 100%; }
+.CodeRay .code { width: 100%; }
+    MAIN
+
+    TOKEN_COLORS = <<-'TOKENS'
+.debug { color: white !important; background: blue !important; }
+
+.annotation { color:#007 }
+.attribute-name { color:#b48 }
+.attribute-value { color:#700 }
+.binary { color:#549 }
+.binary .char { color:#325 }
+.binary .delimiter { color:#325 }
+.char { color:#D20 }
+.char .content { color:#D20 }
+.char .delimiter { color:#710 }
+.class { color:#B06; font-weight:bold }
+.class-variable { color:#369 }
+.color { color:#0A0 }
+.comment { color:#777 }
+.comment .char { color:#444 }
+.comment .delimiter { color:#444 }
+.constant { color:#036; font-weight:bold }
+.decorator { color:#B0B }
+.definition { color:#099; font-weight:bold }
+.delimiter { color:black }
+.directive { color:#088; font-weight:bold }
+.docstring { color:#D42; }
+.doctype { color:#34b }
+.done { text-decoration: line-through; color: gray }
+.entity { color:#800; font-weight:bold }
+.error { color:#F00; background-color:#FAA }
+.escape  { color:#666 }
+.exception { color:#C00; font-weight:bold }
+.float { color:#60E }
+.function { color:#06B; font-weight:bold }
+.function .delimiter { color:#059 }
+.function .content { color:#037 }
+.global-variable { color:#d70 }
+.hex { color:#02b }
+.id  { color:#33D; font-weight:bold }
+.include { color:#B44; font-weight:bold }
+.inline { background-color: hsla(0,0%,0%,0.07); color: black }
+.inline-delimiter { font-weight: bold; color: #666 }
+.instance-variable { color:#33B }
+.integer  { color:#00D }
+.imaginary { color:#f00 }
+.important { color:#D00 }
+.key { color: #606 }
+.key .char { color: #60f }
+.key .delimiter { color: #404 }
+.keyword { color:#080; font-weight:bold }
+.label { color:#970; font-weight:bold }
+.local-variable { color:#950 }
+.map .content { color:#808 }
+.map .delimiter { color:#40A}
+.map { background-color:hsla(200,100%,50%,0.06); }
+.namespace { color:#707; font-weight:bold }
+.octal { color:#40E }
+.operator { }
+.predefined { color:#369; font-weight:bold }
+.predefined-constant { color:#069 }
+.predefined-type { color:#0a8; font-weight:bold }
+.preprocessor { color:#579 }
+.pseudo-class { color:#00C; font-weight:bold }
+.regexp { background-color:hsla(300,100%,50%,0.06); }
+.regexp .content { color:#808 }
+.regexp .delimiter { color:#404 }
+.regexp .modifier { color:#C2C }
+.reserved { color:#080; font-weight:bold }
+.shell { background-color:hsla(120,100%,50%,0.06); }
+.shell .content { color:#2B2 }
+.shell .delimiter { color:#161 }
+.string { background-color:hsla(0,100%,50%,0.05); }
+.string .char { color: #b0b }
+.string .content { color: #D20 }
+.string .delimiter { color: #710 }
+.string .modifier { color: #E40 }
+.symbol { color:#A60 }
+.symbol .content { color:#A60 }
+.symbol .delimiter { color:#740 }
+.tag { color:#070; font-weight:bold }
+.type { color:#339; font-weight:bold }
+.value { color: #088 }
+.variable { color:#037 }
+
+.insert { background: hsla(120,100%,50%,0.12) }
+.delete { background: hsla(0,100%,50%,0.12) }
+.change { color: #bbf; background: #007 }
+.head { color: #f8f; background: #505 }
+.head .filename { color: white; }
+
+.delete .eyecatcher { background-color: hsla(0,100%,50%,0.2); border: 1px solid hsla(0,100%,45%,0.5); margin: -1px; border-bottom: none; border-top-left-radius: 5px; border-top-right-radius: 5px; }
+.insert .eyecatcher { background-color: hsla(120,100%,50%,0.2); border: 1px solid hsla(120,100%,25%,0.5); margin: -1px; border-top: none; border-bottom-left-radius: 5px; border-bottom-right-radius: 5px; }
+
+.insert .insert { color: #0c0; background:transparent; font-weight:bold }
+.delete .delete { color: #c00; background:transparent; font-weight:bold }
+.change .change { color: #88f }
+.head .head { color: #f4f }
+    TOKENS
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/styles/style.rb

@@ -0,0 +1,18 @@
+module CodeRay
+  
+  module Styles
+    
+    # Base class for styles.
+    # 
+    # Styles are used by Encoders::HTML to colorize tokens.
+    class Style
+      extend Plugin
+      plugin_host Styles
+      
+      DEFAULT_OPTIONS = { }  # :nodoc:
+      
+    end
+    
+  end
+  
+end

data/lib/llm/shell/internal/coderay/lib/coderay/styles.rb

@@ -0,0 +1,15 @@
+module CodeRay
+
+  # This module holds the Style class and its subclasses.
+  # 
+  # See Plugin.
+  module Styles
+    
+    extend PluginHost
+    plugin_path File.dirname(__FILE__), 'styles'
+    
+    autoload :Style, CodeRay.coderay_path('styles', 'style')
+    
+  end
+  
+end

data/lib/llm/shell/internal/coderay/lib/coderay/token_kinds.rb

@@ -0,0 +1,85 @@
+module CodeRay
+  
+  # A Hash of all known token kinds and their associated CSS classes.
+  TokenKinds = Hash.new(false)
+  
+  # speedup
+  TokenKinds.compare_by_identity if TokenKinds.respond_to? :compare_by_identity
+  
+  TokenKinds.update(  # :nodoc:
+    :debug               => 'debug',              # highlight for debugging (white on blue background)
+    
+    :annotation          => 'annotation',         # Groovy, Java
+    :attribute_name      => 'attribute-name',     # HTML, CSS
+    :attribute_value     => 'attribute-value',    # HTML
+    :binary              => 'binary',             # Python, Ruby
+    :char                => 'char',               # most scanners, also inside of strings
+    :class               => 'class',              # lots of scanners, for different purposes also in CSS
+    :class_variable      => 'class-variable',     # Ruby, YAML
+    :color               => 'color',              # CSS
+    :comment             => 'comment',            # most scanners
+    :constant            => 'constant',           # PHP, Ruby
+    :content             => 'content',            # inside of strings, most scanners
+    :decorator           => 'decorator',          # Python
+    :definition          => 'definition',         # CSS
+    :delimiter           => 'delimiter',          # inside strings, comments and other types
+    :directive           => 'directive',          # lots of scanners
+    :doctype             => 'doctype',            # Goorvy, HTML, Ruby, YAML
+    :docstring           => 'docstring',          # Python
+    :done                => 'done',               # Taskpaper
+    :entity              => 'entity',             # HTML
+    :error               => 'error',              # invalid token, most scanners
+    :escape              => 'escape',             # Ruby (string inline variables like #$foo, #@bar)
+    :exception           => 'exception',          # Java, PHP, Python
+    :filename            => 'filename',           # Diff
+    :float               => 'float',              # most scanners
+    :function            => 'function',           # CSS, JavaScript, PHP
+    :global_variable     => 'global-variable',    # Ruby, YAML
+    :hex                 => 'hex',                # hexadecimal number; lots of scanners
+    :id                  => 'id',                 # CSS
+    :imaginary           => 'imaginary',          # Python
+    :important           => 'important',          # CSS, Taskpaper
+    :include             => 'include',            # C, Groovy, Java, Python, Sass
+    :inline              => 'inline',             # nested code, eg. inline string evaluation; lots of scanners
+    :inline_delimiter    => 'inline-delimiter',   # used instead of :inline > :delimiter FIXME: Why use inline_delimiter?
+    :instance_variable   => 'instance-variable',  # Ruby
+    :integer             => 'integer',            # most scanners
+    :key                 => 'key',                # lots of scanners, used together with :value
+    :keyword             => 'keyword',            # reserved word that's actually implemented; most scanners
+    :label               => 'label',              # C, PHP
+    :local_variable      => 'local-variable',     # local and magic variables; some scanners
+    :map                 => 'map',                # Lua tables
+    :modifier            => 'modifier',           # used inside on strings; lots of scanners
+    :namespace           => 'namespace',          # Clojure, Java, Taskpaper
+    :octal               => 'octal',              # lots of scanners
+    :predefined          => 'predefined',         # predefined function: lots of scanners
+    :predefined_constant => 'predefined-constant',# lots of scanners
+    :predefined_type     => 'predefined-type',    # C, Java, PHP
+    :preprocessor        => 'preprocessor',       # C, Delphi, HTML
+    :pseudo_class        => 'pseudo-class',       # CSS
+    :regexp              => 'regexp',             # Groovy, JavaScript, Ruby
+    :reserved            => 'reserved',           # most scanners
+    :shell               => 'shell',              # Ruby
+    :string              => 'string',             # most scanners
+    :symbol              => 'symbol',             # Clojure, Ruby, YAML
+    :tag                 => 'tag',                # CSS, HTML
+    :type                => 'type',               # CSS, Java, SQL, YAML
+    :value               => 'value',              # used together with :key; CSS, JSON, YAML
+    :variable            => 'variable',           # Sass, SQL, YAML
+    
+    :change              => 'change',             # Diff
+    :delete              => 'delete',             # Diff
+    :head                => 'head',               # Diff, YAML
+    :insert              => 'insert',             # Diff
+    :eyecatcher          => 'eyecatcher',         # Diff
+    
+    :ident               => false,                # almost all scanners
+    :operator            => false,                # almost all scanners
+    
+    :space               => false,                # almost all scanners
+    :plain               => false                 # almost all scanners
+  )
+  
+  TokenKinds[:method]  = TokenKinds[:function]
+  TokenKinds[:unknown] = TokenKinds[:plain]
+end

data/lib/llm/shell/internal/coderay/lib/coderay/tokens.rb

@@ -0,0 +1,164 @@
+module CodeRay
+  
+  # The Tokens class represents a list of tokens returned from
+  # a Scanner. It's actually just an Array with a few helper methods.
+  #
+  # A token itself is not a special object, just two elements in an Array:
+  # * the _token_ _text_ (the original source of the token in a String) or
+  #   a _token_ _action_ (begin_group, end_group, begin_line, end_line)
+  # * the _token_ _kind_ (a Symbol representing the type of the token)
+  #
+  # It looks like this:
+  #
+  #   ..., '# It looks like this', :comment, ...
+  #   ..., '3.1415926', :float, ...
+  #   ..., '$^', :error, ...
+  #
+  # Some scanners also yield sub-tokens, represented by special
+  # token actions, for example :begin_group and :end_group.
+  #
+  # The Ruby scanner, for example, splits "a string" into:
+  #
+  #  [
+  #   :begin_group, :string,
+  #   '"',          :delimiter,
+  #   'a string',   :content,
+  #   '"',          :delimiter,
+  #   :end_group,   :string
+  #  ]
+  #
+  # Tokens can be used to save the output of a Scanners in a simple
+  # Ruby object that can be send to an Encoder later:
+  #
+  #   tokens = CodeRay.scan('price = 2.59', :ruby).tokens
+  #   tokens.encode(:html)
+  #   tokens.html
+  #   CodeRay.encoder(:html).encode_tokens(tokens)
+  #
+  # Tokens gives you the power to handle pre-scanned code very easily:
+  # You can serialize it to a JSON string and store it in a database, pass it
+  # around to encode it more than once, send it to other algorithms...
+  class Tokens < Array
+    # Remove Array#filter that is a new alias for Array#select on Ruby 2.6,
+    # for method_missing called with filter method.
+    undef_method :filter if instance_methods.include?(:filter)
+    
+    # The Scanner instance that created the tokens.
+    attr_accessor :scanner
+    
+    # Encode the tokens using encoder.
+    #
+    # encoder can be
+    # * a plugin name like :html oder 'statistic'
+    # * an Encoder object
+    #
+    # options are passed to the encoder.
+    def encode encoder, options = {}
+      encoder = Encoders[encoder].new options if encoder.respond_to? :to_sym
+      encoder.encode_tokens self, options
+    end
+    
+    # Turn tokens into a string by concatenating them.
+    def to_s
+      encode CodeRay::Encoders::Encoder.new
+    end
+    
+    # Redirects unknown methods to encoder calls.
+    #
+    # For example, if you call +tokens.html+, the HTML encoder
+    # is used to highlight the tokens.
+    def method_missing meth, options = {}
+      encode meth, options
+    rescue PluginHost::PluginNotFound
+      super
+    end
+    
+    # Split the tokens into parts of the given +sizes+.
+    # 
+    # The result will be an Array of Tokens objects. The parts have
+    # the text size specified by the parameter. In addition, each
+    # part closes all opened tokens. This is useful to insert tokens
+    # betweem them.
+    # 
+    # This method is used by @Scanner#tokenize@ when called with an Array
+    # of source strings. The Diff encoder uses it for inline highlighting.
+    def split_into_parts *sizes
+      return Array.new(sizes.size) { Tokens.new } if size == 2 && first == ''
+      parts = []
+      opened = []
+      content = nil
+      part = Tokens.new
+      part_size = 0
+      size = sizes.first
+      i = 0
+      for item in self
+        case content
+        when nil
+          content = item
+        when String
+          if size && part_size + content.size > size  # token must be cut
+            if part_size < size  # some part of the token goes into this part
+              content = content.dup  # content may no be safe to change
+              part << content.slice!(0, size - part_size) << item
+            end
+            # close all open groups and lines...
+            closing = opened.reverse.flatten.map do |content_or_kind|
+              case content_or_kind
+              when :begin_group
+                :end_group
+              when :begin_line
+                :end_line
+              else
+                content_or_kind
+              end
+            end
+            part.concat closing
+            begin
+              parts << part
+              part = Tokens.new
+              size = sizes[i += 1]
+            end until size.nil? || size > 0
+            # ...and open them again.
+            part.concat opened.flatten
+            part_size = 0
+            redo unless content.empty?
+          else
+            part << content << item
+            part_size += content.size
+          end
+          content = nil
+        when Symbol
+          case content
+          when :begin_group, :begin_line
+            opened << [content, item]
+          when :end_group, :end_line
+            opened.pop
+          else
+            raise ArgumentError, 'Unknown token action: %p, kind = %p' % [content, item]
+          end
+          part << content << item
+          content = nil
+        else
+          raise ArgumentError, 'Token input junk: %p, kind = %p' % [content, item]
+        end
+      end
+      parts << part
+      parts << Tokens.new while parts.size < sizes.size
+      parts
+    end
+    
+    # Return the actual number of tokens.
+    def count
+      size / 2
+    end
+    
+    alias text_token push
+    def begin_group kind; push :begin_group, kind end
+    def end_group kind; push :end_group, kind end
+    def begin_line kind; push :begin_line, kind end
+    def end_line kind; push :end_line, kind end
+    alias tokens concat
+    
+  end
+  
+end

data/lib/llm/shell/internal/coderay/lib/coderay/tokens_proxy.rb

@@ -0,0 +1,55 @@
+module CodeRay
+  
+  # The result of a scan operation is a TokensProxy, but should act like Tokens.
+  # 
+  # This proxy makes it possible to use the classic CodeRay.scan.encode API
+  # while still providing the benefits of direct streaming.
+  class TokensProxy
+    
+    attr_accessor :input, :lang, :options, :block
+    
+    # Create a new TokensProxy with the arguments of CodeRay.scan.
+    def initialize input, lang, options = {}, block = nil
+      @input   = input
+      @lang    = lang
+      @options = options
+      @block   = block
+    end
+    
+    # Call CodeRay.encode if +encoder+ is a Symbol;
+    # otherwise, convert the receiver to tokens and call encoder.encode_tokens.
+    def encode encoder, options = {}
+      if encoder.respond_to? :to_sym
+        CodeRay.encode(input, lang, encoder, options)
+      else
+        encoder.encode_tokens tokens, options
+      end
+    end
+    
+    # Tries to call encode;
+    # delegates to tokens otherwise.
+    def method_missing method, *args, &blk
+      encode method.to_sym, *args
+    rescue PluginHost::PluginNotFound
+      tokens.send(method, *args, &blk)
+    end
+    
+    # The (cached) result of the tokenized input; a Tokens instance.
+    def tokens
+      @tokens ||= scanner.tokenize(input)
+    end
+    
+    # A (cached) scanner instance to use for the scan task.
+    def scanner
+      @scanner ||= CodeRay.scanner(lang, options, &block)
+    end
+    
+    # Overwrite Struct#each.
+    def each *args, &blk
+      tokens.each(*args, &blk)
+      self
+    end
+    
+  end
+  
+end

data/lib/llm/shell/internal/coderay/lib/coderay/version.rb

@@ -0,0 +1,3 @@
+module CodeRay
+  VERSION = '1.1.3'
+end