llm-shell 0.9.2 → 0.10.0

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/json.rb

@@ -0,0 +1,83 @@
+module CodeRay
+module Encoders
+  
+  # A simple JSON Encoder.
+  # 
+  # Example:
+  #  CodeRay.scan('puts "Hello world!"', :ruby).json
+  # yields
+  #  [
+  #    {"type"=>"text", "text"=>"puts", "kind"=>"ident"},
+  #    {"type"=>"text", "text"=>" ", "kind"=>"space"},
+  #    {"type"=>"block", "action"=>"open", "kind"=>"string"},
+  #    {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+  #    {"type"=>"text", "text"=>"Hello world!", "kind"=>"content"},
+  #    {"type"=>"text", "text"=>"\"", "kind"=>"delimiter"},
+  #    {"type"=>"block", "action"=>"close", "kind"=>"string"},
+  #  ]
+  class JSON < Encoder
+    
+    begin
+      require 'json'
+    rescue LoadError
+      begin
+        require 'rubygems' unless defined? Gem
+        gem 'json'
+        require 'json'
+      rescue LoadError
+        $stderr.puts "The JSON encoder needs the JSON library.\n" \
+          "Please gem install json."
+        raise
+      end
+    end
+    
+    register_for :json
+    FILE_EXTENSION = 'json'
+    
+  protected
+    def setup options
+      super
+      
+      @first = true
+      @out << '['
+    end
+    
+    def finish options
+      @out << ']'
+    end
+    
+    def append data
+      if @first
+        @first = false
+      else
+        @out << ','
+      end
+      
+      @out << data.to_json
+    end
+    
+  public
+    def text_token text, kind
+      append :type => 'text', :text => text, :kind => kind
+    end
+    
+    def begin_group kind
+      append :type => 'block', :action => 'open', :kind => kind
+    end
+    
+    def end_group kind
+      append :type => 'block', :action => 'close', :kind => kind
+    end
+    
+    def begin_line kind
+      append :type => 'block', :action => 'begin_line', :kind => kind
+    end
+    
+    def end_line kind
+      append :type => 'block', :action => 'end_line', :kind => kind
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/lines_of_code.rb

@@ -0,0 +1,45 @@
+module CodeRay
+module Encoders
+  
+  # Counts the LoC (Lines of Code). Returns an Integer >= 0.
+  # 
+  # Alias: +loc+
+  # 
+  # Everything that is not comment, markup, doctype/shebang, or an empty line,
+  # is considered to be code.
+  # 
+  # For example,
+  # * HTML files not containing JavaScript have 0 LoC
+  # * in a Java class without comments, LoC is the number of non-empty lines
+  # 
+  # A Scanner class should define the token kinds that are not code in the
+  # KINDS_NOT_LOC constant, which defaults to [:comment, :doctype].
+  class LinesOfCode < TokenKindFilter
+    
+    register_for :lines_of_code
+    
+    NON_EMPTY_LINE = /^\s*\S.*$/
+    
+  protected
+    
+    def setup options
+      if scanner
+        kinds_not_loc = scanner.class::KINDS_NOT_LOC
+      else
+        warn "Tokens have no associated scanner, counting all nonempty lines." if $VERBOSE
+        kinds_not_loc = CodeRay::Scanners::Scanner::KINDS_NOT_LOC
+      end
+      
+      options[:exclude] = kinds_not_loc
+      
+      super options
+    end
+    
+    def finish options
+      output @tokens.text.scan(NON_EMPTY_LINE).size
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/lint.rb

@@ -0,0 +1,59 @@
+module CodeRay
+module Encoders
+  
+  # = Lint Encoder
+  #
+  # Checks for:
+  # 
+  # - empty tokens
+  # - incorrect nesting
+  # 
+  # It will raise an InvalidTokenStream exception when any of the above occurs.
+  # 
+  # See also: Encoders::DebugLint
+  class Lint < Debug
+    
+    register_for :lint
+    
+    InvalidTokenStream         = Class.new StandardError
+    EmptyToken                 = Class.new InvalidTokenStream
+    UnknownTokenKind           = Class.new InvalidTokenStream
+    IncorrectTokenGroupNesting = Class.new InvalidTokenStream
+    
+    def text_token text, kind
+      raise EmptyToken,       'empty token for %p' % [kind] if text.empty?
+      raise UnknownTokenKind, 'unknown token kind %p (text was %p)' % [kind, text] unless TokenKinds.has_key? kind
+    end
+    
+    def begin_group kind
+      @opened << kind
+    end
+    
+    def end_group kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_group)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    def begin_line kind
+      @opened << kind
+    end
+    
+    def end_line kind
+      raise IncorrectTokenGroupNesting, 'We are inside %s, not %p (end_line)' % [@opened.reverse.map(&:inspect).join(' < '), kind] if @opened.last != kind
+      @opened.pop
+    end
+    
+    protected
+    
+    def setup options
+      @opened = []
+    end
+    
+    def finish options
+      raise 'Some tokens still open at end of token stream: %p' % [@opened] unless @opened.empty?
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/null.rb

@@ -0,0 +1,18 @@
+module CodeRay
+module Encoders
+  
+  # = Null Encoder
+  #
+  # Does nothing and returns an empty string.
+  class Null < Encoder
+    
+    register_for :null
+    
+    def text_token text, kind
+      # do nothing
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/page.rb

@@ -0,0 +1,24 @@
+module CodeRay
+module Encoders
+  
+  load :html
+  
+  # Wraps the output into a HTML page, using CSS classes and
+  # line numbers in the table format by default.
+  # 
+  # See Encoders::HTML for available options.
+  class Page < HTML
+    
+    FILE_EXTENSION = 'html'
+    
+    register_for :page
+    
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css          => :class,
+      :wrap         => :page,
+      :line_numbers => :table
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/span.rb

@@ -0,0 +1,23 @@
+module CodeRay
+module Encoders
+  
+  load :html
+  
+  # Wraps HTML output into a SPAN element, using inline styles by default.
+  # 
+  # See Encoders::HTML for available options.
+  class Span < HTML
+    
+    FILE_EXTENSION = 'span.html'
+    
+    register_for :span
+    
+    DEFAULT_OPTIONS = HTML::DEFAULT_OPTIONS.merge \
+      :css          => :style,
+      :wrap         => :span,
+      :line_numbers => false
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/statistic.rb

@@ -0,0 +1,95 @@
+module CodeRay
+module Encoders
+  
+  # Makes a statistic for the given tokens.
+  # 
+  # Alias: +stats+
+  class Statistic < Encoder
+    
+    register_for :statistic
+    
+    attr_reader :type_stats, :real_token_count  # :nodoc:
+    
+    TypeStats = Struct.new :count, :size  # :nodoc:
+    
+  protected
+    
+    def setup options
+      super
+      
+      @type_stats = Hash.new { |h, k| h[k] = TypeStats.new 0, 0 }
+      @real_token_count = 0
+    end
+    
+    STATS = <<-STATS  # :nodoc:
+
+Code Statistics
+
+Tokens            %8d
+  Non-Whitespace  %8d
+Bytes Total       %8d
+
+Token Types (%d):
+  type                     count     ratio    size (average)
+-------------------------------------------------------------
+%s
+    STATS
+    
+    TOKEN_TYPES_ROW = <<-TKR  # :nodoc:
+  %-20s  %8d  %6.2f %%   %5.1f
+    TKR
+    
+    def finish options
+      all = @type_stats['TOTAL']
+      all_count, all_size = all.count, all.size
+      @type_stats.each do |type, stat|
+        stat.size /= stat.count.to_f
+      end
+      types_stats = @type_stats.sort_by { |k, v| [-v.count, k.to_s] }.map do |k, v|
+        TOKEN_TYPES_ROW % [k, v.count, 100.0 * v.count / all_count, v.size]
+      end.join
+      @out << STATS % [
+        all_count, @real_token_count, all_size,
+        @type_stats.delete_if { |k, v| k.is_a? String }.size,
+        types_stats
+      ]
+      
+      super
+    end
+    
+  public
+    
+    def text_token text, kind
+      @real_token_count += 1 unless kind == :space
+      @type_stats[kind].count += 1
+      @type_stats[kind].size += text.size
+      @type_stats['TOTAL'].size += text.size
+      @type_stats['TOTAL'].count += 1
+    end
+    
+    def begin_group kind
+      block_token ':begin_group', kind
+    end
+    
+    def end_group kind
+      block_token ':end_group', kind
+    end
+    
+    def begin_line kind
+      block_token ':begin_line', kind
+    end
+    
+    def end_line kind
+      block_token ':end_line', kind
+    end
+    
+    def block_token action, kind
+      @type_stats['TOTAL'].count += 1
+      @type_stats[action].count += 1
+      @type_stats[kind].count += 1
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/terminal.rb

@@ -0,0 +1,195 @@
+module CodeRay
+  module Encoders
+    
+    # Outputs code highlighted for a color terminal.
+    # 
+    # Note: This encoder is in beta. It currently doesn't use the Styles.
+    # 
+    # Alias: +term+
+    # 
+    # == Authors & License
+    # 
+    # By Rob Aldred (http://robaldred.co.uk)
+    # 
+    # Based on idea by Nathan Weizenbaum (http://nex-3.com)
+    # 
+    # MIT License (http://www.opensource.org/licenses/mit-license.php)
+    class Terminal < Encoder
+      
+      register_for :terminal
+      
+      TOKEN_COLORS = {
+        :debug => "\e[1;37;44m",
+        
+        :annotation => "\e[34m",
+        :attribute_name => "\e[35m",
+        :attribute_value => "\e[31m",
+        :binary => {
+          :self => "\e[31m",
+          :char => "\e[1;31m",
+          :delimiter => "\e[1;31m",
+        },
+        :char => {
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m"
+        },
+        :class => "\e[1;35;4m",
+        :class_variable => "\e[36m",
+        :color => "\e[32m",
+        :comment => {
+          :self => "\e[1;30m",
+          :char => "\e[37m",
+          :delimiter => "\e[37m",
+        },
+        :constant => "\e[1;34;4m",
+        :decorator => "\e[35m",
+        :definition => "\e[1;33m",
+        :directive => "\e[33m",
+        :docstring => "\e[31m",
+        :doctype => "\e[1;34m",
+        :done => "\e[1;30;2m",
+        :entity => "\e[31m",
+        :error => "\e[1;37;41m",
+        :exception => "\e[1;31m",
+        :float => "\e[1;35m",
+        :function => "\e[1;34m",
+        :global_variable => "\e[1;32m",
+        :hex => "\e[1;36m",
+        :id => "\e[1;34m",
+        :include => "\e[31m",
+        :integer => "\e[1;34m",
+        :imaginary => "\e[1;34m",
+        :important => "\e[1;31m",
+        :key => {
+          :self => "\e[35m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;35m",
+        },
+        :keyword => "\e[32m",
+        :label => "\e[1;33m",
+        :local_variable => "\e[33m",
+        :namespace => "\e[1;35m",
+        :octal => "\e[1;34m",
+        :predefined => "\e[36m",
+        :predefined_constant => "\e[1;36m",
+        :predefined_type => "\e[1;32m",
+        :preprocessor => "\e[1;36m",
+        :pseudo_class => "\e[1;34m",
+        :regexp => {
+          :self => "\e[35m",
+          :delimiter => "\e[1;35m",
+          :modifier => "\e[35m",
+          :char => "\e[1;35m",
+        },
+        :reserved => "\e[32m",
+        :shell => {
+          :self => "\e[33m",
+          :char => "\e[1;33m",
+          :delimiter => "\e[1;33m",
+          :escape => "\e[1;33m",
+        },
+        :string => {
+          :self => "\e[31m",
+          :modifier => "\e[1;31m",
+          :char => "\e[1;35m",
+          :delimiter => "\e[1;31m",
+          :escape => "\e[1;31m",
+        },
+        :symbol => {
+          :self => "\e[33m",
+          :delimiter => "\e[1;33m",
+        },
+        :tag => "\e[32m",
+        :type => "\e[1;34m",
+        :value => "\e[36m",
+        :variable => "\e[34m",
+        
+        :insert => {
+          :self => "\e[42m",
+          :insert => "\e[1;32;42m",
+          :eyecatcher => "\e[102m",
+        },
+        :delete => {
+          :self => "\e[41m",
+          :delete => "\e[1;31;41m",
+          :eyecatcher => "\e[101m",
+        },
+        :change => {
+          :self => "\e[44m",
+          :change => "\e[37;44m",
+        },
+        :head => {
+          :self => "\e[45m",
+          :filename => "\e[37;45m"
+        },
+      }
+      
+      TOKEN_COLORS[:keyword] = TOKEN_COLORS[:reserved]
+      TOKEN_COLORS[:method] = TOKEN_COLORS[:function]
+      TOKEN_COLORS[:escape] = TOKEN_COLORS[:delimiter]
+      
+    protected
+      
+      def setup(options)
+        super
+        @opened = []
+        @color_scopes = [TOKEN_COLORS]
+      end
+      
+    public
+      
+      def text_token text, kind
+        if color = @color_scopes.last[kind]
+          color = color[:self] if color.is_a? Hash
+          
+          @out << color
+          @out << (text.index("\n") ? text.gsub("\n", "\e[0m\n" + color) : text)
+          @out << "\e[0m"
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
+        else
+          @out << text
+        end
+      end
+      
+      def begin_group kind
+        @opened << kind
+        @out << open_token(kind)
+      end
+      alias begin_line begin_group
+      
+      def end_group kind
+        if @opened.pop
+          @color_scopes.pop
+          @out << "\e[0m"
+          if outer_color = @color_scopes.last[:self]
+            @out << outer_color
+          end
+        end
+      end
+      
+      def end_line kind
+        @out << (@line_filler ||= "\t" * 100)
+        end_group kind
+      end
+      
+    private
+      
+      def open_token kind
+        if color = @color_scopes.last[kind]
+          if color.is_a? Hash
+            @color_scopes << color
+            color[:self]
+          else
+            @color_scopes << @color_scopes.last
+            color
+          end
+        else
+          @color_scopes << @color_scopes.last
+          ''
+        end
+      end
+    end
+  end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/text.rb

@@ -0,0 +1,46 @@
+module CodeRay
+module Encoders
+  
+  # Concats the tokens into a single string, resulting in the original
+  # code string if no tokens were removed.
+  # 
+  # Alias: +plain+, +plaintext+
+  # 
+  # == Options
+  # 
+  # === :separator
+  # A separator string to join the tokens.
+  # 
+  # Default: empty String
+  class Text < Encoder
+    
+    register_for :text
+    
+    FILE_EXTENSION = 'txt'
+    
+    DEFAULT_OPTIONS = {
+      :separator => nil
+    }
+    
+    def text_token text, kind
+      super
+      
+      if @first
+        @first = false
+      else
+        @out << @sep
+      end if @sep
+    end
+    
+  protected
+    def setup options
+      super
+      
+      @first = true
+      @sep = options[:separator]
+    end
+    
+  end
+  
+end
+end

data/lib/llm/shell/internal/coderay/lib/coderay/encoders/token_kind_filter.rb

@@ -0,0 +1,111 @@
+module CodeRay
+module Encoders
+  
+  load :filter
+  
+  # A Filter that selects tokens based on their token kind.
+  # 
+  # == Options
+  # 
+  # === :exclude
+  # 
+  # One or many symbols (in an Array) which shall be excluded.
+  # 
+  # Default: []
+  # 
+  # === :include
+  # 
+  # One or many symbols (in an array) which shall be included.
+  # 
+  # Default: :all, which means all tokens are included.
+  # 
+  # Exclusion wins over inclusion.
+  # 
+  # See also: CommentFilter
+  class TokenKindFilter < Filter
+    
+    register_for :token_kind_filter
+    
+    DEFAULT_OPTIONS = {
+      :exclude => [],
+      :include => :all
+    }
+    
+  protected
+    def setup options
+      super
+      
+      @group_excluded = false
+      @exclude = options[:exclude]
+      @exclude = Array(@exclude) unless @exclude == :all
+      @include = options[:include]
+      @include = Array(@include) unless @include == :all
+    end
+    
+    def include_text_token? text, kind
+      include_group? kind
+    end
+    
+    def include_group? kind
+       (@include == :all || @include.include?(kind)) &&
+      !(@exclude == :all || @exclude.include?(kind))
+    end
+    
+  public
+    
+    # Add the token to the output stream if +kind+ matches the conditions.
+    def text_token text, kind
+      super if !@group_excluded && include_text_token?(text, kind)
+    end
+    
+    # Add the token group to the output stream if +kind+ matches the
+    # conditions.
+    # 
+    # If it does not, all tokens inside the group are excluded from the
+    # stream, even if their kinds match.
+    def begin_group kind
+      if @group_excluded
+        @group_excluded += 1
+      elsif include_group? kind
+        super
+      else
+        @group_excluded = 1
+      end
+    end
+    
+    # See +begin_group+.
+    def begin_line kind
+      if @group_excluded
+        @group_excluded += 1
+      elsif include_group? kind
+        super
+      else
+        @group_excluded = 1
+      end
+    end
+    
+    # Take care of re-enabling the delegation of tokens to the output stream
+    # if an exluded group has ended.
+    def end_group kind
+      if @group_excluded
+        @group_excluded -= 1
+        @group_excluded = false if @group_excluded.zero?
+      else
+        super
+      end
+    end
+    
+    # See +end_group+.
+    def end_line kind
+      if @group_excluded
+        @group_excluded -= 1
+        @group_excluded = false if @group_excluded.zero?
+      else
+        super
+      end
+    end
+    
+  end
+  
+end
+end