rugments 1.0.0.beta1

data/lib/rugments/regex_lexer.rb

@@ -0,0 +1,432 @@
+module Rugments
+  # @abstract
+  # A stateful lexer that uses sets of regular expressions to
+  # tokenize a string.  Most lexers are instances of RegexLexer.
+  class RegexLexer < Lexer
+    # A rule is a tuple of a regular expression to test, and a callback
+    # to perform if the test succeeds.
+    #
+    # @see StateDSL#rule
+    class Rule
+      attr_reader :callback
+      attr_reader :re
+      attr_reader :beginning_of_line
+      def initialize(re, callback)
+        @re = re
+        @callback = callback
+        @beginning_of_line = re.source[0] == '^'
+      end
+
+      def inspect
+        "#<Rule #{@re.inspect}>"
+      end
+    end
+
+    # a State is a named set of rules that can be tested for or
+    # mixed in.
+    #
+    # @see RegexLexer.state
+    class State
+      attr_reader :name, :rules
+      def initialize(name, rules)
+        @name = name
+        @rules = rules
+      end
+
+      def inspect
+        "#<#{self.class.name} #{@name.inspect}>"
+      end
+    end
+
+    class StateDSL
+      attr_reader :rules
+      def initialize(name, &defn)
+        @name = name
+        @defn = defn
+        @rules = []
+      end
+
+      def to_state(lexer_class)
+        load!
+        rules = @rules.map do |rule|
+          rule.is_a?(String) ? lexer_class.get_state(rule) : rule
+        end
+        State.new(@name, rules)
+      end
+
+      def prepended(&defn)
+        parent_defn = @defn
+        StateDSL.new(@name) do
+          instance_eval(&defn)
+          instance_eval(&parent_defn)
+        end
+      end
+
+      def appended(&defn)
+        parent_defn = @defn
+        StateDSL.new(@name) do
+          instance_eval(&parent_defn)
+          instance_eval(&defn)
+        end
+      end
+
+      protected
+
+      # Define a new rule for this state.
+      #
+      # @overload rule(re, token, next_state=nil)
+      # @overload rule(re, &callback)
+      #
+      # @param [Regexp] re
+      #   a regular expression for this rule to test.
+      # @param [String] tok
+      #   the token type to yield if `re` matches.
+      # @param [#to_s] next_state
+      #   (optional) a state to push onto the stack if `re` matches.
+      #   If `next_state` is `:pop!`, the state stack will be popped
+      #   instead.
+      # @param [Proc] callback
+      #   a block that will be evaluated in the context of the lexer
+      #   if `re` matches.  This block has access to a number of lexer
+      #   methods, including {RegexLexer#push}, {RegexLexer#pop!},
+      #   {RegexLexer#token}, and {RegexLexer#delegate}.  The first
+      #   argument can be used to access the match groups.
+      def rule(re, tok = nil, next_state = nil, &callback)
+        if tok.nil? && callback.nil?
+          fail 'please pass `rule` a token to yield or a callback'
+        end
+
+        callback ||= case next_state
+        when :pop!
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            puts "    popping stack: #{1}" if @debug
+            @stack.pop || fail('empty stack!')
+          end
+        when :push
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            puts "    pushing #{@stack.last.name}" if @debug
+            @stack.push(@stack.last)
+          end
+        when Symbol
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+            state = @states[next_state] || self.class.get_state(next_state)
+            puts "    pushing #{state.name}" if @debug
+            @stack.push(state)
+          end
+        when nil
+          proc do |stream|
+            puts "    yielding #{tok.qualname}, #{stream[0].inspect}" if @debug
+            @output_stream.call(tok, stream[0])
+          end
+        else
+          fail "invalid next state: #{next_state.inspect}"
+        end
+
+        rules << Rule.new(re, callback)
+      end
+
+      # Mix in the rules from another state into this state.  The rules
+      # from the mixed-in state will be tried in order before moving on
+      # to the rest of the rules in this state.
+      def mixin(state)
+        rules << state.to_s
+      end
+
+      private
+
+      def load!
+        return if @loaded
+        @loaded = true
+        instance_eval(&@defn)
+      end
+    end
+
+    # The states hash for this lexer.
+    # @see state
+    def self.states
+      @states ||= {}
+    end
+
+    def self.state_definitions
+      @state_definitions ||= InheritableHash.new(superclass.state_definitions)
+    end
+    @state_definitions = {}
+
+    def self.replace_state(name, new_defn)
+      states[name] = nil
+      state_definitions[name] = new_defn
+    end
+
+    # The routines to run at the beginning of a fresh lex.
+    # @see start
+    def self.start_procs
+      @start_procs ||= InheritableList.new(superclass.start_procs)
+    end
+    @start_procs = []
+
+    # Specify an action to be run every fresh lex.
+    #
+    # @example
+    #   start { puts "I'm lexing a new string!" }
+    def self.start(&b)
+      start_procs << b
+    end
+
+    # Define a new state for this lexer with the given name.
+    # The block will be evaluated in the context of a {StateDSL}.
+    def self.state(name, &b)
+      name = name.to_s
+      state_definitions[name] = StateDSL.new(name, &b)
+    end
+
+    def self.prepend(name, &b)
+      name = name.to_s
+      dsl = state_definitions[name] or fail "no such state #{name.inspect}"
+      replace_state(name, dsl.prepended(&b))
+    end
+
+    def self.append(_state, &b)
+      name = name.to_s
+      dsl = state_definitions[name] or fail "no such state #{name.inspect}"
+      replace_state(name, dsl.appended(&b))
+    end
+
+    # @private
+    def self.get_state(name)
+      return name if name.is_a? State
+
+      states[name.to_sym] ||= begin
+        defn = state_definitions[name.to_s] or fail "unknown state: #{name.inspect}"
+        defn.to_state(self)
+      end
+    end
+
+    # @private
+    def get_state(state_name)
+      self.class.get_state(state_name)
+    end
+
+    # The state stack.  This is initially the single state `[:root]`.
+    # It is an error for this stack to be empty.
+    # @see #state
+    def stack
+      @stack ||= [get_state(:root)]
+    end
+
+    # The current state - i.e. one on top of the state stack.
+    #
+    # NB: if the state stack is empty, this will throw an error rather
+    # than returning nil.
+    def state
+      stack.last || fail('empty stack!')
+    end
+
+    # reset this lexer to its initial state.  This runs all of the
+    # start_procs.
+    def reset!
+      @stack = nil
+      @current_stream = nil
+
+      self.class.start_procs.each do |pr|
+        instance_eval(&pr)
+      end
+    end
+
+    # This implements the lexer protocol, by yielding [token, value] pairs.
+    #
+    # The process for lexing works as follows, until the stream is empty:
+    #
+    # 1. We look at the state on top of the stack (which by default is
+    #    `[:root]`).
+    # 2. Each rule in that state is tried until one is successful.  If one
+    #    is found, that rule's callback is evaluated - which may yield
+    #    tokens and manipulate the state stack.  Otherwise, one character
+    #    is consumed with an `'Error'` token, and we continue at (1.)
+    #
+    # @see #step #step (where (2.) is implemented)
+    def stream_tokens(str, &b)
+      stream = StringScanner.new(str)
+
+      @current_stream = stream
+      @output_stream  = b
+      @states         = self.class.states
+      @null_steps     = 0
+
+      until stream.eos?
+        if @debug
+          puts "lexer: #{self.class.tag}"
+          puts "stack: #{stack.map(&:name).inspect}"
+          puts "stream: #{stream.peek(20).inspect}"
+        end
+
+        success = step(state, stream)
+
+        unless success
+          puts '    no match, yielding Error' if @debug
+          b.call(Token::Tokens::Error, stream.getch)
+        end
+      end
+    end
+
+    # The number of successive scans permitted without consuming
+    # the input stream.  If this is exceeded, the match fails.
+    MAX_NULL_SCANS = 5
+
+    # Runs one step of the lex.  Rules in the current state are tried
+    # until one matches, at which point its callback is called.
+    #
+    # @return true if a rule was tried successfully
+    # @return false otherwise.
+    def step(state, stream)
+      state.rules.each do |rule|
+        if rule.is_a?(State)
+          puts "  entering mixin #{rule.name}" if @debug
+          return true if step(rule, stream)
+          puts "  exiting  mixin #{rule.name}" if @debug
+        else
+          puts "  trying #{rule.inspect}" if @debug
+
+          # XXX HACK XXX
+          # StringScanner's implementation of ^ is b0rken.
+          # see http://bugs.ruby-lang.org/issues/7092
+          # TODO: this doesn't cover cases like /(a|^b)/, but it's
+          # the most common, for now...
+          next if rule.beginning_of_line && !stream.beginning_of_line?
+
+          if size = stream.skip(rule.re)
+            puts "    got #{stream[0].inspect}" if @debug
+
+            instance_exec(stream, &rule.callback)
+
+            if size.zero?
+              @null_steps += 1
+              if @null_steps > MAX_NULL_SCANS
+                puts '    too many scans without consuming the string!' if @debug
+                return false
+              end
+            else
+              @null_steps = 0
+            end
+
+            return true
+          end
+        end
+      end
+
+      false
+    end
+
+    # Yield a token.
+    #
+    # @param tok
+    #   the token type
+    # @param val
+    #   (optional) the string value to yield.  If absent, this defaults
+    #   to the entire last match.
+    def token(tok, val = @current_stream[0])
+      yield_token(tok, val)
+    end
+
+    # Yield tokens corresponding to the matched groups of the current
+    # match.
+    def groups(*tokens)
+      tokens.each_with_index do |tok, i|
+        yield_token(tok, @current_stream[i + 1])
+      end
+    end
+
+    # Delegate the lex to another lexer.  The #lex method will be called
+    # with `:continue` set to true, so that #reset! will not be called.
+    # In this way, a single lexer can be repeatedly delegated to while
+    # maintaining its own internal state stack.
+    #
+    # @param [#lex] lexer
+    #   The lexer or lexer class to delegate to
+    # @param [String] text
+    #   The text to delegate.  This defaults to the last matched string.
+    def delegate(lexer, text = nil)
+      puts "    delegating to #{lexer.inspect}" if @debug
+      text ||= @current_stream[0]
+
+      lexer.lex(text, continue: true) do |tok, val|
+        puts "    delegated token: #{tok.inspect}, #{val.inspect}" if @debug
+        yield_token(tok, val)
+      end
+    end
+
+    def recurse(text = nil)
+      delegate(self.class, text)
+    end
+
+    # Push a state onto the stack.  If no state name is given and you've
+    # passed a block, a state will be dynamically created using the
+    # {StateDSL}.
+    def push(state_name = nil, &b)
+      push_state = if state_name
+                     get_state(state_name)
+                   elsif block_given?
+                     StateDSL.new(b.inspect, &b).to_state(self.class)
+                   else
+                     # use the top of the stack by default
+                     state
+      end
+
+      puts "    pushing #{push_state.name}" if @debug
+      stack.push(push_state)
+    end
+
+    # Pop the state stack.  If a number is passed in, it will be popped
+    # that number of times.
+    def pop!(times = 1)
+      fail 'empty stack!' if stack.empty?
+
+      puts "    popping stack: #{times}" if @debug
+
+      stack.pop(times)
+
+      nil
+    end
+
+    # replace the head of the stack with the given state
+    def goto(state_name)
+      fail 'empty stack!' if stack.empty?
+
+      puts "    going to state #{state_name} " if @debug
+      stack[-1] = get_state(state_name)
+    end
+
+    # reset the stack back to `[:root]`.
+    def reset_stack
+      puts '    resetting stack' if @debug
+      stack.clear
+      stack.push get_state(:root)
+    end
+
+    # Check if `state_name` is in the state stack.
+    def in_state?(state_name)
+      state_name = state_name.to_s
+      stack.any? do |state|
+        state.name == state_name.to_s
+      end
+    end
+
+    # Check if `state_name` is the state on top of the state stack.
+    def state?(state_name)
+      state_name.to_s == state.name
+    end
+
+    private
+
+    def yield_token(tok, val)
+      return if val.nil? || val.empty?
+      puts "    yielding #{tok.qualname}, #{val.inspect}" if @debug
+      @output_stream.yield(tok, val)
+    end
+  end
+end

data/lib/rugments/template_lexer.rb

@@ -0,0 +1,23 @@
+module Rugments
+  # @abstract
+  # A TemplateLexer is one that accepts a :parent option, to specify
+  # which language is being templated.  The lexer class can specify its
+  # own default for the parent lexer, which is otherwise defaulted to
+  # HTML.
+  class TemplateLexer < RegexLexer
+    # the parent lexer - the one being templated.
+    def parent
+      return @parent if instance_variable_defined? :@parent
+      @parent = option(:parent) || 'html'
+      if @parent.is_a? ::String
+        lexer_class = Lexer.find(@parent)
+        @parent = lexer_class.new(options)
+      end
+    end
+
+    start { parent.reset! }
+  end
+end
+
+lib_path = File.expand_path(File.dirname(__FILE__))
+Dir.glob(File.join(lib_path, 'lexers/*.rb')) { |f| require_relative f }

data/lib/rugments/text_analyzer.rb

@@ -0,0 +1,46 @@
+module Rugments
+  class TextAnalyzer < String
+    # Find a shebang.  Returns nil if no shebang is present.
+    def shebang
+      return @shebang if instance_variable_defined? :@shebang
+
+      self =~ /\A\s*#!(.*)$/
+      @shebang = $1
+    end
+
+    # Check if the given shebang is present.
+    #
+    # This normalizes things so that `text.shebang?('bash')` will detect
+    # `#!/bash`, '#!/bin/bash', '#!/usr/bin/env bash', and '#!/bin/bash -x'
+    def shebang?(match)
+      match = /\b#{match}(\s|$)/
+      match === shebang
+    end
+
+    # Return the contents of the doctype tag if present, nil otherwise.
+    def doctype
+      return @doctype if instance_variable_defined? :@doctype
+
+      self =~ %r(\A\s*
+        (?:<\?.*?\?>\s*)? # possible <?xml...?> tag
+        <!DOCTYPE\s+(.+?)>
+      )xm
+      @doctype = $1
+    end
+
+    # Check if the doctype matches a given regexp or string
+    def doctype?(type=//)
+      type === doctype
+    end
+
+    # Return true if the result of lexing with the given lexer contains no
+    # error tokens.
+    def lexes_cleanly?(lexer)
+      lexer.lex(self) do |(tok, _)|
+        return false if tok.name == 'Error'
+      end
+
+      true
+    end
+  end
+end

data/lib/rugments/theme.rb

@@ -0,0 +1,202 @@
+module Rugments
+  class Theme
+    include Token::Tokens
+
+    class Style < Hash
+      def initialize(theme, hsh = {})
+        super()
+        @theme = theme
+        merge!(hsh)
+      end
+
+      [:fg, :bg].each do |mode|
+        define_method mode do
+          return self[mode] unless @theme
+          @theme.palette(self[mode]) if self[mode]
+        end
+      end
+
+      def render(selector, &b)
+        return enum_for(:render, selector).to_a.join("\n") unless b
+
+        return if empty?
+
+        yield "#{selector} {"
+        rendered_rules.each do |rule|
+          yield "  #{rule};"
+        end
+        yield '}'
+      end
+
+      def rendered_rules(&b)
+        return enum_for(:rendered_rules) unless b
+        yield "color: #{fg}" if fg
+        yield "background-color: #{bg}" if bg
+        yield 'font-weight: bold' if self[:bold]
+        yield 'font-style: italic' if self[:italic]
+        yield 'text-decoration: underline' if self[:underline]
+
+        (self[:rules] || []).each(&b)
+      end
+    end
+
+    def styles
+      @styles ||= self.class.styles.dup
+    end
+
+    @palette = {}
+    def self.palette(arg = {})
+      @palette ||= InheritableHash.new(superclass.palette)
+
+      if arg.is_a? Hash
+        @palette.merge! arg
+        @palette
+      else
+        case arg
+        when /#[0-9a-f]+/i
+          arg
+        else
+          @palette[arg] || fail("not in palette: #{arg.inspect}")
+        end
+      end
+    end
+
+    @styles = {}
+    def self.styles
+      @styles ||= InheritableHash.new(superclass.styles)
+    end
+
+    def self.render(opts = {}, &b)
+      new(opts).render(&b)
+    end
+
+    class << self
+      def style(*tokens)
+        style = tokens.last.is_a?(Hash) ? tokens.pop : {}
+
+        style = Style.new(self, style)
+
+        tokens.each do |tok|
+          styles[tok] = style
+        end
+      end
+
+      def get_own_style(token)
+        token.token_chain.each do |anc|
+          return styles[anc] if styles[anc]
+        end
+
+        nil
+      end
+
+      def get_style(token)
+        get_own_style(token) || base_style
+      end
+
+      def base_style
+        styles[Token::Tokens::Text]
+      end
+
+      def name(n = nil)
+        return @name if n.nil?
+
+        @name = n.to_s
+        Theme.registry[@name] = self
+      end
+
+      def find(n)
+        registry[n.to_s]
+      end
+
+      def registry
+        @registry ||= {}
+      end
+    end
+  end
+
+  module HasModes
+    def mode(arg = :absent)
+      return @mode if arg == :absent
+
+      @modes ||= {}
+      @modes[arg] ||= get_mode(arg)
+    end
+
+    def get_mode(mode)
+      return self if self.mode == mode
+
+      new_name = "#{name}.#{mode}"
+      Class.new(self) { name(new_name); mode!(mode) }
+    end
+
+    def mode!(arg)
+      @mode = arg
+      send("make_#{arg}!")
+    end
+  end
+
+  class CSSTheme < Theme
+    def initialize(opts = {})
+      @scope = opts[:scope] || '.highlight'
+    end
+
+    def render(&b)
+      return enum_for(:render).to_a.join("\n") unless b
+
+      # shared styles for tableized line numbers
+      yield "#{@scope} table td { padding: 5px; }"
+      yield "#{@scope} table pre { margin: 0; }"
+
+      styles.each do |tok, style|
+        style.render(css_selector(tok), &b)
+      end
+    end
+
+    def render_base(selector, &b)
+      self.class.base_style.render(selector, &b)
+    end
+
+    def style_for(tok)
+      self.class.get_style(tok)
+    end
+
+    private
+
+    def css_selector(token)
+      inflate_token(token).map do |tok|
+        fail "unknown token: #{tok.inspect}" if tok.shortname.nil?
+
+        single_css_selector(tok)
+      end.join(', ')
+    end
+
+    def single_css_selector(token)
+      return @scope if token == Text
+
+      "#{@scope} .#{token.shortname}"
+    end
+
+    # yield all of the tokens that should be styled the same
+    # as the given token.  Essentially this recursively all of
+    # the subtokens, except those which are more specifically
+    # styled.
+    def inflate_token(tok, &b)
+      return enum_for(:inflate_token, tok) unless block_given?
+
+      yield tok
+      tok.sub_tokens.each do |(_, st)|
+        next if styles[st]
+
+        inflate_token(st, &b)
+      end
+    end
+  end
+end
+
+
+require_relative 'themes/thankful_eyes'
+require_relative 'themes/colorful'
+require_relative 'themes/base16'
+require_relative 'themes/github'
+require_relative 'themes/monokai'
+require_relative 'themes/monokai_sublime'