shoes-highlighter 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 06db840874b7b465b090576dc13bba0274542c67
+  data.tar.gz: 0252050fbd61257f89648d4076fd41338b399e12
+SHA512:
+  metadata.gz: ee92054f48c6b63d9618c38e3246c6e814fbdcbb6d90466c7b29ae272c1efe966db378299d3011b37060683d33697b67f3048f05d1ac71127adc5afbd8fa52c2
+  data.tar.gz: 7b4a67cded1f6f77106419e48e5b46e3ad4fa428a4123f95ed56ad0f194c79143cd9b46e96500fb5ecb37c4a7359da856d6492ba037da1539c6e577a8c7480e7

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoes-highlighter.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,31 @@
+Copyright (C) 2011 Steve Klabnik, Peter Fitzgibbons
+Copyright (c) 2008 why the lucky stiff
+Except:
+  fonts/Coolvetica.ttf (c) 1999 Ray Larabie
+  fonts/Lacuna.ttf (c) 2003 Glashaus, designed by Peter Hoffman
+  samples/expert-minesweeper.rb (c) 2008 que
+  samples/expert-othello.rb (c) 2008 Tieg Zaharia
+  samples/expert-tankspank.rb (c) 2008 Kevin C.
+  samples/good-clock.rb (c) 2008 Thomas Bell
+  samples/good-reminder.rb (c) 2008 Oliver Smith
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without restriction,
+including without limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+  
+The above copyright notice and this permission notice shall be 
+included in all copies or substantial portions of the Software.
+   
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT
+OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,31 @@
+# Shoes::Highlighter
+
+A syntax highlighting gem, extracted from Shoes and Hackety Hack.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'shoes-highlighter'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install shoes-highlighter
+
+## Usage
+
+See https://github.com/shoes/shoes4/blob/master/lib/shoes/ui/help.rb
+
+## Contributing
+
+1. Fork it ( https://github.com/shoes/shoes-highlighter/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/shoes/highlighter.rb

@@ -0,0 +1,8 @@
+require "shoes/highlighter/version"
+require "shoes/highlighter/common"
+require "shoes/highlighter/markup"
+
+class Shoes
+  module Highlighter
+  end
+end

data/lib/shoes/highlighter/common.rb

@@ -0,0 +1,192 @@
+require 'strscan'
+
+class Shoes
+  module Highlighter
+    module Syntax
+      # A single token extracted by a tokenizer. It is simply the lexeme
+      # itself, decorated with a 'group' attribute to identify the type of the
+      # lexeme.
+      class Token < String
+        # the type of the lexeme that was extracted.
+        attr_reader :group
+
+        # the instruction associated with this token (:none, :region_open, or
+        # :region_close)
+        attr_reader :instruction
+
+        # Create a new Token representing the given text, and belonging to the
+        # given group.
+        def initialize(text, group, instruction = :none)
+          super text
+          @group = group
+          @instruction = instruction
+        end
+      end
+
+      # The base class of all tokenizers. It sets up the scanner and manages the
+      # looping until all tokens have been extracted. It also provides convenience
+      # methods to make sure adjacent tokens of identical groups are returned as
+      # a single token.
+      class Tokenizer
+        # The current group being processed by the tokenizer
+        attr_reader :group
+
+        # The current chunk of text being accumulated
+        attr_reader :chunk
+
+        # Start tokenizing. This sets up the state in preparation for tokenization,
+        # such as creating a new scanner for the text and saving the callback block.
+        # The block will be invoked for each token extracted.
+        def start(text, &block)
+          @chunk = ""
+          @group = :normal
+          @callback = block
+          @text = StringScanner.new(text)
+          setup
+        end
+
+        # Subclasses may override this method to provide implementation-specific
+        # setup logic.
+        def setup
+        end
+
+        # Finish tokenizing. This flushes the buffer, yielding any remaining text
+        # to the client.
+        def finish
+          start_group nil
+          teardown
+        end
+
+        # Subclasses may override this method to provide implementation-specific
+        # teardown logic.
+        def teardown
+        end
+
+        # Subclasses must implement this method, which is called for each iteration
+        # of the tokenization process. This method may extract multiple tokens.
+        def step
+          fail NotImplementedError, "subclasses must implement #step"
+        end
+
+        # Begins tokenizing the given text, calling #step until the text has been
+        # exhausted.
+        def tokenize(text, &block)
+          start text, &block
+          step until @text.eos?
+          finish
+        end
+
+        # Specify a set of tokenizer-specific options. Each tokenizer may (or may
+        # not) publish any options, but if a tokenizer does those options may be
+        # used to specify optional behavior.
+        def set(opts = {})
+          (@options ||= {}).update opts
+        end
+
+        # Get the value of the specified option.
+        def option(opt)
+          @options ? @options[opt] : nil
+        end
+
+        private
+
+        EOL = /(?=\r\n?|\n|$)/
+
+        # A convenience for delegating method calls to the scanner.
+        def self.delegate(sym)
+          define_method(sym) { |*a| @text.__send__(sym, *a) }
+        end
+
+        delegate :bol?
+        delegate :eos?
+        delegate :scan
+        delegate :scan_until
+        delegate :check
+        delegate :check_until
+        delegate :getch
+        delegate :matched
+        delegate :pre_match
+        delegate :peek
+        delegate :pos
+
+        # Access the n-th subgroup from the most recent match.
+        def subgroup(n)
+          @text[n]
+        end
+
+        # Append the given data to the currently active chunk.
+        def append(data)
+          @chunk << data
+        end
+
+        # Request that a new group be started. If the current group is the same
+        # as the group being requested, a new group will not be created. If a new
+        # group is created and the current chunk is not empty, the chunk's
+        # contents will be yielded to the client as a token, and then cleared.
+        #
+        # After the new group is started, if +data+ is non-nil it will be appended
+        # to the chunk.
+        def start_group(gr, data = nil)
+          flush_chunk if gr != @group
+          @group = gr
+          @chunk << data if data
+        end
+
+        def start_region(gr, data = nil)
+          flush_chunk
+          @group = gr
+          @callback.call(Token.new(data || "", @group, :region_open))
+        end
+
+        def end_region(gr, data = nil)
+          flush_chunk
+          @group = gr
+          @callback.call(Token.new(data || "", @group, :region_close))
+        end
+
+        def flush_chunk
+          @callback.call(Token.new(@chunk, @group)) unless @chunk.empty?
+          @chunk = ""
+        end
+
+        def subtokenize(syntax, text)
+          tokenizer = Syntax.load(syntax)
+          tokenizer.set @options if @options
+          flush_chunk
+          tokenizer.tokenize(text, &@callback)
+        end
+      end
+
+      # A default tokenizer for handling syntaxes that are not explicitly handled
+      # elsewhere. It simply yields the given text as a single token.
+      class Default
+        # Yield the given text as a single token.
+        def tokenize(text)
+          yield Token.new(text, :normal)
+        end
+      end
+
+      # A hash for registering syntax implementations.
+      SYNTAX = Hash.new(Default)
+
+      # Load the implementation of the requested syntax. If the syntax cannot be
+      # found, or if it cannot be loaded for whatever reason, the Default syntax
+      # handler will be returned.
+      def load(syntax)
+        begin
+          require_relative "lang/#{syntax}"
+        rescue LoadError
+        end
+        SYNTAX[syntax].new
+      end
+      module_function :load
+
+      # Return an array of the names of supported syntaxes.
+      def all
+        lang_dir = File.join(File.dirname(__FILE__), "syntax", "lang")
+        Dir["#{lang_dir}/*.rb"].map { |path| File.basename(path, ".rb") }
+      end
+      module_function :all
+    end
+  end
+end

data/lib/shoes/highlighter/lang/ruby.rb

@@ -0,0 +1,316 @@
+class Shoes
+  module Highlighter
+    module Syntax
+      # A tokenizer for the Ruby language. It recognizes all common syntax
+      # (and some less common syntax) but because it is not a true lexer, it
+      # will make mistakes on some ambiguous cases.
+      class Ruby < Tokenizer
+        # The list of all identifiers recognized as keywords.
+        KEYWORDS =
+          %w(if then elsif else end begin do rescue ensure while for
+             class module def yield raise until unless and or not when
+             case super undef break next redo retry in return alias
+             defined?)
+
+        # Perform ruby-specific setup
+        def setup
+          @selector = false
+          @allow_operator = false
+          @heredocs = []
+        end
+
+        # Step through a single iteration of the tokenization process.
+        def step
+          case
+          when bol? && check(/=begin/)
+            start_group(:comment, scan_until(/^=end#{EOL}/))
+          when bol? && check(/__END__#{EOL}/)
+            start_group(:comment, scan_until(/\Z/))
+          else
+            case
+            when check(/def\s+/)
+              start_group :keyword, scan(/def\s+/)
+              start_group :method,  scan_until(/(?=[;(\s]|#{EOL})/)
+            when check(/class\s+/)
+              start_group :keyword, scan(/class\s+/)
+              start_group :class,  scan_until(/(?=[;\s<]|#{EOL})/)
+            when check(/module\s+/)
+              start_group :keyword, scan(/module\s+/)
+              start_group :module,  scan_until(/(?=[;\s]|#{EOL})/)
+            when check(/::/)
+              start_group :punct, scan(/::/)
+            when check(/:"/)
+              start_group :symbol, scan(/:/)
+              scan_delimited_region :symbol, :symbol, "", true
+              @allow_operator = true
+            when check(/:'/)
+              start_group :symbol, scan(/:/)
+              scan_delimited_region :symbol, :symbol, "", false
+              @allow_operator = true
+            when scan(/:[_a-zA-Z@$][$@\w]*[=!?]?/)
+              start_group :symbol, matched
+              @allow_operator = true
+            when scan(/\?(\\[^\n\r]|[^\\\n\r\s])/)
+              start_group :char, matched
+              @allow_operator = true
+            when check(/(__FILE__|__LINE__|true|false|nil|self)[?!]?/)
+              if @selector || matched[-1] == '?' || matched[-1] == '!'
+                start_group :ident,
+                            scan(/(__FILE__|__LINE__|true|false|nil|self)[?!]?/)
+              else
+                start_group :constant,
+                            scan(/(__FILE__|__LINE__|true|false|nil|self)/)
+              end
+              @selector = false
+              @allow_operator = true
+            when scan(/0([bB][01]+|[oO][0-7]+|[dD][0-9]+|[xX][0-9a-fA-F]+)/)
+              start_group :number, matched
+              @allow_operator = true
+            else
+              case peek(2)
+              when "%r"
+                scan_delimited_region :punct, :regex, scan(/../), true
+                @allow_operator = true
+              when "%w", "%q"
+                scan_delimited_region :punct, :string, scan(/../), false
+                @allow_operator = true
+              when "%s"
+                scan_delimited_region :punct, :symbol, scan(/../), false
+                @allow_operator = true
+              when "%W", "%Q", "%x"
+                scan_delimited_region :punct, :string, scan(/../), true
+                @allow_operator = true
+              when /%[^\sa-zA-Z0-9]/
+                scan_delimited_region :punct, :string, scan(/./), true
+                @allow_operator = true
+              when "<<"
+                saw_word = (chunk[-1, 1] =~ /[\w!?]/)
+                start_group :punct, scan(/<</)
+                if saw_word
+                  @allow_operator = false
+                  return
+                end
+
+                float_right = scan(/-/)
+                append "-" if float_right
+                if (type = scan(/['"]/))
+                  append type
+                  delim = scan_until(/(?=#{type})/)
+                  if delim.nil?
+                    append scan_until(/\Z/)
+                    return
+                  end
+                else
+                  delim = scan(/\w+/) or return
+                end
+                start_group :constant, delim
+                start_group :punct, scan(/#{type}/) if type
+                @heredocs << [float_right, type, delim]
+                @allow_operator = true
+              else
+                case peek(1)
+                when /[\n\r]/
+                  unless @heredocs.empty?
+                    scan_heredoc(*@heredocs.shift)
+                  else
+                    start_group :normal, scan(/\s+/)
+                  end
+                  @allow_operator = false
+                when /\s/
+                  start_group :normal, scan(/\s+/)
+                when "#"
+                  start_group :comment, scan(/#[^\n\r]*/)
+                when /[A-Z]/
+                  start_group @selector ? :ident : :constant, scan(/\w+/)
+                  @allow_operator = true
+                when /[a-z_]/
+                  word = scan(/\w+[?!]?/)
+                  if !@selector && KEYWORDS.include?(word)
+                    start_group :keyword, word
+                    @allow_operator = false
+                  elsif
+                    start_group :ident, word
+                    @allow_operator = true
+                  end
+                  @selector = false
+                when /\d/
+                  start_group :number,
+                              scan(/[\d_]+(\.[\d_]+)?([eE][\d_]+)?/)
+                  @allow_operator = true
+                when '"'
+                  scan_delimited_region :punct, :string, "", true
+                  @allow_operator = true
+                when '/'
+                  if @allow_operator
+                    start_group :punct, scan(%r{/})
+                    @allow_operator = false
+                  else
+                    scan_delimited_region :punct, :regex, "", true
+                    @allow_operator = true
+                  end
+                when "'"
+                  scan_delimited_region :punct, :string, "", false
+                  @allow_operator = true
+                when "."
+                  dots = scan(/\.{1,3}/)
+                  start_group :punct, dots
+                  @selector = (dots.length == 1)
+                when /[@]/
+                  start_group :attribute, scan(/@{1,2}\w*/)
+                  @allow_operator = true
+                when /[$]/
+                  start_group :global, scan(/\$/)
+                  start_group :global, scan(/\w+|./) if check(/./)
+                  @allow_operator = true
+                when /[-!?*\/+=<>(\[\{}:;,&|%]/
+                  start_group :punct, scan(/./)
+                  @allow_operator = false
+                when /[)\]]/
+                  start_group :punct, scan(/./)
+                  @allow_operator = true
+                else
+                  # all else just falls through this, to prevent
+                  # infinite loops...
+                  append getch
+                end
+              end
+            end
+          end
+        end
+
+        private
+
+        # Scan a delimited region of text. This handles the simple cases (strings
+        # delimited with quotes) as well as the more complex cases of %-strings
+        # and here-documents.
+        #
+        # * +delim_group+ is the group to use to classify the delimiters of the
+        #   region
+        # * +inner_group+ is the group to use to classify the contents of the
+        #   region
+        # * +starter+ is the text to use as the starting delimiter
+        # * +exprs+ is a boolean flag indicating whether the region is an
+        #   interpolated string or not
+        # * +delim+ is the text to use as the delimiter of the region. If +nil+,
+        #   the next character will be treated as the delimiter.
+        # * +heredoc+ is either +false+, meaning the region is not a heredoc, or
+        #   <tt>:flush</tt> (meaning the delimiter must be flushed left), or
+        #   <tt>:float</tt> (meaning the delimiter doens't have to be flush left).
+        def scan_delimited_region(delim_group, inner_group, starter, exprs,
+                                  delim = nil, heredoc = false)
+          # begin
+          unless delim
+            start_group delim_group, starter
+            delim = scan(/./)
+            append delim
+
+            delim = case delim
+                    when '{' then '}'
+                    when '(' then ')'
+                    when '[' then ']'
+                    when '<' then '>'
+                    else delim
+                    end
+          end
+
+          start_region inner_group
+
+          items = "\\\\|"
+          if heredoc
+            items << "(^"
+            items << '\s*' if heredoc == :float
+            items << "#{Regexp.escape(delim)}\s*?)#{EOL}"
+          else
+            items << "#{Regexp.escape(delim)}"
+          end
+          items << "|#(\\$|@@?|\\{)" if exprs
+          items = Regexp.new(items)
+
+          loop do
+            p = pos
+            match = scan_until(items)
+            if match.nil?
+              start_group inner_group, scan_until(/\Z/)
+              break
+            else
+              text = pre_match[p..-1]
+              start_group inner_group, text if text.length > 0
+              case matched.strip
+              when "\\"
+                unless exprs
+                  case peek(1)
+                  when "'"
+                    scan(/./)
+                    start_group :escape, "\\'"
+                  when "\\"
+                    scan(/./)
+                    start_group :escape, "\\\\"
+                  else
+                    start_group inner_group, "\\"
+                  end
+                else
+                  start_group :escape, "\\"
+                  c = getch
+                  append c
+                  case c
+                  when 'x'
+                    append scan(/[a-fA-F0-9]{1,2}/)
+                  when /[0-7]/
+                    append scan(/[0-7]{0,2}/)
+                  end
+                end
+              when delim
+                end_region inner_group
+                start_group delim_group, matched
+                break
+              when /^#/
+                do_highlight = (option(:expressions) == :highlight)
+                start_region :expr if do_highlight
+                start_group :expr, matched
+                case matched[1]
+                when '{'
+                  depth = 1
+                  content = ""
+                  while depth > 0
+                    p = pos
+                    c = scan_until(/[\{}]/)
+                    if c.nil?
+                      content << scan_until(/\Z/)
+                      break
+                    else
+                      depth += (matched == "{" ? 1 : -1)
+                      content << pre_match[p..-1]
+                      content << matched if depth > 0
+                    end
+                  end
+                  if do_highlight
+                    subtokenize "ruby", content
+                    start_group :expr, "}"
+                  else
+                    append content + "}"
+                  end
+                when '$', '@'
+                  append scan(/\w+/)
+                end
+                end_region :expr if do_highlight
+              else fail "unexpected match on #{matched}"
+              end
+            end
+          end
+        end
+
+        # Scan a heredoc beginning at the current position.
+        #
+        # * +float+ indicates whether the delimiter may be floated to the right
+        # * +type+ is +nil+, a single quote, or a double quote
+        # * +delim+ is the delimiter to look for
+        def scan_heredoc(float, type, delim)
+          scan_delimited_region(:constant, :string, "", type != "'",
+                                delim, float ? :float : :flush)
+        end
+      end
+
+      SYNTAX["ruby"] = Ruby
+    end
+  end
+end

data/lib/shoes/highlighter/markup.rb

@@ -0,0 +1,209 @@
+# syntax highlighting
+
+class Shoes
+  module Highlighter
+    module Markup
+      TOKENIZER = Shoes::Highlighter::Syntax.load "ruby"
+      COLORS = {
+        comment: { stroke: "#887" },
+        keyword: { stroke: "#111" },
+        method: { stroke: "#C09", weight: "bold" },
+        # :class => {:stroke => "#0c4", :weight => "bold"},
+        # :module => {:stroke => "#050"},
+        # :punct => {:stroke => "#668", :weight => "bold"},
+        symbol: { stroke: "#C30" },
+        string: { stroke: "#C90" },
+        number: { stroke: "#396" },
+        regex: { stroke: "#000", fill: "#FFC" },
+        # :char => {:stroke => "#f07"},
+        attribute: { stroke: "#369" },
+        # :global => {:stroke => "#7FB" },
+        expr: { stroke: "#722" },
+        # :escape => {:stroke => "#277" }
+        ident: { stroke: "#994c99" },
+        constant: { stroke: "#630", weight: "bold" },
+        class: { stroke: "#630", weight: "bold" },
+        matching: { stroke: "#ff0", weight: "bold" },
+      }
+
+      def highlight(str, pos = nil, colors = COLORS)
+        tokens = []
+        TOKENIZER.tokenize(str) do |t|
+          if t.group == :punct
+            # split punctuation into single characters tokens
+            # TODO: to it in the parser
+            tokens += t.split('').map { |s| Shoes::Highlighter::Syntax::Token.new(s, :punct) }
+          else
+            # add token as is
+            tokens << t
+          end
+        end
+
+        res = []
+        tokens.each do |token|
+          res <<
+            if colors[token.group]
+              # span(token, colors[token.group])
+              tmp = fg(token, colors[token.group][:stroke])
+              colors[token.group][:fill] ? bg(tmp, colors[token.group][:fill]) : tmp
+            elsif colors[:any]
+              # span(token, colors[:any])
+              tmp = fg(token, colors[:any][:stroke])
+              colors[:any][:fill] ? bg(tmp, colors[:any][:fill]) : tmp
+            else
+              token
+            end
+        end
+
+        if pos.nil? || pos < 0
+          return res
+        end
+
+        token_index, matching_index = matching_token(tokens, pos)
+
+        if token_index
+          # res[token_index] = span(tokens[token_index], colors[:matching])
+          tmp = fg(tokens[token_index], colors[:matching][:stroke])
+          res[token_index] = colors[:matching][:fill] ? bg(tmp, colors[:matching][:fill]) : tmp
+          if matching_index
+            # res[matching_index] = span(tokens[matching_index], colors[:matching])
+            tmp = fg(tokens[matching_index], colors[:matching][:stroke])
+            res[matching_index] = colors[:matching][:fill] ? bg(tmp, colors[:matching][:fill]) : tmp
+          end
+        end
+
+        res
+      end
+
+      private
+
+      def matching_token(tokens, pos)
+        curr_pos = 0
+        token_index = nil
+        tokens.each_with_index do |t, i|
+          curr_pos += t.size
+          if token_index.nil? && curr_pos >= pos
+            token_index = i
+            break
+          end
+        end
+        if token_index.nil? then return nil end
+
+        match = matching_token_at_index(tokens, token_index)
+        if match.nil? && curr_pos == pos && token_index < tokens.size - 1
+          # try the token before the cursor, instead of the one after
+          token_index += 1
+          match = matching_token_at_index(tokens, token_index)
+        end
+
+        if match
+          [token_index, match]
+        else
+          nil
+        end
+      end
+
+      def matching_token_at_index(tokens, index)
+        starts, ends, direction = matching_tokens(tokens, index)
+        if starts.nil?
+          return nil
+        end
+
+        stack_level = 1
+        index += direction
+        while index >= 0 && index < tokens.size
+          # TODO separate space in the tokenizer
+          t = tokens[index].gsub(/\s/, '')
+          if ends.include?(t) && !as_modifier?(tokens, index)
+            stack_level -= 1
+            return index if stack_level == 0
+          elsif starts.include?(t) && !as_modifier?(tokens, index)
+            stack_level += 1
+          end
+          index += direction
+        end
+        # no matching token found
+        nil
+      end
+
+      # returns an array of tokens matching and the direction
+      def matching_tokens(tokens, index)
+        # TODO separate space in the tokenizer
+        token = tokens[index].gsub(/\s/, '')
+        starts = [token]
+        if OPEN_BRACKETS[token]
+          direction = 1
+          ends = [OPEN_BRACKETS[token]]
+        elsif CLOSE_BRACKETS[token]
+          direction = -1
+          ends = [CLOSE_BRACKETS[token]]
+        elsif OPEN_BLOCK.include?(token)
+          if as_modifier?(tokens, index)
+            return nil
+          end
+          direction = 1
+          ends = ['end']
+          starts = OPEN_BLOCK
+        elsif token == 'end'
+          direction = -1
+          ends = OPEN_BLOCK
+        else
+          return nil
+        end
+
+        [starts, ends, direction]
+      end
+
+      def as_modifier?(tokens, index)
+        unless MODIFIERS.include? tokens[index].gsub(/\s/, '')
+          return false
+        end
+
+        index -= 1
+        # find last index before the token that is no space
+        index -= 1 while index >= 0 && tokens[index] =~ /\A[ \t]*\z/
+
+        if index < 0
+          # first character of the string
+          false
+        elsif tokens[index] =~ /\n[ \t]*\Z/
+          # first token of the line
+          false
+        elsif tokens[index].group == :punct
+          # preceded by a punctuation token on the same line
+          i = tokens[index].rindex(/\S/)
+          punc = tokens[index][i, 1]
+          # true if the preceeding statement was terminating
+          !NON_TERMINATING.include?(punc)
+        else
+          # preceded by a non punctuation token on the same line
+          true
+        end
+      end
+
+      OPEN_BRACKETS = {
+        '{' => '}',
+        '(' => ')',
+        '[' => ']',
+      }
+
+      # close_bracket = {}
+      # OPEN_BRACKETS.each{|open, close| opens_bracket[close] = open}
+      # CLOSE_BRACKETS = opens_bracket
+      # the following is more readable :)
+      CLOSE_BRACKETS = {
+        '}' => '{',
+        ')' => '(',
+        ']' => '[',
+      }
+
+      BRACKETS = CLOSE_BRACKETS.keys + OPEN_BRACKETS.keys
+
+      OPEN_BLOCK = %w(def class module do if unless while until begin for)
+
+      MODIFIERS = %w(if unless while until)
+
+      NON_TERMINATING = %w{+ - * / , . = ~ < > ( [}
+    end
+  end
+end

data/lib/shoes/highlighter/version.rb

@@ -0,0 +1,5 @@
+class Shoes
+  module Highlighter
+    VERSION = "1.0.0"
+  end
+end

data/shoes-highlighter.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'shoes/highlighter/version'
+
+Gem::Specification.new do |s|
+  s.name          = "shoes-highlighter"
+  s.version     = Shoes::Highlighter::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Team Shoes"]
+  s.email       = ["shoes@librelist.com"]
+  s.homepage    = "https://github.com/shoes/shoes4"
+  s.summary     = 'A syntax highlighting library used by Shoes'
+  s.description = 'A syntax highlighting library used by Shoes. Originally extracted from Hackety-Hack.'
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split($/)
+  s.test_files    = s.files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "bundler", "~> 1.7"
+  s.add_development_dependency "rake", "~> 10.0"
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: shoes-highlighter
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Team Shoes
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: A syntax highlighting library used by Shoes. Originally extracted from
+  Hackety-Hack.
+email:
+- shoes@librelist.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/shoes/highlighter.rb
+- lib/shoes/highlighter/common.rb
+- lib/shoes/highlighter/lang/ruby.rb
+- lib/shoes/highlighter/markup.rb
+- lib/shoes/highlighter/version.rb
+- shoes-highlighter.gemspec
+homepage: https://github.com/shoes/shoes4
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: A syntax highlighting library used by Shoes
+test_files: []