rux 1.0.0

data/lib/rux/lexer.rb

@@ -0,0 +1,64 @@
+module Rux
+  class Lexer
+    class EOFError < StandardError; end
+    class TransitionError < StandardError; end
+
+    attr_reader :source_buffer
+
+    def initialize(source_buffer)
+      @source_buffer = source_buffer
+      @stack = [RubyLexer.new(source_buffer, 0)]
+      @generator = to_enum(:each_token)
+    end
+
+    def advance
+      @generator.next
+    rescue StopIteration
+      [nil, ['$eof']]
+    end
+
+    private
+
+    def each_token
+      @p = 0
+
+      while true
+        begin
+          token = current.advance
+        rescue StopIteration
+          # This error means the current lexer has run it's course and should
+          # be considered finished. The lexer should have already yielded a
+          # :tRESET token to position the previous lexer (i.e. the one
+          # logically before it in the stack) at the place it left off.
+          @stack.pop
+          break unless current  # no current lexer means we're done
+          current.reset_to(@p)
+          next
+        end
+
+        type, (_, pos) = token
+        break unless pos
+
+        unless type
+          @stack.push(current.next_lexer(pos.begin_pos))
+          next
+        end
+
+        case type
+          when :tRESET
+            @p = pos.begin_pos
+          when :tSKIP
+            next
+          else
+            yield token
+
+            @p = pos.end_pos
+        end
+      end
+    end
+
+    def current
+      @stack.last
+    end
+  end
+end

data/lib/rux/parser.rb

@@ -0,0 +1,244 @@
+require 'parser'
+
+module Rux
+  class Parser
+    class UnexpectedTokenError < StandardError; end
+    class TagMismatchError < StandardError; end
+
+    class << self
+      def parse_file(path)
+        buffer = ::Parser::Source::Buffer.new(path).read
+        lexer = ::Rux::Lexer.new(buffer)
+        new(lexer).parse
+      end
+
+      def parse(str)
+        buffer = ::Parser::Source::Buffer.new('(source)', source: str)
+        lexer = ::Rux::Lexer.new(buffer)
+        new(lexer).parse
+      end
+    end
+
+    def initialize(lexer)
+      @lexer = lexer
+      @stack = []
+      @current = get_next
+    end
+
+    def parse
+      curlies = 1
+      children = []
+
+      loop do
+        type = type_of(current)
+        break unless type
+
+        case type
+          when :tLCURLY, :tLBRACE, :tRUX_LITERAL_RUBY_CODE_START
+            curlies += 1
+          when :tRCURLY, :tRBRACE, :tRUX_LITERAL_RUBY_CODE_END
+            curlies -= 1
+        end
+
+        break if curlies == 0
+
+        if rb = ruby
+          children << rb
+        elsif type_of(current) == :tRUX_TAG_OPEN_START
+          children << tag
+        else
+          raise UnexpectedTokenError,
+            'expected ruby code or the start of a rux tag but found '\
+              "#{type_of(current)} instead"
+        end
+      end
+
+      AST::ListNode.new(children)
+    end
+
+    private
+
+    def ruby
+      ruby_start = pos_of(current).begin_pos
+
+      loop do
+        type = type_of(current)
+
+        if type.nil? || RuxLexer.state_table.include?(type_of(current))
+          break
+        end
+
+        consume(type_of(current))
+      end
+
+      unless type_of(current)
+        return AST::RubyNode.new(
+          @lexer.source_buffer.source[ruby_start..-1]
+        )
+      end
+
+      if pos_of(current).begin_pos != ruby_start
+        AST::RubyNode.new(
+          @lexer.source_buffer.source[ruby_start...(pos_of(current).end_pos - 1)]
+        )
+      end
+    end
+
+    def tag
+      consume(:tRUX_TAG_OPEN_START)
+      tag_name = text_of(current)
+      tag_pos = pos_of(current)
+      consume(:tRUX_TAG_OPEN, :tRUX_TAG_SELF_CLOSING)
+      maybe_consume(:tRUX_ATTRIBUTE_SPACES)
+      attrs = attributes
+      maybe_consume(:tRUX_ATTRIBUTE_SPACES)
+      maybe_consume(:tRUX_TAG_OPEN_END)
+      tag_node = AST::TagNode.new(tag_name, attrs)
+
+      if is?(:tRUX_TAG_SELF_CLOSING_END)
+        consume(:tRUX_TAG_SELF_CLOSING_END)
+        return tag_node
+      end
+
+      @stack.push(tag_name)
+
+      until is?(:tRUX_TAG_CLOSE_START)
+        if is?(:tRUX_LITERAL, :tRUX_LITERAL_RUBY_CODE_START)
+          lit = literal
+          tag_node.children << lit if lit
+        else
+          tag_node.children << tag
+        end
+      end
+
+      consume(:tRUX_TAG_CLOSE_START)
+
+      closing_tag_name = text_of(current)
+
+      if @stack.last != closing_tag_name
+        closing_tag_pos = pos_of(current)
+
+        raise TagMismatchError, "closing tag '#{closing_tag_name}' on line "\
+          "#{closing_tag_pos.line} did not match opening tag '#{tag_name}' "\
+          "on line #{tag_pos.line}"
+      end
+
+      @stack.pop
+
+      consume(:tRUX_TAG_CLOSE)
+      consume(:tRUX_TAG_CLOSE_END)
+
+      tag_node
+    end
+
+    def attributes
+      {}.tap do |attrs|
+        while is?(:tRUX_ATTRIBUTE_NAME)
+          key, value = attribute
+          attrs[key] = value
+
+          maybe_consume(:tRUX_ATTRIBUTE_SPACES)
+        end
+      end
+    end
+
+    def attribute
+      maybe_consume(:tRUX_ATTRIBUTE_SPACES)
+      attr_name = text_of(current)
+      consume(:tRUX_ATTRIBUTE_NAME)
+      maybe_consume(:tRUX_ATTRIBUTE_EQUALS_SPACES)
+
+      attr_value = if maybe_consume(:tRUX_ATTRIBUTE_EQUALS)
+        maybe_consume(:tRUX_ATTRIBUTE_VALUE_SPACES)
+        attribute_value
+      else
+        # if no equals sign, assume boolean attribute
+        AST::StringNode.new("\"true\"")
+      end
+
+      [attr_name, attr_value]
+    end
+
+    def attribute_value
+      if is?(:tRUX_ATTRIBUTE_VALUE_RUBY_CODE_START)
+        attr_ruby_code
+      else
+        AST::StringNode.new(text_of(current)).tap do
+          consume(:tRUX_ATTRIBUTE_VALUE)
+        end
+      end
+    end
+
+    def attr_ruby_code
+      consume(:tRUX_ATTRIBUTE_VALUE_RUBY_CODE_START)
+
+      ruby.tap do
+        consume(:tRUX_ATTRIBUTE_VALUE_RUBY_CODE_END)
+      end
+    end
+
+    def literal
+      if is?(:tRUX_LITERAL_RUBY_CODE_START)
+        literal_ruby_code
+      else
+        lit = squeeze_lit(text_of(current))
+        consume(:tRUX_LITERAL)
+        AST::TextNode.new(lit) unless lit.empty?
+      end
+    end
+
+    def squeeze_lit(lit)
+      lit.gsub(/\s/, ' ').squeeze(' ')
+    end
+
+    def literal_ruby_code
+      consume(:tRUX_LITERAL_RUBY_CODE_START)
+
+      parse.tap do |res|
+        consume(:tRUX_LITERAL_RUBY_CODE_END)
+      end
+    end
+
+    def current
+      @current
+    end
+
+    def is?(*types)
+      types.include?(type_of(current))
+    end
+
+    def maybe_consume(type)
+      if type_of(current) == type
+        @current = get_next
+        true
+      else
+        false
+      end
+    end
+
+    def consume(*types)
+      if !types.include?(type_of(current))
+        raise UnexpectedTokenError,
+          "expected [#{types.map(&:to_s).join(', ')}], got '#{type_of(current)}'"
+      end
+
+      @current = get_next
+    end
+
+    def type_of(token)
+      token[0]
+    end
+
+    def text_of(token)
+      token[1][0]
+    end
+
+    def pos_of(token)
+      token[1][1]
+    end
+
+    def get_next
+      @lexer.advance
+    end
+  end
+end

data/lib/rux/ruby_lexer.rb

@@ -0,0 +1,143 @@
+module Rux
+  class RubyLexer < ::Parser::Lexer
+    # These are populated when ::Parser::Lexer loads and are therefore
+    # not inherited. We have to copy them over manually.
+    ::Parser::Lexer.instance_variables.each do |ivar|
+      instance_variable_set(ivar, ::Parser::Lexer.instance_variable_get(ivar))
+    end
+
+    LOOKAHEAD = 3
+
+    def initialize(source_buffer, init_pos)
+      super(ruby_version)
+
+      self.source_buffer = source_buffer
+      @generator = to_enum(:each_token)
+      @rux_token_queue = []
+      @p = init_pos
+    end
+
+    alias_method :advance_orig, :advance
+
+    def advance
+      @generator.next
+    end
+
+    def reset_to(pos)
+      @ts = @te = @p = pos
+      @eof = false
+      @rux_token_queue.clear
+      populate_queue
+    end
+
+    def next_lexer(pos)
+      RuxLexer.new(@source_buffer, pos)
+    end
+
+    private
+
+    def ruby_version
+      @ruby_version ||= RUBY_VERSION
+        .split('.')[0..-2]
+        .join('')
+        .to_i
+    end
+
+    def each_token(&block)
+      # We detect whether or not we're at the beginning of a rux tag by looking
+      # ahead by 1 token; that's why the first element in @rux_token_queue is
+      # yielded immediately. If the lexer _starts_ at a rux tag however,
+      # lookahead is a lot more difficult. To mitigate, we insert a dummy skip
+      # token here. That way, at_rux? checks the right tokens in the queue and
+      # correctly identifies the start of a rux tag.
+      @rux_token_queue << [:tSKIP, ['$skip', make_range(@p, @p)]]
+
+      @eof = false
+      curlies = 1
+      populate_queue
+
+      until @rux_token_queue.empty?
+        if at_rux?
+          yield @rux_token_queue.shift
+
+          @eof = true
+          _, (_, pos) = @rux_token_queue[0]
+
+          # @eof is set to false by reset_to above, which is called after
+          # popping the previous lexer off the lexer stack (see lexer.rb)
+          while @eof
+            yield [nil, ['$eof', pos]]
+          end
+        end
+
+        token = @rux_token_queue.shift
+        type, (_, pos) = token
+
+        case type
+          when :tLCURLY, :tLBRACE
+            curlies += 1
+          when :tRCURLY, :tRBRACE
+            curlies -= 1
+        end
+
+        # if curlies are balanced, we're done lexing ruby code, so yield a
+        # reset token to tell the system where we stopped, then break to stop
+        # our enumerator (will raise a StopIteration)
+        if curlies == 0
+          yield [:tRESET, ['$eof', pos]]
+          break
+        end
+
+        yield token
+
+        populate_queue
+      end
+    end
+
+    def populate_queue
+      until @rux_token_queue.size >= LOOKAHEAD
+        begin
+          cur_token = advance_orig
+        rescue NoMethodError
+          # Internal lexer errors can happen since we're asking the ruby lexer
+          # to start at an arbitrary position inside the source buffer. It may
+          # encounter foreign rux tokens it's not expecting, etc. Best to stop
+          # trying to look ahead and call it quits.
+          break
+        end
+
+        break unless cur_token[0]
+        @rux_token_queue << cur_token
+      end
+    end
+
+    def at_rux?
+      at_lt? && !at_inheritance?
+    end
+
+    def at_lt?
+      is?(@rux_token_queue[1], :tLT) && (
+        is?(@rux_token_queue[2], :tCONSTANT) ||
+        is?(@rux_token_queue[2], :tIDENTIFIER)
+      )
+    end
+
+    def at_inheritance?
+      is?(@rux_token_queue[0], :tCONSTANT) &&
+        is?(@rux_token_queue[1], :tLT) &&
+        is?(@rux_token_queue[2], :tCONSTANT)
+    end
+
+    def is?(tok, sym)
+      tok && tok[0] == sym
+    end
+
+    def is_not?(tok, sym)
+      tok && tok[0] != sym
+    end
+
+    def make_range(start, stop)
+      ::Parser::Source::Range.new(@source_buffer, start, stop)
+    end
+  end
+end