lkml 0.1.0 → 0.2.0

data/lib/lkml/parser.rb

@@ -1,26 +1,34 @@
 # frozen_string_literal: true
 
-# Parses a sequence of tokenized LookML into a parse tree.
+require "logger"
 
-require_relative 'tokens'
-require_relative 'tree'
+require_relative "tokens"
+require_relative "tree"
+require_relative "utils"
 
 module Lkml
+  unless const_defined?(:AttributeError)
+    class AttributeError < StandardError
+    end
+  end
+
+  DELIMITER = ". "
+
   class CommaSeparatedValues
-    attr_accessor :trailing_comma, :leading_comma
+    attr_accessor :values, :trailing_comma, :leading_comma
 
     def initialize
-      @_values = []
+      @values = []
       @trailing_comma = nil
       @leading_comma = nil
     end
 
     def append(value)
-      @_values << value
+      @values << value
     end
 
-    def values
-      @_values
+    def frozen_tuple
+      @values.dup.freeze
     end
   end
 
@@ -28,15 +36,22 @@ module Lkml
     attr_accessor :tokens, :index, :progress, :depth, :log_debug
 
     def initialize(stream)
-      stream.each { |token| raise TypeError, "Unsupported token: #{token}" unless token.is_a?(Tokens::Token) }
+      stream.each do |token|
+        raise TypeError, "Type #{token.class} for #{token} is not a valid token type." unless token.is_a?(Tokens::Token)
+      end
       @tokens = stream
       @index = 0
       @progress = 0
       @depth = -1
+      @logger = Logger.new($stderr)
+      @logger.level = Logger::WARN
+      @log_debug = false
     end
 
-    def jump_to_index(index)
-      @index = index
+    attr_reader :logger
+
+    def jump_to_index(idx)
+      @index = idx
     end
 
     def peek
@@ -55,18 +70,21 @@ module Lkml
 
     def consume_token_value
       token = consume
-      raise "Token #{token} does not have a consumable value." if token.value.nil?
+      raise AttributeError unless token.is_a?(Tokens::ContentToken)
 
       token.value
     end
 
     def consume_trivia(only_newlines: false)
-      valid_tokens = [Tokens::CommentToken]
-      valid_tokens += only_newlines ? [Tokens::LinebreakToken] : [Tokens::WhitespaceToken]
-
-      trivia = ''
+      valid = if only_newlines
+                [Tokens::CommentToken, Tokens::LinebreakToken]
+              else
+                [Tokens::CommentToken,
+                 Tokens::WhitespaceToken]
+              end
+      trivia = +""
       loop do
-        break unless check(*valid_tokens)
+        break unless check(*valid)
 
         trivia += consume_token_value
       end
@@ -77,262 +95,349 @@ module Lkml
       mark = @index
       consume_trivia if skip_trivia
 
+      if @log_debug
+        names = token_types.map(&:name).join(" or ")
+        tok = peek_safe
+        @logger.debug(format("%sCheck %s == %s", delimiter_repeat(1 + @depth), tok.inspect, names))
+      end
+
+      token_types.each do |token_type|
+        unless token_type <= Tokens::Token || token_type == Tokens::Token
+          raise TypeError, "#{token_type} is not a valid token type."
+        end
+      end
+
       token = begin
-        peek
-      rescue StandardError
+        @tokens.fetch(@index)
+      rescue IndexError
         nil
       end
-      result = token_types.any? { |type| token.is_a?(type) }
-
+      result = token && token_types.any? { |t| token.is_a?(t) }
       jump_to_index(mark) if skip_trivia
       result
     end
 
+    def peek_safe
+      @tokens.fetch(@index)
+    end
+
     def parse
       advance if check(Tokens::StreamStartToken)
       prefix = consume_trivia
       container = parse_container
       suffix = consume_trivia
-      DocumentNode.new(container, prefix: prefix, suffix: suffix)
+      Tree::DocumentNode.new(container, prefix, suffix)
     end
 
     def parse_container
-      items = []
-      until check(Tokens::StreamEndToken, Tokens::BlockEndToken, skip_trivia: true)
-        block = parse_block
-        if block
-          items << block
-          next
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[expression] = (block / pair / list)*"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
         end
+        items = []
+        until check(Tokens::StreamEndToken, Tokens::BlockEndToken, skip_trivia: true)
+          block = parse_block
+          if block
+            items << block
+            next
+          end
 
-        pair = parse_pair
-        if pair
-          items << pair
-          next
-        end
+          pair = parse_pair
+          if pair
+            items << pair
+            next
+          end
+
+          list = parse_list
+          if list
+            items << list
+            next
+          end
+
+          token = @tokens[@progress]
+          raise SyntaxError, "Unable to find a matching expression for '#{token.id}' " \
+                             "on line #{token.line_number}"
 
-        list = parse_list
-        if list
-          items << list
-          next
         end
 
-        token = @tokens[@progress]
-        raise SyntaxError, "Unable to find a matching expression for '#{token}' on line #{token.line_number}"
+        Tree::ContainerNode.new(items: items.freeze, top_level: @depth.zero?)
       end
-
-      ContainerNode.new(items, top_level: @depth.zero?)
     end
 
     def parse_block
-      key = parse_key
-      return key if key.nil?
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[block] = key literal? '{' expression '}'"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
+        end
 
-      name = if check(Tokens::LiteralToken)
-               token = consume
-               SyntaxToken.new(token.value, token.line_number)
-             end
+        key = parse_key
+        next unless key
 
-      prefix = consume_trivia
-      return nil unless check(Tokens::BlockStartToken)
+        name = nil
+        if check(Tokens::LiteralToken)
+          token = consume
+          name = Tree::SyntaxToken.new(token.value, token.line_number)
+        end
 
-      advance
-      suffix = consume_trivia
-      left_brace = LeftCurlyBrace.new(prefix: prefix, suffix: suffix)
+        prefix = consume_trivia
+        next nil unless check(Tokens::BlockStartToken)
 
-      container = parse_container
+        advance
+        suffix = consume_trivia
+        left_brace = Tree::LeftCurlyBrace.new("{", nil, prefix, suffix)
 
-      prefix = consume_trivia
-      return unless check(Tokens::BlockEndToken)
+        container = parse_container
 
-      advance
-      suffix = consume_trivia(only_newlines: true)
-      right_brace = RightCurlyBrace.new(prefix: prefix, suffix: suffix)
-
-      BlockNode.new(
-        key[0],
-        colon: key[1],
-        name: name,
-        left_brace: left_brace,
-        container: container,
-        right_brace: right_brace
-      )
+        prefix = consume_trivia
+        next nil unless check(Tokens::BlockEndToken)
+
+        advance
+        suffix = consume_trivia(only_newlines: true)
+        right_brace = Tree::RightCurlyBrace.new("}", nil, prefix, suffix)
+
+        blk = Tree::BlockNode.new(
+          type: key[0],
+          colon: key[1],
+          name: name,
+          left_brace: left_brace,
+          container: container,
+          right_brace: right_brace
+        )
+        @logger.debug("#{delimiter_repeat(@depth)}Successfully parsed block.") if @log_debug
+        blk
+      end
     end
 
     def parse_pair
-      key = parse_key
-      return nil if key.nil?
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[pair] = key value"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
+        end
+
+        key = parse_key
+        next unless key
 
-      value = parse_value(parse_prefix: true, parse_suffix: true)
-      return nil if value.nil?
+        value = parse_value(parse_prefix: true, parse_suffix: true)
+        next unless value
 
-      PairNode.new(key[0], value, colon: key[1])
+        Tree::PairNode.new(type: key[0], value: value, colon: key[1])
+      end
     end
 
     def parse_key
-      prefix = consume_trivia
-      return nil unless check(Tokens::LiteralToken)
-
-      token = consume
-      key = SyntaxToken.new(token.value, token.line_number, prefix: prefix)
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[key] = literal ':'"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
+        end
 
-      prefix = consume_trivia
+        prefix = consume_trivia
+        next nil unless check(Tokens::LiteralToken)
 
-      colon = nil
-      while check(Tokens::ValueToken)
         token = consume
-        suffix = consume_trivia
-        colon = Colon.new(token.line_number, prefix: prefix, suffix: suffix)
-      end
-      return nil unless colon
+        key = Tree::SyntaxToken.new(token.value, token.line_number, prefix)
+
+        prefix = consume_trivia
 
-      [key, colon]
+        colon = nil
+        while check(Tokens::ValueToken)
+          token = consume
+          suffix = consume_trivia
+          colon = Tree::Colon.new(":", token.line_number, prefix, suffix)
+        end
+        next unless colon
+
+        @logger.debug(format("%sSuccessfully parsed key.", delimiter_repeat(@depth))) if @log_debug
+
+        [key, colon]
+      end
     end
 
-    def parse_value(parse_prefix: false, parse_suffix: false) # rubocop:disable Metrics/CyclomaticComplexity
-      prefix = parse_prefix ? consume_trivia : ''
+    def parse_value(parse_prefix: false, parse_suffix: false)
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[value] = literal / quoted_literal / expression_block"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
+        end
 
-      if check(Tokens::LiteralToken)
-        token = consume
-        if token.value == '-' && consume_trivia
-          return nil unless check(Tokens::LiteralToken)
+        prefix = parse_prefix ? consume_trivia : ""
 
+        if check(Tokens::LiteralToken)
           token = consume
-          token.value = "-#{token.value}"
+          if token.value == "-" && !consume_trivia.empty?
+            next nil unless check(Tokens::LiteralToken)
 
-        end
-        suffix = parse_suffix ? consume_trivia : ''
-        SyntaxToken.new(token.value, token.line_number, prefix: prefix, suffix: suffix)
-      elsif check(Tokens::QuotedLiteralToken)
-        token = consume
-        suffix = parse_suffix ? consume_trivia : ''
-        QuotedSyntaxToken.new(token.value, token.line_number, prefix: prefix, suffix: suffix)
-      elsif check(Tokens::ExpressionBlockToken)
-        token = consume
-        match = token.value.match(/\A(\s*)(.*?)(\s*)\z/)
-        expr_prefix, value, expr_suffix = if match
-                                            [match[1], match[2], match[3]]
-                                          else
-                                            ['', token.value, '']
-                                          end
-        prefix += expr_prefix
+            token = consume
+            token.value = "-#{token.value}"
+
+          end
+          suffix = parse_suffix ? consume_trivia : ""
+          @logger.debug(format("%sSuccessfully parsed value.", delimiter_repeat(@depth))) if @log_debug
+          Tree::SyntaxToken.new(token.value, token.line_number, prefix, suffix)
+        elsif check(Tokens::QuotedLiteralToken)
+          token = consume
+          suffix = parse_suffix ? consume_trivia : ""
+          @logger.debug(format("%sSuccessfully parsed value.", delimiter_repeat(@depth))) if @log_debug
+          Tree::QuotedSyntaxToken.new(token.value, token.line_number, prefix, suffix)
+        elsif check(Tokens::ExpressionBlockToken)
+          token = consume
+          expr_prefix, value, expr_suffix = Utils.strip(token.value)
+          prefix += expr_prefix
 
-        return nil unless check(Tokens::ExpressionBlockEndToken)
+          next nil unless check(Tokens::ExpressionBlockEndToken)
 
-        advance
+          advance
 
-        suffix = parse_suffix ? consume_trivia : ''
-        ExpressionSyntaxToken.new(value, token.line_number, prefix: prefix, suffix: suffix, expr_suffix: expr_suffix)
+          suffix = parse_suffix ? consume_trivia : ""
+          @logger.debug(format("%sSuccessfully parsed value.", delimiter_repeat(@depth))) if @log_debug
+          Tree::ExpressionSyntaxToken.new(value, token.line_number, prefix, suffix, expr_suffix)
+        end
       end
     end
 
     def parse_list
-      key = parse_key
-      return key if key.nil?
-
-      prefix = consume_trivia
-      return nil unless check(Tokens::ListStartToken)
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[list] = key '[' csv? ']'"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
+        end
 
-      advance
-      left_bracket = LeftBracket.new(prefix: prefix)
+        key = parse_key
+        next unless key
 
-      csv = parse_csv || CommaSeparatedValues.new
+        prefix = consume_trivia
+        next nil unless check(Tokens::ListStartToken)
 
-      return unless check(Tokens::ListEndToken, skip_trivia: true)
+        advance
+        left_bracket = Tree::LeftBracket.new("[", nil, prefix, "")
 
-      prefix = consume_trivia
-      advance
-      suffix = consume_trivia
-      right_bracket = RightBracket.new(prefix: prefix, suffix: suffix)
-      ListNode.new(
-        key[0],
-        items: csv.values,
-        left_bracket: left_bracket,
-        right_bracket: right_bracket,
-        colon: key[1],
-        leading_comma: csv.leading_comma,
-        trailing_comma: csv.trailing_comma
-      )
-    end
+        csv = parse_csv
+        csv ||= CommaSeparatedValues.new
 
-    def parse_csv # rubocop:disable Metrics/CyclomaticComplexity
-      pair_mode = false
-      csv = CommaSeparatedValues.new
-      csv.leading_comma = parse_comma
+        next nil unless check(Tokens::ListEndToken, skip_trivia: true)
 
-      pair = parse_pair
-      if pair
-        csv.append(pair)
-        pair_mode = true
-      elsif check(Tokens::LiteralToken, Tokens::QuotedLiteralToken, skip_trivia: true)
-        value = parse_value(parse_prefix: true, parse_suffix: true)
-        csv.append(value)
-      else
-        return nil
+        prefix = consume_trivia
+        advance
+        suffix = consume_trivia
+        right_bracket = Tree::RightBracket.new("]", nil, prefix, suffix)
+
+        Tree::ListNode.new(
+          type: key[0],
+          colon: key[1],
+          left_bracket: left_bracket,
+          items: csv.frozen_tuple,
+          right_bracket: right_bracket,
+          leading_comma: csv.leading_comma,
+          trailing_comma: csv.trailing_comma
+        )
       end
+    end
 
-      until check(Tokens::ListEndToken, skip_trivia: true)
-        return nil unless check(Tokens::CommaToken)
-
-        index = @index
-        advance
-        if check(Tokens::ListEndToken, skip_trivia: true)
-          jump_to_index(index)
-          csv.trailing_comma = parse_comma
-          break
+    def parse_csv
+      backtrack_if_none do
+        if @log_debug
+          grammar = "[csv] = \",\"? (literal / quoted_literal) (\",\" (literal / quoted_literal))* \",\"?"
+          @logger.debug(format("%sTry to parse %s", delimiter_repeat(@depth), grammar))
         end
 
-        if pair_mode
-          pair = parse_pair
-          return nil if pair.nil?
+        pair_mode = false
+        csv = CommaSeparatedValues.new
+        csv.leading_comma = parse_comma
 
+        pair = parse_pair
+        if pair
           csv.append(pair)
+          pair_mode = true
         elsif check(Tokens::LiteralToken, Tokens::QuotedLiteralToken, skip_trivia: true)
           value = parse_value(parse_prefix: true, parse_suffix: true)
           csv.append(value)
-        elsif check(Tokens::ListEndToken, skip_trivia: true)
-          break
         else
-          return nil
+          next nil
+        end
+
+        # `next` inside `until` continues the until-loop (Ruby), not the outer backtrack block — use break + flag.
+        csv_abort = false
+        until check(Tokens::ListEndToken, skip_trivia: true)
+          unless check(Tokens::CommaToken)
+            csv_abort = true
+            break
+          end
+
+          comma_index = @index
+          advance
+          if check(Tokens::ListEndToken, skip_trivia: true)
+            jump_to_index(comma_index)
+            csv.trailing_comma = parse_comma
+            break
+          end
+
+          if pair_mode
+            pair = parse_pair
+            unless pair
+              csv_abort = true
+              break
+            end
+
+            csv.append(pair)
+          elsif check(Tokens::LiteralToken, Tokens::QuotedLiteralToken, skip_trivia: true)
+            value = parse_value(parse_prefix: true, parse_suffix: true)
+            csv.append(value)
+          elsif check(Tokens::ListEndToken, skip_trivia: true)
+            break
+          else
+            csv_abort = true
+            break
+          end
         end
-      end
 
-      csv
+        next nil if csv_abort
+
+        @logger.debug(format("%sSuccessfully parsed comma-separated values.", delimiter_repeat(@depth))) if @log_debug
+        csv
+      end
     end
 
     def parse_comma
-      prefix = consume_trivia
+      backtrack_if_none do
+        prefix = consume_trivia
+        next nil unless check(Tokens::CommaToken)
+
+        advance
+        suffix = if check(Tokens::ListEndToken, skip_trivia: true)
+                   ""
+                 else
+                   consume_trivia
+                 end
+        Tree::Comma.new(",", nil, prefix, suffix)
+      end
+    end
 
-      return unless check(Tokens::CommaToken)
+    private
 
-      advance
-      suffix = check(Tokens::ListEndToken, skip_trivia: true) ? '' : consume_trivia
-      Comma.new(prefix: prefix, suffix: suffix)
+    def delimiter_repeat(count)
+      n = Integer(count)
+      return +"" if n <= 0
+
+      DELIMITER * n
     end
 
-    def self.backtrack_if_none(method_name)
-      original_method = instance_method(method_name)
-
-      define_method(method_name) do |*args, **kwargs|
-        mark = @index
-        @depth += 1
-        result = original_method.bind(self).call(*args, **kwargs)
-        @depth -= 1
-        if result.nil?
-          @progress = [@index, @progress].max
-          jump_to_index(mark)
+    def backtrack_if_none
+      mark = @index
+      @depth += 1
+      result =
+        begin
+          yield
+        ensure
+          @depth -= 1
         end
-        result
+      if result.nil?
+        @progress = [@index, @progress].max
+        jump_to_index(mark)
       end
+      result
     end
-
-    backtrack_if_none :parse_container
-    backtrack_if_none :parse_block
-    backtrack_if_none :parse_pair
-    backtrack_if_none :parse_key
-    backtrack_if_none :parse_value
-    backtrack_if_none :parse_list
-    backtrack_if_none :parse_csv
-    backtrack_if_none :parse_comma
   end
 end