lkml 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0d77c017023caf25f3f6acadeff2cdc098d1ddbb6c75b8ca9f0ac7968abcefef
+  data.tar.gz: 6613bd618a1793463394f7e4b0408eb30467fb435da58735d61b51fc00d1a22e
+SHA512:
+  metadata.gz: 668606773d646b80de95eef27aa09d060018a6b5018c4c7d2982e8ad23bc13257437272af13b5284b55d72f2c6ef2f72ff99fdf56e610aad07b87087524ef037
+  data.tar.gz: dc231523581f3a6125b9a94af667adeb2c6c4211143f8c614655b2da42b3efff6e23cc2b18df05dfc8c9ceacb8768a4a66b1309e3181454c65a8e07f0df71747

data/LICENSE.md

@@ -0,0 +1,10 @@
+MIT License
+
+Copyright (c) 2019 Josh Temple
+Copyright (c) 2025 Sylvain Utard
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,11 @@
+# Lkml
+
+A LookML parser and serializer implemented in pure Ruby.
+
+> This is a Ruby rewrite of the amazing [joshtemple/lkml](https://github.com/joshtemple/lkml) python library.
+
+Why should you use `lkml`?
+
+- Tested on **over 160K lines of LookML** from public repositories on GitHub
+- Written in pure, modern Ruby with **no external dependencies**
+- A **full unit test suite** with excellent coverage

data/lib/lkml/keys.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+# Defines constant sequences of LookML keys and helper methods.
+
+# These are repeatable keys in LookML that the parser should collapse into a single
+# Ruby hash key. For example, LookML can have multiple dimensions, so the parser
+# will combine those dimensions into an array of hashes with a top-level key,
+# `dimensions`.
+
+module Lkml
+  PLURAL_KEYS = %w[
+    access_filter
+    access_grant
+    action
+    aggregate_table
+    allowed_value
+    assert
+    bind_filters
+    column
+    constant
+    datagroup
+    remote_dependency
+    derived_column
+    dimension
+    dimension_group
+    explore
+    extends
+    filter
+    filters
+    form_param
+    include
+    join
+    link
+    map_layer
+    measure
+    named_value_format
+    option
+    override_constant
+    param
+    parameter
+    query
+    set
+    sql_step
+    test
+    user_attribute_param
+    view
+    when
+  ].freeze
+
+  # These are keys in LookML that should be recognized as expression blocks (end with ;;).
+
+  EXPR_BLOCK_KEYS = %w[
+    expression_custom_filter
+    expression
+    html
+    sql_trigger_value
+    sql_table_name
+    sql_distinct_key
+    sql_start
+    sql_always_having
+    sql_always_where
+    sql_trigger
+    sql_foreign_key
+    sql_where
+    sql_end
+    sql_create
+    sql_latitude
+    sql_longitude
+    sql_step
+    sql_on
+    sql
+    sql_preamble
+  ].freeze
+
+  # These are keys that the serializer should quote the value of (e.g. `label: "Label"`).
+  # An example of an unquoted literal would be `hidden: no`.
+
+  QUOTED_LITERAL_KEYS = %w[
+    label
+    view_label
+    group_label
+    group_item_label
+    suggest_persist_for
+    default_value
+    direction
+    value_format
+    name
+    url
+    icon_url
+    form_url
+    default
+    tags
+    value
+    description
+    sortkeys
+    indexes
+    partition_keys
+    connection
+    include
+    max_cache_age
+    allowed_values
+    timezone
+    persist_for
+    cluster_keys
+    distribution
+    extents_json_url
+    feature_key
+    file
+    property_key
+    property_label_key
+    else
+    interval_trigger
+  ].freeze
+
+  # These are keys for fields in Looker that have a "name" attribute. Since lkml uses the
+  # key `name` to represent the name of the field (e.g. for `dimension: dimension_name {`,
+  # the `name` key would hold the value `dimension_name`.)
+
+  KEYS_WITH_NAME_FIELDS = %w[
+    user_attribute_param
+    param
+    form_param
+    option
+  ].freeze
+end

data/lib/lkml/lexer.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require_relative 'keys'
+require_relative 'tokens'
+
+# Splits a LookML string into a sequence of tokens.
+module Lkml
+  class Lexer
+    attr_reader :text, :index, :tokens, :line_number
+
+    CHARACTER_TO_TOKEN = {
+      "\0" => Tokens::StreamEndToken,
+      '{' => Tokens::BlockStartToken,
+      '}' => Tokens::BlockEndToken,
+      '[' => Tokens::ListStartToken,
+      ']' => Tokens::ListEndToken,
+      ',' => Tokens::CommaToken,
+      ':' => Tokens::ValueToken,
+      ';' => Tokens::ExpressionBlockEndToken
+    }.freeze
+
+    def initialize(text)
+      # Initializes the Lexer with a LookML string and sets the index.
+      @text = "#{text}\u0000"
+      @index = 0
+      @tokens = []
+      @line_number = 1
+    end
+
+    def peek
+      # Returns the character at the current index of the text being lexed.
+      @text[@index]
+    end
+
+    def peek_multiple(length)
+      # Returns the next n characters from the current index in the text being lexed.
+      @text[@index, length]
+    end
+
+    def advance(length = 1)
+      # Moves the index forward by n characters.
+      @index += length
+      nil
+    end
+
+    def consume
+      # Returns the current index character and advances the index 1 character.
+      advance
+      @text[@index - 1]
+    end
+
+    def scan # rubocop:disable Metrics/CyclomaticComplexity
+      # Tokenizes LookML into a sequence of tokens.
+      @tokens << Tokens::StreamStartToken.new(@line_number)
+      loop do
+        ch = peek
+        case ch
+        when "\0"
+          @tokens << CHARACTER_TO_TOKEN[ch].new(@line_number)
+          break
+        when "\n", "\t", ' '
+          @tokens << scan_whitespace
+        when '#'
+          advance
+          @tokens << scan_comment
+        when ';'
+          if peek_multiple(2) == ';;'
+            advance(2)
+            @tokens << CHARACTER_TO_TOKEN[ch].new(@line_number)
+          end
+        when '"'
+          advance
+          @tokens << scan_quoted_literal
+        when *CHARACTER_TO_TOKEN.keys
+          advance
+          @tokens << CHARACTER_TO_TOKEN[ch].new(@line_number)
+        else
+          if self.class.check_for_expression_block(peek_multiple(25))
+            # TODO: Handle edges here with whitespace and comments
+            @tokens << scan_literal
+            advance
+            @tokens << Tokens::ValueToken.new(@line_number)
+            @tokens << scan_expression_block
+          else
+            # TODO: This should actually check for valid literals first
+            # and throw an error if it doesn't match
+            @tokens << scan_literal
+          end
+        end
+      end
+      @tokens
+    end
+
+    def self.check_for_expression_block(string)
+      # Returns true if the input string is an expression block.
+      EXPR_BLOCK_KEYS.any? { |key| string.start_with?("#{key}:") }
+    end
+
+    def scan_whitespace
+      # Returns a token from one or more whitespace characters.
+      chars = ''
+      next_char = peek
+      while ["\n", "\t", ' '].include?(next_char)
+        if next_char == "\n"
+          while next_char == "\n"
+            chars += consume
+            @line_number += 1
+            next_char = peek
+          end
+          return Tokens::LinebreakToken.new(chars, @line_number)
+        else
+          chars += consume
+          next_char = peek
+        end
+      end
+      Tokens::InlineWhitespaceToken.new(chars, @line_number)
+    end
+
+    def scan_comment
+      # Returns a token from a comment.
+      chars = '#'
+      chars += consume until ["\0", "\n"].include?(peek)
+      Tokens::CommentToken.new(chars, @line_number)
+    end
+
+    def scan_expression_block
+      # Returns a token from an expression block string.
+      chars = ''
+      while peek_multiple(2) != ';;'
+        @line_number += 1 if peek == "\n"
+        chars += consume
+      end
+      Tokens::ExpressionBlockToken.new(chars, @line_number)
+    end
+
+    def scan_literal
+      # Returns a token from a literal string.
+      chars = ''
+      chars += consume until ["\0", ' ', "\n", "\t", ':', '}', '{', ',', ']'].include?(peek)
+      Tokens::LiteralToken.new(chars, @line_number)
+    end
+
+    def scan_quoted_literal
+      # Returns a token from a quoted literal string.
+      chars = ''
+      loop do
+        ch = peek
+        break if ch == '"'
+
+        if ch == '\\'
+          chars += consume # Extra consume to skip the escaped character
+        elsif ch == "\n"
+          @line_number += 1
+        end
+        chars += consume
+      end
+      advance
+      Tokens::QuotedLiteralToken.new(chars, @line_number)
+    end
+  end
+end

data/lib/lkml/parser.rb

@@ -0,0 +1,338 @@
+# frozen_string_literal: true
+
+# Parses a sequence of tokenized LookML into a parse tree.
+
+require_relative 'tokens'
+require_relative 'tree'
+
+module Lkml
+  class CommaSeparatedValues
+    attr_accessor :trailing_comma, :leading_comma
+
+    def initialize
+      @_values = []
+      @trailing_comma = nil
+      @leading_comma = nil
+    end
+
+    def append(value)
+      @_values << value
+    end
+
+    def values
+      @_values
+    end
+  end
+
+  class Parser
+    attr_accessor :tokens, :index, :progress, :depth, :log_debug
+
+    def initialize(stream)
+      stream.each { |token| raise TypeError, "Unsupported token: #{token}" unless token.is_a?(Tokens::Token) }
+      @tokens = stream
+      @index = 0
+      @progress = 0
+      @depth = -1
+    end
+
+    def jump_to_index(index)
+      @index = index
+    end
+
+    def peek
+      @tokens[@index]
+    end
+
+    def advance(length = 1)
+      @index += length
+      nil
+    end
+
+    def consume
+      advance
+      @tokens[@index - 1]
+    end
+
+    def consume_token_value
+      token = consume
+      raise "Token #{token} does not have a consumable value." if token.value.nil?
+
+      token.value
+    end
+
+    def consume_trivia(only_newlines: false)
+      valid_tokens = [Tokens::CommentToken]
+      valid_tokens += only_newlines ? [Tokens::LinebreakToken] : [Tokens::WhitespaceToken]
+
+      trivia = ''
+      loop do
+        break unless check(*valid_tokens)
+
+        trivia += consume_token_value
+      end
+      trivia
+    end
+
+    def check(*token_types, skip_trivia: false)
+      mark = @index
+      consume_trivia if skip_trivia
+
+      token = begin
+        peek
+      rescue StandardError
+        nil
+      end
+      result = token_types.any? { |type| token.is_a?(type) }
+
+      jump_to_index(mark) if skip_trivia
+      result
+    end
+
+    def parse
+      advance if check(Tokens::StreamStartToken)
+      prefix = consume_trivia
+      container = parse_container
+      suffix = consume_trivia
+      DocumentNode.new(container, prefix: prefix, suffix: suffix)
+    end
+
+    def parse_container
+      items = []
+      until check(Tokens::StreamEndToken, Tokens::BlockEndToken, skip_trivia: true)
+        block = parse_block
+        if block
+          items << block
+          next
+        end
+
+        pair = parse_pair
+        if pair
+          items << pair
+          next
+        end
+
+        list = parse_list
+        if list
+          items << list
+          next
+        end
+
+        token = @tokens[@progress]
+        raise SyntaxError, "Unable to find a matching expression for '#{token}' on line #{token.line_number}"
+      end
+
+      ContainerNode.new(items, top_level: @depth.zero?)
+    end
+
+    def parse_block
+      key = parse_key
+      return key if key.nil?
+
+      name = if check(Tokens::LiteralToken)
+               token = consume
+               SyntaxToken.new(token.value, token.line_number)
+             end
+
+      prefix = consume_trivia
+      return nil unless check(Tokens::BlockStartToken)
+
+      advance
+      suffix = consume_trivia
+      left_brace = LeftCurlyBrace.new(prefix: prefix, suffix: suffix)
+
+      container = parse_container
+
+      prefix = consume_trivia
+      return unless check(Tokens::BlockEndToken)
+
+      advance
+      suffix = consume_trivia(only_newlines: true)
+      right_brace = RightCurlyBrace.new(prefix: prefix, suffix: suffix)
+
+      BlockNode.new(
+        key[0],
+        colon: key[1],
+        name: name,
+        left_brace: left_brace,
+        container: container,
+        right_brace: right_brace
+      )
+    end
+
+    def parse_pair
+      key = parse_key
+      return nil if key.nil?
+
+      value = parse_value(parse_prefix: true, parse_suffix: true)
+      return nil if value.nil?
+
+      PairNode.new(key[0], value, colon: key[1])
+    end
+
+    def parse_key
+      prefix = consume_trivia
+      return nil unless check(Tokens::LiteralToken)
+
+      token = consume
+      key = SyntaxToken.new(token.value, token.line_number, prefix: prefix)
+
+      prefix = consume_trivia
+
+      colon = nil
+      while check(Tokens::ValueToken)
+        token = consume
+        suffix = consume_trivia
+        colon = Colon.new(token.line_number, prefix: prefix, suffix: suffix)
+      end
+      return nil unless colon
+
+      [key, colon]
+    end
+
+    def parse_value(parse_prefix: false, parse_suffix: false) # rubocop:disable Metrics/CyclomaticComplexity
+      prefix = parse_prefix ? consume_trivia : ''
+
+      if check(Tokens::LiteralToken)
+        token = consume
+        if token.value == '-' && consume_trivia
+          return nil unless check(Tokens::LiteralToken)
+
+          token = consume
+          token.value = "-#{token.value}"
+
+        end
+        suffix = parse_suffix ? consume_trivia : ''
+        SyntaxToken.new(token.value, token.line_number, prefix: prefix, suffix: suffix)
+      elsif check(Tokens::QuotedLiteralToken)
+        token = consume
+        suffix = parse_suffix ? consume_trivia : ''
+        QuotedSyntaxToken.new(token.value, token.line_number, prefix: prefix, suffix: suffix)
+      elsif check(Tokens::ExpressionBlockToken)
+        token = consume
+        match = token.value.match(/\A(\s*)(.*?)(\s*)\z/)
+        expr_prefix, value, expr_suffix = if match
+                                            [match[1], match[2], match[3]]
+                                          else
+                                            ['', token.value, '']
+                                          end
+        prefix += expr_prefix
+
+        return nil unless check(Tokens::ExpressionBlockEndToken)
+
+        advance
+
+        suffix = parse_suffix ? consume_trivia : ''
+        ExpressionSyntaxToken.new(value, token.line_number, prefix: prefix, suffix: suffix, expr_suffix: expr_suffix)
+      end
+    end
+
+    def parse_list
+      key = parse_key
+      return key if key.nil?
+
+      prefix = consume_trivia
+      return nil unless check(Tokens::ListStartToken)
+
+      advance
+      left_bracket = LeftBracket.new(prefix: prefix)
+
+      csv = parse_csv || CommaSeparatedValues.new
+
+      return unless check(Tokens::ListEndToken, skip_trivia: true)
+
+      prefix = consume_trivia
+      advance
+      suffix = consume_trivia
+      right_bracket = RightBracket.new(prefix: prefix, suffix: suffix)
+      ListNode.new(
+        key[0],
+        items: csv.values,
+        left_bracket: left_bracket,
+        right_bracket: right_bracket,
+        colon: key[1],
+        leading_comma: csv.leading_comma,
+        trailing_comma: csv.trailing_comma
+      )
+    end
+
+    def parse_csv # rubocop:disable Metrics/CyclomaticComplexity
+      pair_mode = false
+      csv = CommaSeparatedValues.new
+      csv.leading_comma = parse_comma
+
+      pair = parse_pair
+      if pair
+        csv.append(pair)
+        pair_mode = true
+      elsif check(Tokens::LiteralToken, Tokens::QuotedLiteralToken, skip_trivia: true)
+        value = parse_value(parse_prefix: true, parse_suffix: true)
+        csv.append(value)
+      else
+        return nil
+      end
+
+      until check(Tokens::ListEndToken, skip_trivia: true)
+        return nil unless check(Tokens::CommaToken)
+
+        index = @index
+        advance
+        if check(Tokens::ListEndToken, skip_trivia: true)
+          jump_to_index(index)
+          csv.trailing_comma = parse_comma
+          break
+        end
+
+        if pair_mode
+          pair = parse_pair
+          return nil if pair.nil?
+
+          csv.append(pair)
+        elsif check(Tokens::LiteralToken, Tokens::QuotedLiteralToken, skip_trivia: true)
+          value = parse_value(parse_prefix: true, parse_suffix: true)
+          csv.append(value)
+        elsif check(Tokens::ListEndToken, skip_trivia: true)
+          break
+        else
+          return nil
+        end
+      end
+
+      csv
+    end
+
+    def parse_comma
+      prefix = consume_trivia
+
+      return unless check(Tokens::CommaToken)
+
+      advance
+      suffix = check(Tokens::ListEndToken, skip_trivia: true) ? '' : consume_trivia
+      Comma.new(prefix: prefix, suffix: suffix)
+    end
+
+    def self.backtrack_if_none(method_name)
+      original_method = instance_method(method_name)
+
+      define_method(method_name) do |*args, **kwargs|
+        mark = @index
+        @depth += 1
+        result = original_method.bind(self).call(*args, **kwargs)
+        @depth -= 1
+        if result.nil?
+          @progress = [@index, @progress].max
+          jump_to_index(mark)
+        end
+        result
+      end
+    end
+
+    backtrack_if_none :parse_container
+    backtrack_if_none :parse_block
+    backtrack_if_none :parse_pair
+    backtrack_if_none :parse_key
+    backtrack_if_none :parse_value
+    backtrack_if_none :parse_list
+    backtrack_if_none :parse_csv
+    backtrack_if_none :parse_comma
+  end
+end