lkml 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0d77c017023caf25f3f6acadeff2cdc098d1ddbb6c75b8ca9f0ac7968abcefef
-  data.tar.gz: 6613bd618a1793463394f7e4b0408eb30467fb435da58735d61b51fc00d1a22e
+  metadata.gz: 947467843616af2d9823cae9e566dd7c7125f7d5eed824b00f23593d05c9874b
+  data.tar.gz: 17aca72ede33a42f05be5ce902db30a755842bb19eb51dddc6ef0cb7bbf19016
 SHA512:
-  metadata.gz: 668606773d646b80de95eef27aa09d060018a6b5018c4c7d2982e8ad23bc13257437272af13b5284b55d72f2c6ef2f72ff99fdf56e610aad07b87087524ef037
-  data.tar.gz: dc231523581f3a6125b9a94af667adeb2c6c4211143f8c614655b2da42b3efff6e23cc2b18df05dfc8c9ceacb8768a4a66b1309e3181454c65a8e07f0df71747
+  metadata.gz: 10ac65ed0b899f40360bf307b92fb7521d41cb53eeeac34f5b2b9293b91cd07c950251a1288f21c76b0aa6a5d6cb19cbc1b69d9b78a54fd7713e72533124d733
+  data.tar.gz: 48de5ca911b5344afd31d31d6c469e7b333b269b572b88a0a740d27133f35f26f93523866a6f912e2728a88969cf1383b08fc0ac88d1e2a6b23e01b90f87db41

data/.github/workflows/ci.yml

@@ -0,0 +1,28 @@
+name: CI
+
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [ "3.3", "3.4", "4.0" ]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Lint with RuboCop
+        run: bundle exec rubocop
+
+      - name: Run tests
+        run: bundle exec rake test
+        env:
+          LKML_RUN_GITHUB_TESTS: 1

data/.gitignore

@@ -0,0 +1,57 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+.rspec_status
+
+# Used by dotenv library to load environment variables.
+.env
+
+# Ignore Byebug command history file.
+.byebug_history
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+/docs/api/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+Gemfile.lock
+gemfiles/*.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+

data/.rubocop.yml

@@ -0,0 +1,25 @@
+AllCops:
+  NewCops: enable
+  TargetRubyVersion: 3.3
+  SuggestExtensions: false
+
+# Library code does not use RDoc-style class/module comments.
+Style/Documentation:
+  Enabled: false
+
+# Annotated format tokens add noise for debug logging and printf-style benchmarks.
+Style/FormatStringToken:
+  Enabled: false
+
+# Dev dependencies are declared in the gemspec for this gem.
+Gemspec/DevelopmentDependencies:
+  Enabled: false
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120
+
+Metrics:
+  Enabled: false

data/.ruby-version

@@ -0,0 +1 @@
+3.3.6

data/CONTRIBUTING.md

@@ -0,0 +1,56 @@
+# Contributing to lkml
+
+## Local setup
+
+- Fork the repository and clone your fork
+- Create a branch from the default branch (e.g. `feature/your-change`)
+- Install dependencies:
+
+```bash
+bundle install
+```
+
+## Tests
+
+Run the full suite:
+
+```bash
+bundle exec rake test
+```
+
+Add or update tests under `test/` using Minitest (`test/test_*.rb`). Keep fixtures in `test/resources/` as needed.
+
+### GitHub sample acceptance tests
+
+A large set of LookML files lives under `test/resources/github/`. Tests in `test/test_github.rb` are **skipped by default**. Enable them with:
+
+```bash
+LKML_RUN_GITHUB_TESTS=1 bundle exec ruby -Itest test/test_github.rb
+```
+
+### Downloading fresh samples from GitHub
+
+`script/download_lookml.rb` queries the GitHub API. Set:
+
+- `GITHUB_USERNAME`
+- `GITHUB_PERSONAL_ACCESS_TOKEN`
+
+Then:
+
+```bash
+ruby script/download_lookml.rb
+```
+
+Files are written to `test/resources/github/`.
+
+## Style
+
+RuboCop is configured for this repo:
+
+```bash
+bundle exec rubocop
+```
+
+## Pull requests
+
+Open a PR against the main repository’s default branch. Reference related issues; use closing keywords (e.g. `Closes #19`) when appropriate.

data/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,76 @@
+PATH
+  remote: .
+  specs:
+    lkml (1.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.3)
+    json (2.19.4)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    minitest (5.27.0)
+    parallel (2.0.1)
+    parser (3.3.11.1)
+      ast (~> 2.4.1)
+      racc
+    prism (1.9.0)
+    racc (1.8.1)
+    rainbow (3.1.1)
+    rake (13.4.2)
+    regexp_parser (2.12.0)
+    rubocop (1.86.1)
+      json (~> 2.3)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
+      parallel (>= 1.10)
+      parser (>= 3.3.0.2)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.49.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
+    ruby-progressbar (1.13.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+
+PLATFORMS
+  arm64-darwin-25
+  ruby
+
+DEPENDENCIES
+  lkml!
+  logger
+  minitest (~> 5.25)
+  rake (~> 13.2)
+  rubocop (~> 1.69)
+
+CHECKSUMS
+  ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
+  json (2.19.4) sha256=670a7d333fb3b18ca5b29cb255eb7bef099e40d88c02c80bd42a3f30fe5239ac
+  language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
+  lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
+  lkml (1.2.0)
+  logger (1.7.0) sha256=196edec7cc44b66cfb40f9755ce11b392f21f7967696af15d274dde7edff0203
+  minitest (5.27.0) sha256=2d3b17f8a36fe7801c1adcffdbc38233b938eb0b4966e97a6739055a45fa77d5
+  parallel (2.0.1) sha256=337782d3e39f4121e67563bf91dd8ece67f48923d90698614773a0ec9a5b2c7d
+  parser (3.3.11.1) sha256=d17ace7aabe3e72c3cc94043714be27cc6f852f104d81aa284c2281aecc65d54
+  prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
+  racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
+  rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a
+  rake (13.4.2) sha256=cb825b2bd5f1f8e91ca37bddb4b9aaf345551b4731da62949be002fa89283701
+  regexp_parser (2.12.0) sha256=35a916a1d63190ab5c9009457136ae5f3c0c7512d60291d0d1378ba18ce08ebb
+  rubocop (1.86.1) sha256=44415f3f01d01a21e01132248d2fd0867572475b566ca188a0a42133a08d4531
+  rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035
+  ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
+  unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
+  unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
+
+BUNDLED WITH
+  4.0.7

data/LICENSE.md

@@ -1,7 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Josh Temple
-Copyright (c) 2025 Sylvain Utard
+Copyright (c) 2019 Josh Temple, 2026 Altertable
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

data/README.md

@@ -1,11 +1,85 @@
-# Lkml
+# lkml
 
-A LookML parser and serializer implemented in pure Ruby.
+A fast LookML parser and serializer implemented in **pure Ruby** (stdlib only).
 
-> This is a Ruby rewrite of the amazing [joshtemple/lkml](https://github.com/joshtemple/lkml) python library.
+`Lkml.load` parses LookML text into Ruby hashes (with string keys, suitable for JSON). `Lkml.dump` / `Lkml.generate` serialize hashes back to LookML.
 
-Why should you use `lkml`?
+Why use **lkml**?
 
-- Tested on **over 160K lines of LookML** from public repositories on GitHub
-- Written in pure, modern Ruby with **no external dependencies**
-- A **full unit test suite** with excellent coverage
+- Exercised on a large corpus of real-world LookML (including bundled GitHub samples)
+- Parses typical view/model files in a few milliseconds (CPU time, excluding I/O)
+- No runtime gem dependencies
+- Full Minitest suite including lexer, parser, round-trip, and CLI checks
+
+## Install
+
+```bash
+gem install lkml
+```
+
+Or add to your `Gemfile`:
+
+```ruby
+gem "lkml"
+```
+
+Requires Ruby **3.3+**.
+
+## Usage
+
+```ruby
+require "lkml"
+
+hash = Lkml.load('view: orders { sql_table_name: public.orders ;; }')
+# => {"views"=>[{"name"=>"orders", "sql_table_name"=>"public.orders"}]}
+
+lookml = Lkml.dump(hash)
+# or
+lookml = Lkml.generate(hash)
+```
+
+Parse to the concrete syntax tree instead of a hash:
+
+```ruby
+tree = Lkml.parse("view: x { dimension: id { type: number sql: ${TABLE}.id ;; } }")
+tree.to_s # round-trip string
+```
+
+### CLI
+
+```bash
+lkml path/to/file.view.lkml
+```
+
+Pretty-prints JSON.
+
+Options:
+
+- `-v` / `--verbose` — debug logging on stderr
+
+## Development
+
+```bash
+bundle install
+bundle exec rake test
+```
+
+Optional acceptance run over bundled GitHub samples (slow):
+
+```bash
+LKML_RUN_GITHUB_TESTS=1 bundle exec ruby -Itest test/test_github.rb
+```
+
+Benchmark (CPU time over `test/resources/github/*.lkml`):
+
+```bash
+ruby script/benchmark.rb
+```
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md).
+
+## License
+
+MIT — see [LICENSE.md](LICENSE.md).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+task default: :test

data/bin/lkml

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative "../lib/lkml/cli"
+
+Lkml::CLI.run(ARGV)

data/lib/lkml/cli.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "json"
+require "logger"
+require "optparse"
+
+require_relative "../lkml"
+
+module Lkml
+  module CLI
+    module_function
+
+    def parse_args(argv)
+      options = { log_level: Logger::WARN }
+      parser = OptionParser.new do |opts|
+        opts.banner = "usage: lkml [-v|--verbose] FILE"
+
+        opts.on("-v", "--verbose", "Increase logging verbosity to debug") do
+          options[:log_level] = Logger::DEBUG
+        end
+      end
+      parser.parse!(argv)
+
+      raise OptionParser::MissingArgument, "FILE is required" if argv.empty?
+
+      options[:file_path] = argv[0]
+      options
+    end
+
+    def run(argv = ARGV)
+      logger = Logger.new($stderr)
+      logger.level = Logger::WARN
+
+      opts = parse_args(argv.dup)
+
+      logger.level = opts[:log_level]
+
+      File.open(opts[:file_path], "r:UTF-8") do |file|
+        result = Lkml.load(file)
+        puts JSON.pretty_generate(result)
+      end
+    end
+  end
+end

data/lib/lkml/keys.rb

@@ -1,125 +1,75 @@
 # frozen_string_literal: true
 
-# Defines constant sequences of LookML keys and helper methods.
-
-# These are repeatable keys in LookML that the parser should collapse into a single
-# Ruby hash key. For example, LookML can have multiple dimensions, so the parser
-# will combine those dimensions into an array of hashes with a top-level key,
-# `dimensions`.
+require_relative "tokens"
 
 module Lkml
-  PLURAL_KEYS = %w[
-    access_filter
-    access_grant
-    action
-    aggregate_table
-    allowed_value
-    assert
-    bind_filters
-    column
-    constant
-    datagroup
-    remote_dependency
-    derived_column
-    dimension
-    dimension_group
-    explore
-    extends
-    filter
-    filters
-    form_param
-    include
-    join
-    link
-    map_layer
-    measure
-    named_value_format
-    option
-    override_constant
-    param
-    parameter
-    query
-    set
-    sql_step
-    test
-    user_attribute_param
-    view
-    when
-  ].freeze
+  module Keys
+    PLURAL_KEYS = %w[
+      access_filter access_grant action aggregate_table allowed_value assert bind_filters
+      column constant datagroup remote_dependency local_dependency derived_column dimension
+      dimension_group explore extends filter filters form_param include join link map_layer
+      measure named_value_format option override_constant param parameter query set sql_step
+      test user_attribute_param view when
+    ].freeze
+
+    EXPR_BLOCK_KEYS = %w[
+      expression_custom_filter expression html sql_trigger_value sql_table_name
+      sql_distinct_key sql_start sql_always_having sql_always_where sql_trigger
+      sql_foreign_key sql_where sql_end sql_create sql_latitude sql_longitude sql_step
+      sql_on sql sql_preamble
+    ].freeze
 
-  # These are keys in LookML that should be recognized as expression blocks (end with ;;).
+    QUOTED_LITERAL_KEYS = %w[
+      label view_label group_label group_item_label suggest_persist_for default_value
+      direction value_format name url icon_url form_url default tags value description
+      sortkeys indexes partition_keys connection include max_cache_age allowed_values
+      timezone persist_for cluster_keys distribution extents_json_url feature_key file
+      property_key property_label_key else interval_trigger
+    ].freeze
 
-  EXPR_BLOCK_KEYS = %w[
-    expression_custom_filter
-    expression
-    html
-    sql_trigger_value
-    sql_table_name
-    sql_distinct_key
-    sql_start
-    sql_always_having
-    sql_always_where
-    sql_trigger
-    sql_foreign_key
-    sql_where
-    sql_end
-    sql_create
-    sql_latitude
-    sql_longitude
-    sql_step
-    sql_on
-    sql
-    sql_preamble
-  ].freeze
+    KEYS_WITH_NAME_FIELDS = %w[user_attribute_param param form_param option].freeze
 
-  # These are keys that the serializer should quote the value of (e.g. `label: "Label"`).
-  # An example of an unquoted literal would be `hidden: no`.
+    CHARACTER_TO_TOKEN = {
+      "\0" => Tokens::StreamEndToken,
+      "{" => Tokens::BlockStartToken,
+      "}" => Tokens::BlockEndToken,
+      "[" => Tokens::ListStartToken,
+      "]" => Tokens::ListEndToken,
+      "," => Tokens::CommaToken,
+      ":" => Tokens::ValueToken,
+      ";" => Tokens::ExpressionBlockEndToken
+    }.freeze
 
-  QUOTED_LITERAL_KEYS = %w[
-    label
-    view_label
-    group_label
-    group_item_label
-    suggest_persist_for
-    default_value
-    direction
-    value_format
-    name
-    url
-    icon_url
-    form_url
-    default
-    tags
-    value
-    description
-    sortkeys
-    indexes
-    partition_keys
-    connection
-    include
-    max_cache_age
-    allowed_values
-    timezone
-    persist_for
-    cluster_keys
-    distribution
-    extents_json_url
-    feature_key
-    file
-    property_key
-    property_label_key
-    else
-    interval_trigger
-  ].freeze
+    module_function
 
-  # These are keys for fields in Looker that have a "name" attribute. Since lkml uses the
-  # key `name` to represent the name of the field (e.g. for `dimension: dimension_name {`,
-  # the `name` key would hold the value `dimension_name`.)
+    def pluralize(key)
+      case key
+      when "filters", "bind_filters", "extends"
+        "#{key}__all"
+      when "query"
+        "queries"
+      when "remote_dependency"
+        "remote_dependencies"
+      when "local_dependency"
+        "local_dependencies"
+      else
+        "#{key}s"
+      end
+    end
 
-  KEYS_WITH_NAME_FIELDS = %w[
-    user_attribute_param
-    param
-    form_param
-    option
-  ].freeze
+    def singularize(key)
+      case key
+      when "queries"
+        "query"
+      when "remote_dependencies"
+        "remote_dependency"
+      when "local_dependencies"
+        "local_dependency"
+      when /__all\z/
+        key[0..-6]
+      else
+        key.end_with?("s") ? key.sub(/s+\z/, "") : key
+      end
+    end
+  end
 end