lkml 0.1.0 → 0.2.0

data/lib/lkml/utils.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Lkml
+  module Utils
+    module_function
+
+    def strip(str)
+      start = 0
+      end_idx = -1
+      nchars = str.length
+
+      start += 1 while start < nchars && str[start].match?(/\s/)
+
+      end_idx -= 1 while start != nchars && str[end_idx].match?(/\s/)
+
+      lspace = start.zero? ? "" : str[0, start]
+      rspace = if end_idx == -1
+                 ""
+               else
+                 str[(end_idx + 1)..] || ""
+               end
+
+      core = if end_idx == -1
+               str[start..] || ""
+             else
+               str[start..end_idx] || ""
+             end
+
+      [lspace, core, rspace]
+    end
+  end
+end

data/lib/lkml/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Lkml
-  VERSION = '0.1.0'
+  VERSION = "0.2.0"
 end

data/lib/lkml/visitors.rb

@@ -1,91 +1,96 @@
 # frozen_string_literal: true
 
-require 'logger'
-
-require_relative 'tree'
+require_relative "tree"
 
 module Lkml
-  class BasicVisitor < Visitor
-    def _visit(node)
-      if node.is_a?(SyntaxToken)
-        nil
-      elsif node.respond_to?(:children) && node.children
-        node.children.each { |child| child.accept(self) }
+  module Visitors
+    class BasicVisitor
+      include Tree::Visitor
+
+      def _visit(node)
+        case node
+        when Tree::SyntaxToken
+          nil
+        else
+          ch = node.respond_to?(:children) ? node.children : nil
+          ch&.each { |child| child.accept(self) }
+        end
       end
-    end
 
-    def visit(document)
-      _visit(document)
-    end
+      def visit(document)
+        _visit(document)
+      end
 
-    def visit_container(node)
-      _visit(node)
-    end
+      def visit_container(node)
+        _visit(node)
+      end
 
-    def visit_block(node)
-      _visit(node)
-    end
+      def visit_block(node)
+        _visit(node)
+      end
 
-    def visit_list(node)
-      _visit(node)
-    end
+      def visit_list(node)
+        _visit(node)
+      end
 
-    def visit_pair(node)
-      _visit(node)
-    end
+      def visit_pair(node)
+        _visit(node)
+      end
 
-    def visit_token(token)
-      _visit(token)
+      def visit_token(token)
+        _visit(token)
+      end
     end
-  end
 
-  class LookMlVisitor < BasicVisitor
-    def _visit(node)
-      node.to_s
+    class LookMlVisitor < BasicVisitor
+      def _visit(node)
+        node.to_s
+      end
     end
-  end
 
-  class BasicTransformer < Visitor
-    def _visit_items(node)
-      if node.respond_to?(:children) && node.children
-        new_children = node.children.map { |child| child.accept(self) }
-        node.class.new(items: new_children)
-      else
-        node
+    class BasicTransformer
+      include Tree::Visitor
+
+      def _visit_items(node)
+        ch = node.children
+        return node unless ch && !ch.empty?
+
+        new_children = ch.map { |child| child.accept(self) }
+        case node
+        when Tree::ContainerNode, Tree::ListNode
+          node.with(items: new_children.freeze)
+        else
+          node
+        end
       end
-    end
 
-    def _visit_container(node)
-      if node.respond_to?(:container) && node.container
-        new_child = node.container.accept(self)
-        node.class.new(container: new_child)
-      else
-        node
+      def visit(document)
+        return document unless document.container
+
+        document.with(container: document.container.accept(self))
       end
-    end
 
-    def visit(node)
-      _visit_container(node)
-    end
+      def visit_container(node)
+        _visit_items(node)
+      end
 
-    def visit_container(node)
-      _visit_items(node)
-    end
+      def visit_list(node)
+        _visit_items(node)
+      end
 
-    def visit_list(node)
-      _visit_items(node)
-    end
+      def visit_block(node)
+        return node unless node.container
 
-    def visit_block(node)
-      _visit_container(node)
-    end
+        node.with(container: node.container.accept(self))
+      end
 
-    def visit_pair(node)
-      node
-    end
+      def visit_pair(node)
+        node
+      end
 
-    def visit_token(token)
-      token
+      def visit_token(token)
+        token
+      end
     end
   end
 end

data/lib/lkml.rb

@@ -1,26 +1,61 @@
 # frozen_string_literal: true
 
-require_relative 'lkml/keys'
-require_relative 'lkml/lexer'
-require_relative 'lkml/parser'
-require_relative 'lkml/simple'
-require_relative 'lkml/tokens'
-require_relative 'lkml/tree'
-require_relative 'lkml/visitors'
+require_relative "lkml/version"
+require_relative "lkml/tokens"
+require_relative "lkml/keys"
+require_relative "lkml/utils"
+require_relative "lkml/tree"
+require_relative "lkml/visitors"
+require_relative "lkml/lexer"
+require_relative "lkml/parser"
+require "stringio"
+
+require_relative "lkml/simple"
 
 module Lkml
-  def self.parse(text)
+  module_function
+
+  def parse(text)
     lexer = Lexer.new(text)
     tokens = lexer.scan
     Parser.new(tokens).parse
   end
 
-  def self.dump(obj, io = nil)
-    parser = DictParser.new
-    result = parser.parse(obj).to_s
-    return result unless io
+  def load(stream)
+    text =
+      case stream
+      when String then stream
+      else
+        raise TypeError, "Input stream must be a string or file object." unless stream.respond_to?(:read)
+
+        stream.read
 
-    io.write(result)
-    nil
+      end
+    tree = parse(text)
+    Simple::DictVisitor.new.visit(tree)
+  end
+
+  def dump(obj, io: nil)
+    tree = Simple::DictParser.new.parse(obj)
+    result = tree.to_s
+    if io
+      io.write(result)
+      nil
+    else
+      result
+    end
+  end
+
+  alias generate dump
+
+  def dict_parser
+    Simple::DictParser
+  end
+
+  def dict_visitor
+    Simple::DictVisitor
   end
 end
+
+Lkml.const_set(:DictParser, Lkml::Simple::DictParser)
+Lkml.const_set(:DictVisitor, Lkml::Simple::DictVisitor)

data/lkml.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "lib/lkml/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "lkml"
+  spec.version = Lkml::VERSION
+  spec.authors = ["Josh Temple", "Sylvain Utard", "Altertable"]
+  spec.email = ["contact@altertable.ai"]
+
+  spec.summary = "A speedy LookML parser and serializer implemented in pure Ruby."
+  spec.description = <<~DESC
+    Parse LookML strings into Ruby hashes and serialize Ruby hashes back to LookML.
+    Implementation uses the same algorithm as the original Python lkml package.
+  DESC
+  spec.homepage = "https://github.com/altertable-ai/lkml"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 3.3.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|\.git)/}) }
+  end
+  spec.bindir = "bin"
+  spec.executables = ["lkml"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "logger"
+  spec.add_development_dependency "minitest", "~> 5.25"
+  spec.add_development_dependency "rake", "~> 13.2"
+  spec.add_development_dependency "rubocop", "~> 1.69"
+end

data/script/benchmark.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Rough CPU-time benchmark over bundled GitHub LookML samples (same idea as scripts/benchmark.py).
+
+require_relative "../lib/lkml"
+
+dir = File.expand_path("../test/resources/github", __dir__)
+times = []
+
+Dir.glob(File.join(dir, "*.lkml")).each do |path|
+  text = File.read(path, encoding: "UTF-8")
+  start = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+  begin
+    Lkml.load(text)
+  rescue SyntaxError, KeyError
+    next
+  end
+  finish = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
+  times << (finish - start)
+end
+
+if times.empty?
+  warn "No successful parses; nothing to measure."
+  exit 1
+end
+
+avg = times.sum / times.size
+variance = times.sum { |t| (t - avg)**2 } / (times.size - 1)
+stdev = Math.sqrt(variance)
+
+printf(
+  "Average process execution time: %.1f +/- %.1f ms\n",
+  avg * 100,
+  stdev * 100
+)

data/script/download_lookml.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Downloads sample LookML via GitHub Code Search API (same workflow as scripts/download_lookml.py).
+# Requires GITHUB_USERNAME and GITHUB_PERSONAL_ACCESS_TOKEN in the environment.
+# Writes files under test/resources/github/ (create the directory if needed).
+
+require "base64"
+require "fileutils"
+require "json"
+require "net/http"
+require "uri"
+
+def parse_next_link(link_header)
+  return nil if link_header.nil? || link_header.empty?
+
+  link_header.split(",").each do |segment|
+    next unless segment.include?('rel="next"')
+
+    return Regexp.last_match(1) if segment =~ /<([^>]+)>/
+  end
+  nil
+end
+
+username = ENV.fetch("GITHUB_USERNAME")
+token = ENV.fetch("GITHUB_PERSONAL_ACCESS_TOKEN")
+
+directory = File.expand_path("../test/resources/github", __dir__)
+FileUtils.mkdir_p(directory)
+
+start_url = "https://api.github.com/search/code?q=view+language:lookml"
+next_url = nil
+page = 1
+
+loop do
+  uri = URI(next_url || start_url)
+  req = Net::HTTP::Get.new(uri)
+  req.basic_auth(username, token)
+
+  res = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |h| h.request(req) }
+  res.value
+
+  next_url = parse_next_link(res["Link"])
+
+  warn next_url.inspect
+
+  payload = JSON.parse(res.body)
+  urls = payload.fetch("items").map { |item| item.fetch("url") }
+
+  warn "Downloading all content from page #{page}"
+  urls.each do |url|
+    u = URI(url)
+    r = Net::HTTP::Get.new(u)
+    r.basic_auth(username, token)
+    file_res = Net::HTTP.start(u.host, u.port, use_ssl: true) { |h| h.request(r) }
+    file_res.value
+    body = JSON.parse(file_res.body)
+    name = body.fetch("name")
+    encoded = body.fetch("content")
+    content = Base64.decode64(encoded.gsub("\n", "")).force_encoding("UTF-8")
+
+    next if name.end_with?(".lookml") || content.start_with?("-") || content.include?("- view")
+
+    File.write(File.join(directory, name), content)
+  end
+
+  break if next_url.nil?
+
+  page += 1
+end

metadata

@@ -1,38 +1,114 @@
 --- !ruby/object:Gem::Specification
 name: lkml
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
+- Josh Temple
 - Sylvain Utard
-autorequire:
+- Altertable
 bindir: bin
 cert_chain: []
-date: 2025-01-19 00:00:00.000000000 Z
-dependencies: []
-description: ''
-email: sylvain.utard@gmail.com
-executables: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.25'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.2'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.69'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.69'
+description: |
+  Parse LookML strings into Ruby hashes and serialize Ruby hashes back to LookML.
+  Implementation uses the same algorithm as the original Python lkml package.
+email:
+- contact@altertable.ai
+executables:
+- lkml
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ci.yml"
+- ".gitignore"
+- ".rubocop.yml"
+- ".ruby-version"
+- CONTRIBUTING.md
+- Gemfile
+- Gemfile.lock
 - LICENSE.md
 - README.md
+- Rakefile
+- bin/lkml
 - lib/lkml.rb
+- lib/lkml/cli.rb
 - lib/lkml/keys.rb
 - lib/lkml/lexer.rb
 - lib/lkml/parser.rb
 - lib/lkml/simple.rb
 - lib/lkml/tokens.rb
 - lib/lkml/tree.rb
+- lib/lkml/utils.rb
 - lib/lkml/version.rb
 - lib/lkml/visitors.rb
-homepage: ''
+- lkml.gemspec
+- script/benchmark.rb
+- script/download_lookml.rb
+homepage: https://github.com/altertable-ai/lkml
 licenses:
 - MIT
 metadata:
+  homepage_uri: https://github.com/altertable-ai/lkml
+  source_code_uri: https://github.com/altertable-ai/lkml
   rubygems_mfa_required: 'true'
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -40,15 +116,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.2'
+      version: 3.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
-summary: LookML Ruby Parser
+summary: A speedy LookML parser and serializer implemented in pure Ruby.
 test_files: []