appquery 0.1.0 → 0.2.0

data/lib/app_query/tokenizer.rb

@@ -0,0 +1,356 @@
+# frozen_string_literal: true
+
+module AppQuery
+  class Tokenizer
+    class LexError < StandardError; end
+
+    attr_reader :input, :tokens, :pos, :start
+
+    def self.tokenize(...)
+      new(...).run
+    end
+
+    def initialize(input, state: nil, start: nil, pos: nil)
+      @input = input
+      @tokens = []
+      @start = start || 0
+      @pos = pos || @start
+      @return = Array(state || :lex_sql)
+    end
+
+    def err(msg)
+      linepos = linepos_by_pos[pos] || linepos_by_pos[pos.pred]
+
+      msg += <<~ERR
+
+        #{input}
+        #{" " * linepos}^
+      ERR
+      raise LexError, msg
+    end
+
+    def eos?
+      pos == input.size
+    end
+
+    def chars_read
+      input[start...pos]
+    end
+
+    def read_char(n = 1)
+      @pos = [pos + n, input.size].min
+      self
+    end
+
+    def rest
+      input[pos...]
+    end
+
+    def match?(re)
+      rest[Regexp.new("\\A%s" % re)]
+    end
+
+    def emit_token(t, v: nil)
+      @tokens << {v: v || chars_read, t: t, start: start, end: pos}
+      @start = @pos
+      self
+    end
+
+    def push_return(*steps)
+      (@return ||= []).push(*steps)
+      self
+    end
+
+    def read_until(pattern)
+      loop do
+        break if match?(pattern) || eos?
+        read_char
+      end
+    end
+
+    def lex_sql
+      if last_emitted? t: "CTE_SELECT", ignore: %w[WHITESPACE COMMENT]
+        push_return :lex_select
+      elsif match?(/\s/)
+        push_return :lex_sql, :lex_whitespace
+      elsif match_comment?
+        push_return :lex_sql, :lex_comment
+      elsif match?(/with/i)
+        push_return :lex_sql, :lex_with
+      else
+        push_return :lex_select
+      end
+    end
+
+    def lex_with
+      err "Expected 'WITH'" unless match? %r{WITH\s}i
+      read_until(/\s/)
+      read_until(/\S/)
+      emit_token "WITH"
+
+      push_return :lex_recursive_cte
+    end
+
+    def lex_prepend_cte
+      if eos?
+        emit_token "COMMA", v: ","
+        emit_token "WHITESPACE", v: "\n"
+      else
+        # emit_token "WHITESPACE", v: " "
+        push_return :lex_prepend_cte, :lex_recursive_cte
+      end
+    end
+
+    def lex_append_cte
+      emit_token "COMMA", v: ","
+      emit_token "WHITESPACE", v: "\n  "
+      push_return :lex_recursive_cte
+    end
+
+    def lex_recursive_cte
+      if match?(/recursive\s/i)
+        read_until(/\s/)
+        # make trailing whitespace part of next token
+        # this makes adding cte's easier
+        read_until(/\S/)
+        emit_token "RECURSIVE"
+      end
+
+      push_return :lex_cte
+    end
+
+    def last_emitted(ignore:)
+      if ignore.none?
+        @tokens.last
+      else
+        t = @tokens.dup
+        while (result = t.pop)
+          break if !ignore.include?(result[:t])
+        end
+        result
+      end
+    end
+
+    def last_emitted?(ignore_whitespace: true, ignore: [], **kws)
+      ignore = if ignore.any?
+        ignore
+      elsif ignore_whitespace
+        %w[COMMENT WHITESPACE]
+      else
+        []
+      end
+      last_emitted(ignore:)&.slice(*kws.keys) == kws
+    end
+
+    def lex_cte
+      if match_comment?
+        push_return :lex_cte, :lex_comment
+      elsif last_emitted? t: "CTE_IDENTIFIER", ignore_whitespace: true
+        if match?(/AS(\s|\()/i)
+          read_char 2
+          emit_token "AS"
+
+          push_return :lex_cte, :lex_cte_select, :lex_maybe_materialized, :lex_whitespace
+        elsif match?(%r{\(})
+          # "foo " "(id)"
+          push_return :lex_cte, :lex_cte_columns
+        else
+          err "Expected 'AS' or CTE columns following CTE-identifier, e.g. 'foo AS' 'foo()'"
+        end
+      elsif last_emitted? t: "CTE_COLUMNS_CLOSE", ignore_whitespace: true
+        if match?(/AS(\s|\()/i)
+          read_char 2
+          emit_token "AS"
+
+          push_return :lex_cte, :lex_cte_select, :lex_maybe_materialized, :lex_whitespace
+        else
+          err "Expected 'AS' following CTE-columns"
+        end
+      elsif last_emitted? t: "CTE_SELECT", ignore_whitespace: true
+        if match?(/,/)
+          # but wait, there's more!
+          read_char
+          emit_token "CTE_COMMA"
+          push_return :lex_cte, :lex_whitespace
+        end
+      else
+        push_return :lex_cte, :lex_cte_identifier
+      end
+    end
+
+    def lex_maybe_materialized
+      if match?(/materialized/i)
+        read_until(/\(/)
+        emit_token "MATERIALIZED"
+      elsif match?(%r{\(})
+        # done
+      elsif match?(/not\s/i)
+        read_char 3
+        read_until(/\S/)
+        emit_token "NOT_MATERIALIZED"
+        err "Expected 'MATERIALIZED'" unless match?(/materialized/i)
+
+        push_return :lex_maybe_materialized
+      else
+        err "Expected CTE select or NOT? MATERIALIZED"
+      end
+    end
+
+    def match_comment?
+      match?(%r{--|/\*})
+    end
+
+    def lex_cte_columns
+      err "Expected CTE columns, e.g. '(id, other)'" unless match? %r{\(}
+
+      read_char
+      read_until(/\S/)
+      emit_token "CTE_COLUMNS_OPEN"
+
+      loop do
+        if match?(/\)/)
+          err "Expected a column name" unless last_emitted? t: "CTE_COLUMN"
+
+          read_char
+          emit_token "CTE_COLUMNS_CLOSE"
+          break
+        elsif match?(/,/)
+          # "( " ","
+          err "Expected a column name" unless last_emitted? t: "CTE_COLUMN"
+          read_char # ','
+
+          read_until(/\S/)
+          emit_token "CTE_COLUMN_DIV"
+        elsif match?(/"/)
+          unless last_emitted? t: "CTE_COLUMNS_OPEN"
+            err "Expected comma" unless last_emitted? t: "CTE_COLUMN_DIV"
+          end
+
+          read_char
+          read_until(/"/)
+          read_char
+
+          emit_token "CTE_COLUMN"
+        elsif match?(/[_A-Za-z]/)
+          unless last_emitted? t: "CTE_COLUMNS_OPEN"
+            err "Expected comma" unless last_emitted? t: "CTE_COLUMN_DIV"
+          end
+
+          read_until %r{,|\s|\)}
+
+          emit_token "CTE_COLUMN"
+        elsif match?(/\s/)
+          read_until(/\S/)
+        else
+          # e.g. "(id," "1)" or eos?
+          err "Expected valid column name"
+        end
+      end
+
+      push_return :lex_whitespace
+    end
+
+    def lex_cte_select
+      err "Expected CTE select, e.g. '(select 1)'" unless match? %r{\(}
+      read_char
+
+      level = 1
+      loop do
+        read_until(/\)|\(/)
+        if eos?
+          err "CTE select ended prematurely"
+        elsif match?(/\(/)
+          level += 1
+        elsif match?(/\)/)
+          level -= 1
+          break if level.zero?
+        end
+        read_char
+      end
+
+      err "Expected non-empty CTE select, e.g. '(select 1)'" if chars_read.strip == "("
+      read_char
+      emit_token "CTE_SELECT"
+
+      push_return :lex_whitespace
+    end
+
+    def lex_cte_identifier
+      err "Expected CTE identifier, e.g. 'foo', '\"foo bar\"' " unless match? %r{[_"A-Za-z]}
+
+      if match?(/"/)
+        read_char
+        read_until(/"/)
+        read_char
+      else
+        read_until %r{\s|\(}
+      end
+      emit_token "CTE_IDENTIFIER"
+
+      push_return :lex_whitespace
+    end
+
+    # there should always be a SELECT
+    def lex_select
+      read_until(/\Z/)
+      read_char
+
+      if last_emitted? t: "COMMENT", ignore_whitespace: false
+        emit_token "WHITESPACE", v: "\n"
+      end
+      emit_token "SELECT"
+    end
+
+    def lex_comment
+      err "Expected comment, i.e. '--' or '/*'" unless match_comment?
+
+      if match?("--")
+        read_until(/\n/)
+      else
+        read_until %r{\*/}
+        err "Expected comment close '*/'." if eos?
+        read_char 2
+      end
+
+      emit_token "COMMENT"
+      push_return :lex_whitespace
+    end
+
+    # optional
+    def lex_whitespace
+      if match?(/\s/)
+        read_until(/\S/)
+
+        emit_token "WHITESPACE"
+      end
+    end
+
+    def run(pos: nil)
+      loop do
+        break if step.nil?
+      end
+      eos? ? tokens : self
+    end
+
+    def step
+      if (state = @return.pop)
+        method(state).call
+        self
+      end
+    end
+
+    private
+
+    def linepos_by_pos
+      linepos = 0
+      input.each_char.each_with_index.each_with_object([]) do |(c, ix), acc|
+        acc[ix] = linepos
+        if c == "\n"
+          linepos = 0
+        else
+          linepos += 1
+        end
+      end
+    end
+  end
+end

data/lib/app_query/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AppQuery
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/lib/app_query.rb

@@ -1,8 +1,245 @@
 # frozen_string_literal: true
 
 require_relative "app_query/version"
+require_relative "app_query/tokenizer"
+require "active_record"
 
 module AppQuery
   class Error < StandardError; end
-  # Your code goes here...
+
+  Configuration = Struct.new(:query_path)
+
+  def self.configuration
+    @configuration ||= AppQuery::Configuration.new
+  end
+
+  def self.configure
+    yield configuration if block_given?
+  end
+
+  def self.reset_configuration!
+    configure do |config|
+      config.query_path = "app/queries"
+    end
+  end
+  reset_configuration!
+
+  def self.[](v)
+    query_name = v.to_s
+    full_path = (Pathname.new(configuration.query_path) / "#{query_name}.sql").expand_path
+    Q.new(full_path.read, name: "AppQuery #{query_name}")
+  end
+
+  class Result < ActiveRecord::Result
+    attr_accessor :cast
+    alias_method :cast?, :cast
+
+    def initialize(columns, rows, overrides = nil, cast: false)
+      super(columns, rows, overrides)
+      @cast = cast
+      # Rails v6.1: prevent mutate on frozen object on #first
+      @hash_rows = [] if columns.empty?
+    end
+
+    def column(name = nil)
+      return [] if empty?
+      unless name.nil? || includes_column?(name)
+        raise ArgumentError, "Unknown column #{name.inspect}. Should be one of #{columns.inspect}."
+      end
+      ix = name.nil? ? 0 : columns.index(name)
+      rows.map { _1[ix] }
+    end
+
+    def self.from_ar_result(r, cast = nil)
+      if r.empty?
+        EMPTY
+      else
+        cast &&= case cast
+        when Array
+          r.columns.zip(cast).to_h
+        when Hash
+          cast
+        else
+          {}
+        end
+        if !cast || (cast.empty? && r.column_types.empty?)
+          # nothing to cast
+          new(r.columns, r.rows, r.column_types)
+        else
+          overrides = (r.column_types || {}).merge(cast)
+          rows = r.cast_values(overrides)
+          # One column is special :( ;(
+          # > ActiveRecord::Base.connection.select_all("select array[1,2]").rows
+          # => [["{1,2}"]]
+          # > ActiveRecord::Base.connection.select_all("select array[1,2]").cast_values
+          # => [[1, 2]]
+          rows = rows.map { [_1] } if r.columns.one?
+          new(r.columns, rows, overrides, cast: true)
+        end
+      end
+    end
+
+    empty_array = [].freeze
+    EMPTY_HASH = {}.freeze
+    private_constant :EMPTY_HASH
+
+    EMPTY = new(empty_array, empty_array, EMPTY_HASH).freeze
+    private_constant :EMPTY
+  end
+
+  class Q
+    attr_reader :name, :sql
+
+    def initialize(sql, name: nil)
+      @sql = sql
+      @name = name
+    end
+
+    def select_all(binds: [], select: nil, cast: false)
+      with_select(select).then do |aq|
+        ActiveRecord::Base.connection.select_all(aq.to_s, name, binds).then do |result|
+          Result.from_ar_result(result, cast)
+        end
+      end
+    end
+
+    def select_one(binds: [], select: nil, cast: false)
+      select_all(binds:, select:, cast:).first || {}
+    end
+
+    def select_value(binds: [], select: nil, cast: false)
+      select_one(binds:, select:, cast:).values.first
+    end
+
+    def tokens
+      @tokens ||= tokenizer.run
+    end
+
+    def tokenizer
+      @tokenizer ||= Tokenizer.new(to_s)
+    end
+
+    def cte_names
+      tokens.filter { _1[:t] == "CTE_IDENTIFIER" }.map { _1[:v] }
+    end
+
+    def with_select(sql)
+      return self unless sql
+      if cte_names.include?("_")
+        self.class.new(tokens.each_with_object([]) do |token, acc|
+          v = (token[:t] == "SELECT") ? sql : token[:v]
+          acc << v
+        end.join, name: name)
+      else
+        append_cte("_ as (\n  #{select}\n)").with_select(sql)
+      end
+    end
+
+    def select
+      tokens.find { _1[:t] == "SELECT" }&.[](:v)
+    end
+
+    def recursive?
+      !!tokens.find { _1[:t] == "RECURSIVE" }
+    end
+
+    # example:
+    #  AppQuery("select 1").prepend_cte("foo as(select 1)")
+    def prepend_cte(cte)
+      # early raise when cte is not valid sql
+      to_append = Tokenizer.tokenize(cte, state: :lex_prepend_cte).then do |tokens|
+        recursive? ? tokens.reject { _1[:t] == "RECURSIVE" } : tokens
+      end
+
+      if cte_names.none?
+        self.class.new("WITH #{cte}\n#{self}")
+      else
+        split_at_type = recursive? ? "RECURSIVE" : "WITH"
+        self.class.new(tokens.map do |token|
+          if token[:t] == split_at_type
+            token[:v] + to_append.map { _1[:v] }.join
+          else
+            token[:v]
+          end
+        end.join)
+      end
+    end
+
+    # example:
+    #  AppQuery("select 1").append_cte("foo as(select 1)")
+    def append_cte(cte)
+      # early raise when cte is not valid sql
+      add_recursive, to_append = Tokenizer.tokenize(cte, state: :lex_append_cte).then do |tokens|
+        [!recursive? && tokens.find { _1[:t] == "RECURSIVE" },
+          tokens.reject { _1[:t] == "RECURSIVE" }]
+      end
+
+      if cte_names.none?
+        self.class.new("WITH #{cte}\n#{self}")
+      else
+        nof_ctes = cte_names.size
+
+        self.class.new(tokens.map do |token|
+          nof_ctes -= 1 if token[:t] == "CTE_SELECT"
+
+          if nof_ctes.zero?
+            nof_ctes -= 1
+            token[:v] + to_append.map { _1[:v] }.join
+          elsif token[:t] == "WITH" && add_recursive
+            token[:v] + add_recursive[:v]
+          else
+            token[:v]
+          end
+        end.join)
+      end
+    end
+
+    # Replaces an existing cte.
+    # Raises `ArgumentError` when cte does not exist.
+    def replace_cte(cte)
+      add_recursive, to_append = Tokenizer.tokenize(cte, state: :lex_recursive_cte).then do |tokens|
+        [!recursive? && tokens.find { _1[:t] == "RECURSIVE" },
+          tokens.reject { _1[:t] == "RECURSIVE" }]
+      end
+
+      cte_name = to_append.find { _1[:t] == "CTE_IDENTIFIER" }&.[](:v)
+      unless cte_names.include?(cte_name)
+        raise ArgumentError, "Unknown cte #{cte_name.inspect}. Options: #{cte_names}."
+      end
+      cte_ix = cte_names.index(cte_name)
+
+      return self unless cte_ix
+
+      cte_found = false
+
+      self.class.new(tokens.map do |token|
+        if cte_found ||= token[:t] == "CTE_IDENTIFIER" && token[:v] == cte_name
+          unless (cte_found = (token[:t] != "CTE_SELECT"))
+            next to_append.map { _1[:v] }.join
+          end
+
+          next
+        elsif token[:t] == "WITH" && add_recursive
+          token[:v] + add_recursive[:v]
+        else
+          token[:v]
+        end
+      end.join)
+    end
+
+    def to_s
+      @sql
+    end
+  end
 end
+
+def AppQuery(...)
+  AppQuery::Q.new(...)
+end
+
+begin
+  require "rspec"
+rescue LoadError
+end
+
+require_relative "app_query/rspec" if Object.const_defined? :RSpec

data/lib/appquery.rb

@@ -0,0 +1 @@
+require "app_query"

data/lib/rails/generators/query/USAGE

@@ -0,0 +1,10 @@
+Description:
+    Generates a new query-file and invokes your template
+    engine and test framework generators.
+
+Example:
+    `bin/rails generate query reports/weekly`
+
+    creates an SQL file and test:
+        Query:  app/queries/reports/weekly.sql
+        Spec:   spec/queries/reports/weekly_query_spec.rb

data/lib/rails/generators/query/query_generator.rb

@@ -0,0 +1,20 @@
+module Rails
+  module Generators
+    class QueryGenerator < NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      def create_query_file
+        template "query.sql",
+          File.join(AppQuery.configuration.query_path, class_path, "#{file_name}.sql")
+
+        # in_root do
+        #  if behavior == :invoke && !File.exist?(application_mailbox_file_name)
+        #    template "application_mailbox.rb", application_mailbox_file_name
+        #  end
+        # end
+      end
+
+      hook_for :test_framework
+    end
+  end
+end

data/lib/rails/generators/query/templates/query.sql.tt

@@ -0,0 +1,14 @@
+-- Instantiate this query with AppQuery[<%= (class_path << file_name).join("/").inspect %>]
+
+WITH
+  articles(article_id, article_title) AS (
+    VALUES (1, 'Some title'),
+           (2, 'Another article')
+  ),
+  authors(author_id, author_name) AS (
+    VALUES (1, 'Some Author'),
+           (2, 'Another Author')
+  )
+
+SELECT *
+FROM artciles

data/lib/rails/generators/rspec/query_generator.rb

@@ -0,0 +1,20 @@
+module Rspec
+  module Generators
+    class QueryGenerator < ::Rails::Generators::NamedBase
+      source_root File.expand_path("templates", __dir__)
+
+      def create_test_file
+        template "query_spec.rb",
+          File.join("spec/queries", class_path, "#{file_name}_query_spec.rb")
+      end
+
+      hide!
+
+      private
+
+      def query_path
+        AppQuery.configuration.query_path
+      end
+    end
+  end
+end

data/lib/rails/generators/rspec/templates/query_spec.rb.tt

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "AppQuery <%= (class_path << file_name).join("/") %>", type: :query, default_binds: nil do
+  describe "CTE articles" do
+    specify do
+      expect(select_all(select: "select * from :cte").cast_entries).to \
+        include(a_hash_including("article_id" => 1))
+    end
+  end
+end

data/sig/appquery.rbs

@@ -1,4 +1,4 @@
-module Appquery
+module AppQuery
   VERSION: String
   # See the writing guide of rbs: https://github.com/ruby/rbs#guides
 end