query-composer 1.0.0

data/examples/library.rb

@@ -0,0 +1,200 @@
+# A simple example that demonstrates the use of Query::Composer in a
+# library reporting system. Given a data model that includes sets of
+# libraries, topics, books, and patrons, and permis books to be lended
+# from a library to a patron on a given date, this script builds and
+# executes a query that shows how many books from a given set of topics
+# and libraries each patron borrowed during a given period of time, and
+# compares it to the corresponding period of the previous month.
+
+require 'active_record'
+require 'query/composer'
+require 'query/base'
+
+# connect to the DB
+ActiveRecord::Base.establish_connection(
+  adapter: "sqlite3",
+  database: ":memory:"
+)
+
+# generate the schema
+ActiveRecord::Schema.verbose = false
+ActiveRecord::Schema.define do
+  create_table :libraries do |t|
+    t.string :name
+  end
+
+  create_table :topics do |t|
+    t.string :name
+  end
+
+  create_table :patrons do |t|
+    t.string :name
+  end
+
+  create_table :books do |t|
+    t.string :name
+    t.integer :library_id
+    t.integer :topic_id
+  end
+
+  create_table :lendings do |t|
+    t.integer :book_id
+    t.integer :patron_id
+    t.date    :created_at
+  end
+end
+
+# populate the database
+
+ActiveRecord::Base.connection.execute <<-SQL
+  INSERT INTO libraries (id, name)
+    VALUES (1, 'Gotham'),
+           (2, 'Hogwarts')
+SQL
+
+ActiveRecord::Base.connection.execute <<-SQL
+  INSERT INTO topics (id, name)
+    VALUES (1, 'Warts'),
+           (2, 'Seaweed'),
+           (3, 'Dryer Lint'),
+           (4, 'Sitcoms')
+SQL
+
+ActiveRecord::Base.connection.execute <<-SQL
+  INSERT INTO patrons (id, name)
+    VALUES (1, 'Harry'),
+           (2, 'Mary'),
+           (3, 'Larry'),
+           (4, 'Carry'),
+           (5, 'Terry'),
+           (6, 'Cheri')
+SQL
+
+ActiveRecord::Base.connection.execute <<-SQL
+  INSERT INTO books (id, name, library_id, topic_id)
+    VALUES (1, 'Odd Growths', 1, 1),
+           (2, 'Nose Accessories', 2, 1),
+           (3, 'Health Foods', 1, 2),
+           (4, 'Green', 1, 2),
+           (5, 'Slimy Things', 1, 2),
+           (6, 'Household Chores', 2, 3),
+           (7, 'Starting Fires', 2, 3),
+           (8, 'Laughter', 1, 4),
+           (9, 'Funny People', 1, 4),
+           (10, 'Silliness', 2, 4)
+SQL
+
+ActiveRecord::Base.connection.execute <<-SQL
+  INSERT INTO lendings (book_id, patron_id, created_at)
+    VALUES (1, 1, '2016-01-01'),
+           (2, 2, '2016-01-04'),
+           (3, 3, '2016-01-05'),
+           (4, 4, '2016-01-06'),
+           (5, 4, '2016-01-08'),
+           (6, 4, '2016-01-08'),
+           (7, 5, '2016-01-11'),
+           (8, 6, '2016-01-12'),
+           (9, 6, '2016-01-14'),
+           (10, 6, '2016-01-15'),
+           (1, 6, '2016-02-01'),
+           (2, 6, '2016-02-04'),
+           (3, 5, '2016-02-05'),
+           (4, 5, '2016-02-06'),
+           (5, 4, '2016-02-08'),
+           (6, 3, '2016-02-08'),
+           (7, 3, '2016-02-11'),
+           (8, 2, '2016-02-12'),
+           (9, 2, '2016-02-14'),
+           (10, 1, '2016-02-15')
+SQL
+
+# define the models
+
+class Library < ActiveRecord::Base
+  has_many :books
+end
+
+class Topic < ActiveRecord::Base
+  has_many :books
+end
+
+class Patron < ActiveRecord::Base
+  has_many :lendings
+  has_many :books, through: :lendings
+end
+
+class Book < ActiveRecord::Base
+  belongs_to :library
+  belongs_to :topic
+  has_many :lendings
+  has_many :patrons, through: :lendings
+end
+
+class Lending < ActiveRecord::Base
+  belongs_to :patron
+  belongs_to :book
+end
+
+# Construct the reporting query
+
+composer = Query::Composer.new
+
+composer.use(:libraries_set) { Library.where(id: [ 1, 2 ]) }
+composer.use(:topics_set) { Topic.where(id: [ 1, 2, 3, 4 ]) }
+composer.use(:patrons_set) { Patron.all }
+
+composer.use(:books_set) do |libraries_set, topics_set|
+  books = Book.arel_table
+
+  Query::Base.new(books).
+    project(books[:id]).
+    join(libraries_set).
+      on(books[:library_id].eq(libraries_set[:id])).
+    join(topics_set).
+      on(books[:topic_id].eq(topics_set[:id]))
+end
+
+composer.use(:current_set) do |books_set|
+  lendings_set(books_set, '2016-02-01', '2016-02-15')
+end
+
+composer.use(:prior_set) do |books_set|
+  lendings_set(books_set, '2016-01-01', '2016-01-15')
+end
+
+composer.use(:combined_set) do |patrons_set, current_set, prior_set|
+  Query::Base.new(patrons_set).
+    project(patrons_set[Arel.star],
+            current_set[:total].as("current_total"),
+            prior_set[:total].as("prior_total")).
+    join(current_set).
+      on(current_set[:patron_id].eq(patrons_set[:id])).
+    join(prior_set, Arel::Nodes::OuterJoin).
+      on(prior_set[:patron_id].eq(patrons_set[:id]))
+end
+
+def lendings_set(books_set, from_date, to_date)
+  lendings = Lending.arel_table
+
+  patron_id = lendings[:patron_id]
+  count = patron_id.count.as("total")
+
+  Query::Base.new(lendings).
+    project(patron_id, count).
+    join(books_set).
+      on(lendings[:book_id].eq(books_set[:id])).
+    where(lendings[:created_at].between(from_date..to_date)).
+    group(patron_id)
+end
+
+sql = composer.build(:combined_set).to_sql
+puts "---- SQL ----"
+puts sql
+
+puts
+puts "%-6s | %3s | %5s" % %w(Patron Now Prior)
+puts "%-6s-+-%3s-+-%5s-" % ["-"*6, "-"*3, "-"*5]
+
+Patron.find_by_sql(sql).each do |patron|
+  puts "%-6s | %3d | %5d" % [ patron.name, patron.current_total, patron.prior_total||0]
+end

data/lib/query/base.rb

@@ -0,0 +1,55 @@
+require 'arel'
+
+module Query
+  class Base
+    attr_reader :primary_table, :arel
+
+    def initialize(primary, *args)
+      @primary_table = _make_table(primary)
+
+      @arel = Arel::SelectManager.new(ActiveRecord::Base).
+        from(@primary_table)
+
+      _configure(*args)
+    end
+
+    def reproject(*projections)
+      arel.projections = projections
+      self
+    end
+
+    def all
+      arel.project(primary_table[Arel.star])
+      self
+    end
+
+    # ensures #as returns an Arel node, and not the Query object
+    # (so that it plays nice with our custom #with method, above).
+    def as(name)
+      arel.as(name)
+    end
+
+    def to_sql
+      @arel.to_sql
+    end
+
+    alias to_s to_sql
+
+    def method_missing(sym, *args, &block)
+      arel.send(sym, *args, &block)
+      self
+    end
+
+    def _make_table(value)
+      case value
+      when Arel::Table then value
+      when Arel::Nodes::TableAlias then value
+      else Arel::Table.new(value)
+      end
+    end
+
+    def _configure(*args)
+      # overridden by subclasses for per-query configuration
+    end
+  end
+end

data/lib/query/composer.rb

@@ -0,0 +1,315 @@
+require 'arel'
+
+module Query
+
+  # A class for composing queries into large, complicated reporting
+  # monstrosities that return data for trends, histograms, and all
+  # kinds of other things.
+  #
+  # The idea is that you first create a composer object:
+  #
+  #   q = Query::Composer.new
+  #
+  # Then, you tell the composer about a few queries:
+  #
+  #   q.use(:entities)  { User.all }
+  #   q.use(:companies) { Company.all }
+  #
+  # These queries are *independent*, in that they have no dependencies.
+  # But we can add some queries now that depend on those. We declare
+  # another query, giving it one or more parameters. Those parameter
+  # names must match the identifiers of queries given to the composer.
+  # Here, we have a query that is dependent on the "entities" and
+  # "companies" queries, above.
+  #
+  #   q.use(:entities_with_extra) do |entities, companies|
+  #     team_table = Arel::Table.new(:teams)
+  #
+  #     Arel::SelectManager.new(ActiveRecord::Base).
+  #       from(entities).
+  #       project(
+  #         entities[Arel.star],
+  #         team_table[:name].as('team_name'),
+  #         companies[:name].as('company_name')).
+  #       join(team_table).
+  #         on(team_table[:id].eq(entities[:team_id])).
+  #       join(companies).
+  #         on(companies[:id].eq(entities[:company_id]))
+  #   end
+  #
+  # After you've defined a bunch of these queries, you should have
+  # one of them (and ONLY one of them) that nothing else depends on.
+  # This is the "root" query--the one that returns the data set you're
+  # looking for. The composer can now do its job and accumulate and
+  # aggregate all those queries together, by calling the #build method
+  # with the identifier for the root query you want to build.
+  #
+  #   query = q.build(:some_query_identifier)
+  #
+  # By default, this will create a query with each component represented
+  # as derived tables (nested subqueries):
+  #
+  #   SELECT "a".*,
+  #          "b"."name" AS "company_name",
+  #          "c"."name" AS "team_name"
+  #     FROM (
+  #       SELECT "users".* FROM "users"
+  #     ) a
+  #     INNER JOIN (
+  #       SELECT "companies".* FROM "companies"
+  #     ) b
+  #     ON "b"."id" = "a"."company_id"
+  #     INNER JOIN (
+  #       SELECT "teams".* FROM "teams"
+  #     ) c
+  #     ON "c"."id" = "a"."team_id"
+  #     WHERE ...
+  #
+  # If you would rather use CTEs (Common Table Expressions, or "with"
+  # queries), you can pass ":use_cte => true" to generate the following:
+  #
+  #   WITH
+  #     "a" AS (SELECT "users".* FROM "users"),
+  #     "b" AS (SELECT "companies".* FROM "companies"),
+  #     "c" AS (
+  #       SELECT "a".*,
+  #              "teams"."name" as "team_name",
+  #              "b"."name" as "company_name"
+  #         FROM "a"
+  #        INNER JOIN "teams"
+  #           ON "teams"."id" = "a"."team_id"
+  #        INNER JOIN "b"
+  #           ON "b".id = "a"."company_id")
+  #     ...
+  #   SELECT ...
+  #     FROM ...
+  #
+  # Be aware, though, that some DBMS's (like Postgres) do not optimize
+  # CTE's, and so the resulting queries may be very inefficient.
+  #
+  # If you don't want the short, opaque identifiers to be used as
+  # aliases, you can pass ":use_aliases => false" to #build:
+  #
+  #   query = q.build(:entities_with_extra, :use_aliases => false)
+  #
+  # That way, the query identifiers themselves will be used as the
+  # query aliases.
+
+  class Composer
+    class Error < RuntimeError; end
+    class UnknownQuery < Error; end
+    class CircularDependency < Error; end
+    class InvalidQuery < Error; end
+
+    @@prefer_cte = false
+    @@prefer_aliases = true
+
+    class <<self
+      def prefer_cte?
+        @@prefer_cte
+      end
+
+      # By default, the composer generates queries that use derived
+      # tables. If you'd rather default to CTE's,
+      # set Query::Composer.prefer_cte to true.
+      def prefer_cte=(preference)
+        @@prefer_cte = preference
+      end
+
+      def prefer_aliases?
+        @@prefer_aliases
+      end
+
+      # By default, the composer generates queries that use shortened
+      # names as aliases for the full names of the components. If you'd
+      # rather use the full names instead of aliases, 
+      def prefer_aliases=(preference)
+        @@prefer_aliases = preference
+      end
+    end
+
+    # Create an empty query object. If a block is given, the query
+    # object will be yielded to it.
+    def initialize
+      @parts = {}
+      yield self if block_given?
+    end
+
+    # Indicate that the named identifier should be defined by the given
+    # block. The names used for the parameters of the block are significant,
+    # and must exactly match the identifiers of other elements in the
+    # query.
+    #
+    # The block should return an Arel object, for use in composing the
+    # larger reporting query. If the return value of the block responds
+    # to :arel, the result of that method will be returned instead.
+    def use(name, &definition)
+      @parts[name] = definition
+      self
+    end
+
+    # Aliases the given query component with the new name. This can be
+    # useful for redefining an existing component, where you still
+    # want to retain the old definition.
+    #
+    #   composer.use(:source) { Something.all }
+    #   composer.alias(:old_source, :source)
+    #   composer.use(:source) { |old_source| ... }
+    def alias(new_name, name)
+      @parts[new_name] = @parts[name]
+      self
+    end
+
+    # Removes the named component from the composer.
+    def delete(name)
+      @parts.delete(name)
+      self
+    end
+
+    # Return an Arel object representing the query starting at the
+    # component named `root`. Supported options are:
+    #
+    # * :use_cte (false) - the query should use common table expressions.
+    #   If false, the query will use derived tables, instead.
+    # * :use_aliases (true) - the query will use short, opaque identifiers
+    #   for aliases. If false, the query will use the full dependency
+    #   names to identify the elements.
+    def build(root, options={})
+      deps = _resolve(root)
+      aliases = _alias_queries(deps, options)
+
+      if _use_cte?(options)
+        _query_with_cte(root, deps, aliases)
+      else
+        _query_with_derived_table(root, deps, aliases)
+      end
+    end
+
+    def _use_cte?(options)
+      options.fetch(:use_cte, self.class.prefer_cte?)
+    end
+
+    def _use_aliases?(options)
+      options.fetch(:use_aliases, self.class.prefer_aliases?)
+    end
+
+    # Builds an Arel object using derived tables.
+    def _query_with_derived_table(root, deps, aliases)
+      queries = {}
+
+      deps.each do |name|
+        queries[name] = _invoke(name, queries).as(aliases[name].name)
+      end
+
+      _invoke(root, queries)
+    end
+
+    # Builds an Arel object using common table expressions.
+    def _query_with_cte(root, deps, aliases)
+      query = _invoke(root, aliases)
+      components = []
+
+      deps.each do |name|
+        component = _invoke(name, aliases)
+        aliased = Arel::Nodes::As.new(aliases[name], component)
+        components << aliased
+      end
+
+      query.with(*components) if components.any?
+      query
+    end
+
+    # Invokes the named dependency, using the given aliases mapping.
+    def _invoke(name, aliases)
+      block = @parts[name]
+      params = block.parameters.map { |(_, name)| aliases[name] }
+      result = block.call(*params)
+
+      if result.respond_to?(:arel)
+        result.arel
+      elsif result.respond_to?(:to_sql)
+        result
+      else
+        raise InvalidQuery, "query elements must quack like #arel or #to_sql (`#{name}` returned #{result.class})"
+      end
+    end
+
+    # Ensure that all referenced dependencies exist in the graph.
+    # Otherwise, raise Query::Composer::UnknownQuery.
+    def _validate_dependencies!(name)
+      raise UnknownQuery, "`#{name}`" unless @parts.key?(name)
+      dependencies = []
+
+      @parts[name].parameters.each do |(_, pname)|
+        unless @parts.key?(pname)
+          raise UnknownQuery, "`#{pname}` referenced by `#{name}`"
+        end
+
+        dependencies << pname
+      end
+
+      dependencies
+    end
+
+    # Resolves the tree of dependent components by traversing the graph
+    # starting at `root`. Returns an array of identifiers where elements
+    # later in the list depend on zero or more elements earlier in the
+    # list. The resulting list includes only the dependencies of the
+    # `root` element, but not the `root` element itself.
+    def _resolve(root)
+      _resolve2(root).flatten.uniq - [root]
+    end
+
+    # This is a utility function, used only by #_resolve. It recursively
+    # tranverses the tree, depth-first, and returns a "tree" (array of
+    # recursively nested arrays) representing the graph at root. The
+    # root of each subtree is at the end of the corresponding array.
+    #
+    #   [ [ [:a], [:b], :c ], [ [:d], [:e], :f ], :root ]
+    def _resolve2(root, dependents=[])
+      deps = _validate_dependencies!(root)
+      return [ root ] if deps.empty?
+
+      # Circular dependency exists if anything in the dependents
+      # (that which depends on root) exists in root's own dependency
+      # list
+
+      dependents = [ root, *dependents ]
+      overlap = deps & dependents
+      if overlap.any?
+        raise CircularDependency, "#{root} -> #{overlap.join(', ')}"
+      end
+
+      all = []
+
+      deps.each do |dep|
+        all << _resolve2(dep, dependents)
+      end
+
+
+      all << root
+    end
+
+    # Build a mapping of dependency names, to Arel::Table objects. The
+    # Arel::Table names will use opaque, short identifiers ("a", "b", etc.),
+    # unless the :use_aliases option is false, when the dependency names
+    # themselves will be used.
+    def _alias_queries(deps, options={})
+      use_aliases = _use_aliases?(options)
+
+      aliases = {}
+      current_alias = "a"
+
+      deps.each do |key|
+        if use_aliases
+          aliases[key] = Arel::Table.new(current_alias)
+          current_alias = current_alias.succ
+        else
+          aliases[key] = Arel::Table.new(key)
+        end
+      end
+
+      aliases
+    end
+  end
+end