pluckr 0.1.0

data/lib/pluckr/batch.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+module Pluckr
+  # Runs several unrelated aggregates over ActiveRecord relations in one
+  # statement:
+  #
+  #   stats = Pluckr.batch do |b|
+  #     b.count  user.videos,                        as: :video_count
+  #     b.sum    user.videos.where(size: 100..), :size, as: :big_video_bytes
+  #     b.exists user.photos,                        as: :has_photos
+  #     b.avg    user.videos, :size,                 as: :average_size
+  #     b.count  Account.active,                     as: :active_accounts
+  #   end
+  #
+  #   stats.video_count  # => 3
+  #
+  # Each entry is whatever SQL the relation already compiles to, so default
+  # scopes, association scopes, `:through`, polymorphic and STI conditions are
+  # respected - ActiveRecord applied them before Pluckr saw the relation.
+  #
+  # This is the ad-hoc counterpart to `Pluckr::Query`: no class, no reuse, and
+  # bound to concrete records rather than to a row of a result set.
+  class Batch
+    # Eager loading is pointless inside a scalar aggregate subquery and joining
+    # the associated tables can change the row count; row locks are not allowed
+    # alongside an aggregate.
+    IGNORED_RELATION_VALUES = %i[includes eager_load preload lock].freeze
+
+    COLUMNLESS_OPERATIONS = %i[count exists].freeze
+
+    def self.build(&block)
+      raise ConfigurationError, "`Pluckr.batch` requires a block" unless block
+
+      batch = new
+      block.call(batch)
+      batch
+    end
+
+    def initialize
+      @nodes = []
+    end
+
+    # ---------------------------------------------------------------- DSL --
+
+    def count(source, column = nil, as:)
+      add(:count, source, column, as)
+    end
+
+    def sum(source, column, as:)
+      add(:sum, source, column, as)
+    end
+
+    def avg(source, column, as:)
+      add(:avg, source, column, as)
+    end
+    alias average avg
+
+    def min(source, column, as:)
+      add(:min, source, column, as)
+    end
+
+    def max(source, column, as:)
+      add(:max, source, column, as)
+    end
+
+    def exists(source, as:)
+      add(:exists, source, nil, as)
+    end
+
+    # ----------------------------------------------------------- execution --
+
+    def to_sql
+      compiler.aggregate_only_sql
+    end
+
+    def fetch
+      scope = schema
+      compiler = Compiler::Sql.new(schema: scope)
+      row = Pluckr.select_all(compiler.connection, compiler.aggregate_only_sql,
+                              name: "Pluckr::Batch", label: "Pluckr Batch").first
+
+      Result::Builder.new(scope, label: "Pluckr::Batch").build(row)
+    end
+
+    def entries
+      @nodes.map(&:output)
+    end
+
+    def inspect
+      "#<Pluckr::Batch #{entries.join(", ")}>"
+    end
+
+    private
+
+    def add(operation, source, column, output)
+      relation = relation_for(operation, output, source)
+      column = validate_column!(operation, output, column, relation.klass)
+      validate_grouping!(operation, output, column, relation)
+      validate_projection!(operation, output, column, relation)
+      validate_connection!(output, relation)
+      validate_output!(output)
+
+      @nodes << Schema::Aggregate.new(
+        output: output,
+        operation: operation,
+        relation: normalize(relation),
+        column: column
+      )
+      self
+    end
+
+    # The derived table has to expose exactly what the outer aggregate reads,
+    # without changing what the caller asked for.
+    def normalize(relation)
+      ignored = IGNORED_RELATION_VALUES.dup
+      # ORDER BY only matters when it decides which rows a limit/offset keeps.
+      ignored << :order unless relation.limit_value || relation.offset_value
+      relation = relation.unscope(*ignored)
+
+      # A projection is meaningful when it is what DISTINCT or GROUP BY applies
+      # to, and noise otherwise.
+      return relation if relation.select_values.any? && projection_matters?(relation)
+
+      return relation.unscope(:select) if relation.group_values.empty?
+
+      # `SELECT *` alongside GROUP BY is invalid on PostgreSQL and MySQL.
+      relation.unscope(:select).select(*relation.group_values)
+    end
+
+    def projection_matters?(relation)
+      relation.distinct_value || relation.group_values.any?
+    end
+
+    def relation_for(operation, output, source)
+      return source.all if source.is_a?(Class) && source < ActiveRecord::Base
+      return source if source.is_a?(ActiveRecord::Relation)
+
+      raise ConfigurationError,
+            "`#{operation} ..., as: :#{output}` expects an ActiveRecord relation or model, " \
+            "got #{source.inspect}"
+    end
+
+    # A grouped relation returns one row per group, so the only thing that can
+    # be asked of it is how many groups there are.
+    def validate_grouping!(operation, output, column, relation)
+      return if relation.group_values.empty?
+      return if COLUMNLESS_OPERATIONS.include?(operation) && column.nil?
+
+      raise ConfigurationError,
+            "`#{operation} ..., as: :#{output}`: the relation is grouped, so only `count` and " \
+            "`exists` (without a column) are meaningful - aggregate inside the relation instead"
+    end
+
+    # count and exists work without a column; the rest need one.
+    def validate_column!(operation, output, column, model)
+      if column.nil?
+        return nil if COLUMNLESS_OPERATIONS.include?(operation)
+
+        raise ConfigurationError, "`#{operation} ..., as: :#{output}` requires a column"
+      end
+
+      unless model.column_names.include?(column.to_s)
+        raise UnknownField, "#{model.name} does not have column `#{column}`"
+      end
+
+      column
+    end
+
+    # With DISTINCT the caller's projection is kept, so an aggregate column that
+    # is not part of it would not exist in the derived table.
+    def validate_projection!(operation, output, column, relation)
+      return if column.nil? || !relation.distinct_value
+
+      projected = relation.select_values.select { |value| value.is_a?(Symbol) || value.is_a?(String) }
+      return if projected.empty? || projected.map(&:to_s).include?(column.to_s)
+
+      raise ConfigurationError,
+            "`#{operation} ..., as: :#{output}`: the relation selects #{projected.join(", ")} " \
+            "with DISTINCT, so column `#{column}` is not available to aggregate"
+    end
+
+    # One statement means one connection.
+    def validate_connection!(output, relation)
+      return if @nodes.empty?
+
+      first = @nodes.first.relation.klass
+      return if first.connection_pool == relation.klass.connection_pool
+
+      raise ConfigurationError,
+            "`as: :#{output}`: #{relation.klass.name} and #{first.name} use different database " \
+            "connections, so they cannot be batched into one statement"
+    end
+
+    def validate_output!(output)
+      return unless @nodes.any? { |node| node.output == output.to_sym }
+
+      raise ConfigurationError, "duplicate output name `#{output}` in batch"
+    end
+
+    # Rebuilt per call: entries may be added between to_sql and fetch.
+    def schema
+      raise ConfigurationError, "batch is empty - add at least one aggregate" if @nodes.empty?
+
+      Schema::Scope.new(nil, @nodes.dup) # Scope freezes what it is given
+    end
+
+    def compiler
+      Compiler::Sql.new(schema: schema)
+    end
+  end
+
+  # @see Pluckr::Batch
+  def self.batch(&block)
+    Batch.build(&block).fetch
+  end
+end

data/lib/pluckr/compiler/sql.rb

@@ -0,0 +1,363 @@
+# frozen_string_literal: true
+
+module Pluckr
+  module Compiler
+    # Turns a Schema::Scope into SQL.
+    #
+    # Everything is built with Arel and ActiveRecord, so PostgreSQL, MySQL and
+    # SQLite all get correctly quoted statements from the same code path.
+    #
+    # Rules of the MVP:
+    #   * plain fields          -> selected columns, never `SELECT users.*`
+    #   * singular associations -> LEFT OUTER JOIN + hidden presence marker
+    #   * exists                -> correlated EXISTS subquery
+    #   * count/sum/min/max     -> scalar subquery (never JOIN + GROUP BY)
+    #
+    # The compiler only reads the AST. It never touches DSL state.
+    class Sql
+      attr_reader :schema, :source_model
+
+      def initialize(schema:, source_model: nil)
+        @schema = schema
+        @source_model = source_model
+      end
+
+      # Applies Pluckr projections/joins on top of a plain ActiveRecord relation.
+      #
+      # Compiled fresh every time: `scope:` callables may depend on runtime state
+      # and the compiler must stay free of shared mutable state.
+      def apply(relation)
+        validate_connections!
+        build = Build.new
+        walk(build, schema, [], source_model.arel_table)
+
+        relation.select(*build.projections).joins(build.joins)
+      end
+
+      # Single-statement SQL for aggregate-only (dashboard and batch) queries.
+      def aggregate_only_sql
+        validate_connections!
+
+        parts = schema.nodes.map.with_index(1) do |node, index|
+          "#{scalar_subquery(node, index)} AS #{quote_alias(node.output)}"
+        end
+
+        "SELECT #{parts.join(", ")}"
+      end
+
+      def connection
+        connection_model.connection
+      end
+
+      # A single statement cannot span two databases. Relation-rooted nodes are
+      # checked when they are added, in Pluckr::Batch.
+      def validate_connections!
+        models = ([source_model] + independent_models).compact.uniq
+        return if models.size < 2
+
+        pool = models.first.connection_pool
+        offender = models.find { |model| model.connection_pool != pool }
+        return unless offender
+
+        raise ConfigurationError,
+              "#{offender.name} and #{models.first.name} use different database connections, " \
+              "so they cannot be read in one statement"
+      end
+
+      private
+
+      def independent_models(scope = schema)
+        scope.nodes.flat_map do |node|
+          case node
+          when Schema::Aggregate then node.independent? && node.relation.nil? ? [node.model] : []
+          when Schema::One then independent_models(node.scope)
+          else []
+          end
+        end
+      end
+
+      # Accumulator for one compilation pass.
+      class Build
+        attr_reader :projections, :joins
+
+        def initialize
+          @projections = []
+          @joins = []
+          @sequence = 0
+        end
+
+        def next_sequence
+          @sequence += 1
+        end
+      end
+
+      def walk(build, scope, path, table)
+        scope.nodes.each do |node|
+          case node
+          when Schema::Field
+            project build, table[node.column], Aliases.output(path, node.output)
+          when Schema::Exists
+            project build, exists_expression(build, node, table), Aliases.output(path, node.output)
+          when Schema::Aggregate
+            expression =
+              if node.independent?
+                scalar_subquery(node)
+              else
+                correlated_aggregate(build, node, table)
+              end
+            project build, expression, Aliases.output(path, node.output)
+          when Schema::One
+            walk_one(build, node, path, table)
+          else
+            raise Error, "unsupported schema node #{node.class}"
+          end
+        end
+      end
+
+      def walk_one(build, node, path, owner_table)
+        return walk_picked(build, node, path, owner_table) if node.picked?
+
+        child_path = path + [node.output.to_s]
+        association = node.association
+        child_table = association.klass.arel_table.alias(Aliases.table(child_path))
+
+        condition = association.join_condition(child_table, owner_table)
+        sti = sti_condition(association.klass, child_table)
+        condition = condition.and(sti) if sti
+
+        build.joins << Arel::Nodes::OuterJoin.new(child_table, Arel::Nodes::On.new(condition))
+
+        # Presence marker: tells "no associated row" apart from "row with NULL columns".
+        project build, child_table[association.primary_key], Aliases.presence(child_path)
+
+        walk(build, node.scope, child_path, child_table)
+      end
+
+      # `first`/`last`: one row of a collection, without joining it. Each selected
+      # column is its own correlated subquery, ordered the same way and ending in
+      # the primary key, so every column comes from the same row.
+      #
+      #   (SELECT "c"."body" FROM "comments" "c" WHERE "c"."user_id" = "users"."id"
+      #     ORDER BY "c"."created_at" DESC, "c"."id" DESC LIMIT 1) AS "last_comment.body"
+      def walk_picked(build, node, path, owner_table)
+        child_path = path + [node.output.to_s]
+
+        # Presence marker: the row exists even if every selected column is NULL.
+        project build, picked_column(build, node, owner_table, node.association.primary_key),
+                Aliases.presence(child_path)
+
+        node.scope.nodes.each do |field|
+          project build, picked_column(build, node, owner_table, field.column),
+                  Aliases.output(child_path, field.output)
+        end
+      end
+
+      def picked_column(build, node, owner_table, column)
+        association = node.association
+        table = subquery_table(build, association)
+        subquery = Arel::SelectManager.new(table)
+                   .project(table[column])
+                   .where(association.join_condition(table, owner_table))
+        subquery = restrict_to_sti(subquery, association.klass, table)
+
+        node.order.each { |name, direction| subquery.order(table[name].public_send(direction)) }
+
+        # Returned as Arel, not SQL: an STI type condition carries a bind
+        # parameter, and only the outer relation can render it.
+        Arel::Nodes::Grouping.new(subquery.take(1).ast)
+      end
+
+      def project(build, expression, output_alias)
+        build.projections << Arel::Nodes::As.new(expression, Arel.sql(quote_alias(output_alias)))
+      end
+
+      # EXISTS (SELECT 1 FROM photos WHERE photos.user_id = users.id LIMIT 1 OFFSET 0)
+      #
+      # The `LIMIT 1 OFFSET 0` is not decoration. Left alone, PostgreSQL turns a
+      # correlated EXISTS in the SELECT list into a hashed subplan - one full scan
+      # of the child table - which dominates a paged query. `OFFSET 0` is the
+      # standard optimisation fence: the plan becomes an index probe per row that
+      # stops at the first match. It is a no-op everywhere else, and valid on
+      # PostgreSQL, MySQL and SQLite. benchmarks/ has the numbers.
+      def exists_expression(build, node, owner_table)
+        subquery =
+          if node.relation_scoped?
+            correlated_subquery(build, node, owner_table) { Arel.sql("1") }
+          else
+            table = subquery_table(build, node.association)
+            manager = Arel::SelectManager.new(table)
+                      .project(Arel.sql("1"))
+                      .where(node.association.join_condition(table, owner_table))
+            restrict_to_sti(manager, node.association.klass, table)
+          end
+
+        Arel::Nodes::Exists.new(subquery.take(1).skip(0).ast)
+      end
+
+      # (SELECT COUNT(*) FROM videos WHERE videos.user_id = users.id)
+      def correlated_aggregate(build, node, owner_table)
+        if node.relation_scoped?
+          subquery = correlated_subquery(build, node, owner_table) { |table| aggregate_function(node, table) }
+          return Arel::Nodes::Grouping.new(subquery.ast)
+        end
+
+        table = subquery_table(build, node.association)
+        subquery = Arel::SelectManager.new(table)
+                   .project(aggregate_function(node, table))
+                   .where(node.association.join_condition(table, owner_table))
+        subquery = restrict_to_sti(subquery, node.association.klass, table)
+
+        Arel::Nodes::Grouping.new(subquery.ast)
+      end
+
+      # (SELECT COUNT(*) FROM accounts WHERE accounts.active = TRUE)
+      #
+      # Conditions are delegated to ActiveRecord, so values are quoted by the
+      # adapter - never string-interpolated here.
+      def scalar_subquery(node, index = 1)
+        return relation_subquery(node, index) if node.relation
+
+        relation = aggregate_relation(node, node.model.all)
+        relation = relation.select(aggregate_function(node, node.model.arel_table))
+
+        Arel.sql("(#{relation.to_sql})")
+      end
+
+      # A scoped correlated subquery reads through a derived table:
+      #
+      #   (SELECT COUNT(*)
+      #      FROM (SELECT photos.* FROM photos INNER JOIN users ...) pluckr_sub_1
+      #     WHERE pluckr_sub_1.user_id = users.id)
+      #
+      # An aggregate over a relation the caller already built. Its SQL is used
+      # verbatim, inside a derived table, so whatever ActiveRecord applied -
+      # default scopes, association scopes, :through joins, STI conditions -
+      # stays applied, and `limit`/`group` keep their meaning.
+      #
+      #   (SELECT COUNT(*) FROM (SELECT videos.* FROM videos WHERE ...) pluckr_batch_1)
+      def relation_subquery(node, index)
+        name = "#{Aliases::TABLE_PREFIX}_batch_#{index}"
+        derived = Arel::Table.new(name)
+        from = Arel.sql("(#{node.relation.to_sql}) #{connection.quote_table_name(name)}")
+
+        return Arel.sql("EXISTS (#{Arel::SelectManager.new.from(from).project(Arel.sql("1")).take(1).to_sql})") if
+          node.operation == :exists
+
+        Arel.sql("(#{Arel::SelectManager.new.from(from).project(aggregate_function(node, derived)).to_sql})")
+      end
+
+      # `where:`/`scope:` conditions name the model's real table, which may be the
+      # table being correlated to (a self-referential association) or one the scope
+      # joins in. Nesting them inside a derived table keeps those names in their own
+      # scope, so the correlation always binds to the outer row.
+      def correlated_subquery(build, node, owner_table)
+        klass = node.association.klass
+        inner = aggregate_relation(node, klass.all).select(klass.arel_table[Arel.star])
+        name = "#{Aliases::TABLE_PREFIX}_sub_#{build.next_sequence}"
+        derived = Arel::Table.new(name)
+
+        Arel::SelectManager.new
+                           .from(Arel.sql("(#{inner.to_sql}) #{connection.quote_table_name(name)}"))
+                           .project(yield(derived))
+                           .where(node.association.join_condition(derived, owner_table))
+      end
+
+      # Whatever the scope selects, orders or limits is meaningless (or invalid)
+      # inside a scalar aggregate subquery.
+      def aggregate_relation(node, relation)
+        relation = narrow(node, relation)
+        reject_row_shaping!(node, relation)
+
+        relation.unscope(:select, :order)
+      end
+
+      def reject_row_shaping!(node, relation)
+        shaping = { limit: relation.limit_value, offset: relation.offset_value,
+                    group: relation.group_values.presence }.compact.keys
+        return if shaping.empty?
+
+        raise ConfigurationError,
+              "`#{node.output}`: `scope:` may not use #{shaping.join("/")} - the subquery must " \
+              "return exactly one value"
+      end
+
+      def narrow(node, relation)
+        relation = apply_scope(node, relation)
+        conditions = node.resolved_where
+
+        conditions ? relation.where(conditions) : relation
+      end
+
+      def apply_scope(node, relation)
+        return relation unless node.scope
+
+        zero_arity = node.scope.respond_to?(:arity) && node.scope.arity.zero?
+        scoped = zero_arity ? node.scope.call : node.scope.call(relation)
+
+        unless scoped.is_a?(ActiveRecord::Relation) && scoped.klass == relation.klass
+          # Never `inspect` a relation here - that would run a query while raising
+          # a configuration error.
+          got = scoped.is_a?(ActiveRecord::Relation) ? "a relation for #{scoped.klass.name}" : scoped.inspect
+
+          raise ConfigurationError,
+                "`#{node.output}`: `scope:` must return an ActiveRecord::Relation for " \
+                "#{relation.klass.name}, got #{got}"
+        end
+
+        scoped
+      end
+
+      def aggregate_function(node, table)
+        case node.operation
+        when :count
+          node.column ? Arel::Nodes::Count.new([table[node.column]]) : Arel::Nodes::Count.new([Arel.star])
+        when :sum then Arel::Nodes::Sum.new([table[node.column]])
+        when :avg then Arel::Nodes::Avg.new([table[node.column]])
+        when :min then Arel::Nodes::Min.new([table[node.column]])
+        when :max then Arel::Nodes::Max.new([table[node.column]])
+        else
+          raise Error, "unsupported aggregate operation #{node.operation}"
+        end
+      end
+
+      def restrict_to_sti(subquery, klass, table)
+        condition = sti_condition(klass, table)
+        condition ? subquery.where(condition) : subquery
+      end
+
+      # Single table inheritance: an association to an STI child must be
+      # restricted to its own types. `type_condition` accepts the aliased table,
+      # which is why Pluckr uses it instead of rebuilding the condition.
+      def sti_condition(klass, table)
+        return nil unless klass.finder_needs_type_condition?
+
+        klass.send(:type_condition, table)
+      end
+
+      def subquery_table(build, association)
+        association.klass.arel_table.alias("#{Aliases::TABLE_PREFIX}_sub_#{build.next_sequence}")
+      end
+
+      def quote_alias(name)
+        name = name.to_s
+        limit = connection.max_identifier_length
+
+        if name.bytesize > limit
+          raise ConfigurationError,
+                "output alias `#{name}` is #{name.bytesize} bytes, over this database's " \
+                "#{limit}-byte identifier limit - shorten it with `as:`, or rename the " \
+                "enclosing `one` nodes if it is a nested path"
+        end
+
+        connection.quote_column_name(name)
+      end
+
+      def connection_model
+        @connection_model ||=
+          source_model ||
+          schema.nodes.grep(Schema::Aggregate).map(&:target_model).first ||
+          ActiveRecord::Base
+      end
+    end
+  end
+end

data/lib/pluckr/errors.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Pluckr
+  # Base class for every error raised by Pluckr.
+  class Error < StandardError; end
+
+  # Raised when a query is defined or used incorrectly.
+  class ConfigurationError < Error; end
+
+  # `field :nope` where the model has no such column.
+  class UnknownField < ConfigurationError; end
+
+  # `one :nope` / `exists :nope` where the model has no such association.
+  class UnknownAssociation < ConfigurationError; end
+
+  # Association exists but has the wrong cardinality for the DSL node used.
+  class InvalidAssociation < ConfigurationError; end
+
+  # Association exists but Pluckr cannot compile it yet (polymorphic, through, scoped).
+  class UnsupportedAssociation < ConfigurationError; end
+
+  # A root-record node was used in a query without `source`.
+  class MissingSource < ConfigurationError; end
+end