simple-sql 0.5.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7a208873c084434a4267d43cfd76ccae14aa249e7ce2cd40de78e76fe4d0df8
-  data.tar.gz: c6c961737c2ebd4b57b5ab7946b872f229e63a8c22c275177825a8bcb5301cef
+  metadata.gz: 76825a68d4370b6f5e870a09f2013502706b0f35a83a911e5451fc147a9a1940
+  data.tar.gz: 4093415861c077fbdf633dd1d347c6a6e2b2ed3da7df0c95dfaa078919d9b5d4
 SHA512:
-  metadata.gz: 2c5c053640e50ba398acf014f855fb28ea369bdd7806ae7a564f69ce87f1d7323dd58ced4ca9a77bd6e7a595266f16754a5de5d0ce6efc0f24fd29b2ea2acb39
-  data.tar.gz: af48f5add33e6330a526ec2d6486579f84f2e1b2fb7ee0d03aff0a0d65723da07ea8c5932978c26db6cf005aee2627c02ebb7c2b4ba273ad909c95fd0fa8b4bb
+  metadata.gz: 835f1c43a6d63d7647ba0d81c4dffabe39ef599f3623e47c3d62f2073e8f53dc88d54d387514efdb996aabdfebd5b0a599025eb3c0e4c3e2abe54685000323d9
+  data.tar.gz: eda18e175ed172c11b3a8132fb956e66fbd06da57036ce65cb9361da9a1d66ca41edac4085fa4c048db69827025b5c49b8b7bde43d132f217e5defd0665f92e4

data/lib/simple/sql/connection/duplicate.rb

@@ -0,0 +1,86 @@
+class Simple::SQL::Connection
+  #
+  # Creates duplicates of record in a table.
+  #
+  # This method handles timestamp columns (these will be set to the current
+  # time) and primary keys (will be set to NULL.) You can pass in overrides
+  # as a third argument for specific columns.
+  #
+  # Parameters:
+  #
+  # - ids: (Integer, Array<Integer>) primary key ids
+  # - overrides: Hash[column_names => SQL::Fragment]
+  #
+  def duplicate(table, ids, overrides = {})
+    ids = Array(ids)
+    return [] if ids.empty?
+
+    Duplicator.new(self, table, overrides).call(ids)
+  end
+
+  class Duplicator
+    attr_reader :connection
+    attr_reader :table_name
+    attr_reader :custom_overrides
+
+    def initialize(connection, table_name, overrides)
+      @connection       = connection
+      @table_name       = table_name
+      @custom_overrides = validated_overrides(overrides)
+    end
+
+    def call(ids)
+      connection.all query, ids
+    rescue PG::UndefinedColumn => e
+      raise ArgumentError, e.message
+    end
+
+    private
+
+    # This stringify all keys of the overrides hash, and verifies that
+    # all values in there are SQL::Fragments
+    def validated_overrides(overrides)
+      overrides.inject({}) do |hsh, (key, value)|
+        unless value.is_a?(::Simple::SQL::Fragment)
+          raise ArgumentError, "Pass in override values via SQL.fragment (for #{value.inspect})"
+        end
+
+        hsh.update key.to_s => value.to_sql
+      end
+    end
+
+    def timestamp_overrides
+      reflection = connection.reflection
+      reflection.timestamp_columns(table_name).inject({}) do |hsh, column|
+        hsh.update column => "now() AS #{column}"
+      end
+    end
+
+    def copy_columns
+      reflection = connection.reflection
+      (reflection.columns(table_name) - reflection.primary_key_columns(table_name)).inject({}) do |hsh, column|
+        hsh.update column => column
+      end
+    end
+
+    # build SQL query
+    def query
+      sources = {}
+
+      sources.update copy_columns
+      sources.update timestamp_overrides
+      sources.update custom_overrides
+
+      # convert into an Array, to make sure that keys and values aka firsts
+      # and lasts are always in the correct order.
+      sources = sources.to_a
+
+      <<~SQL
+        INSERT INTO #{table_name}(#{sources.map(&:first).join(', ')})
+          SELECT #{sources.map(&:last).join(', ')}
+          FROM #{table_name}
+          WHERE id = ANY($1) RETURNING id
+      SQL
+    end
+  end
+end

data/lib/simple/sql/connection/insert.rb

@@ -0,0 +1,73 @@
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/LineLength
+# rubocop:disable Layout/AlignHash
+
+class Simple::SQL::Connection
+  #
+  # - table_name - the name of the table
+  # - records - a single hash of attributes or an array of hashes of attributes
+  # - on_conflict - uses a postgres ON CONFLICT clause to ignore insert conflicts if true
+  #
+  def insert(table, records, on_conflict: nil, into: nil)
+    if records.is_a?(Hash)
+      inserted_records = insert(table, [records], on_conflict: on_conflict, into: into)
+      return inserted_records.first
+    end
+
+    return [] if records.empty?
+
+    inserter = inserter(table_name: table.to_s, columns: records.first.keys, on_conflict: on_conflict, into: into)
+    inserter.insert(records: records)
+  end
+
+  private
+
+  def inserter(table_name:, columns:, on_conflict:, into:)
+    @inserters ||= {}
+    @inserters[[table_name, columns, on_conflict, into]] ||= Inserter.new(self, table_name, columns, on_conflict, into)
+  end
+
+  class Inserter
+    CONFICT_HANDLING = {
+      nil      => "",
+      :nothing => "ON CONFLICT DO NOTHING",
+      :ignore  => "ON CONFLICT DO NOTHING"
+    }
+
+    #
+    # - table_name - the name of the table
+    # - columns - name of columns, as Array[String] or Array[Symbol]
+    #
+    def initialize(connection, table_name, columns, on_conflict, into)
+      expect! on_conflict => CONFICT_HANDLING.keys
+      raise ArgumentError, "Cannot insert a record without attributes" if columns.empty?
+
+      @connection = connection
+      @columns = columns
+      @into = into
+
+      cols = []
+      vals = []
+
+      cols += columns
+      vals += columns.each_with_index.map { |_, idx| "$#{idx + 1}" }
+
+      timestamp_columns = @connection.reflection.timestamp_columns(table_name) - columns.map(&:to_s)
+
+      cols += timestamp_columns
+      vals += timestamp_columns.map { "now()" }
+
+      returning = into ? "*" : "id"
+
+      @sql = "INSERT INTO #{table_name} (#{cols.join(',')}) VALUES(#{vals.join(',')}) #{CONFICT_HANDLING[on_conflict]} RETURNING #{returning}"
+    end
+
+    def insert(records:)
+      SQL.transaction do
+        records.map do |record|
+          SQL.ask @sql, *record.values_at(*@columns), into: @into
+        end
+      end
+    end
+  end
+end

data/lib/simple/sql/connection/reflection.rb

@@ -0,0 +1,107 @@
+class Simple::SQL::Connection
+  def reflection
+    @reflection ||= Reflection.new(self)
+  end
+
+  class Reflection
+    def initialize(connection)
+      @connection = connection
+    end
+
+    def tables(schema: "public")
+      table_info(schema: schema).keys
+    end
+
+    def primary_key_columns(table_name)
+      @connection.all <<~SQL, table_name
+        SELECT pg_attribute.attname
+        FROM   pg_index
+        JOIN   pg_attribute ON pg_attribute.attrelid = pg_index.indrelid AND pg_attribute.attnum = ANY(pg_index.indkey)
+        WHERE  pg_index.indrelid = $1::regclass
+        AND    pg_index.indisprimary;
+      SQL
+    end
+
+    TIMESTAMP_COLUMN_NAMES = %w(inserted_at created_at updated_at)
+
+    # timestamp_columns are columns that will be set automatically after
+    # inserting or updating a record. This includes:
+    #
+    # - inserted_at (for Ecto)
+    # - created_at (for ActiveRecord)
+    # - updated_at (for Ecto and ActiveRecord)
+    def timestamp_columns(table_name)
+      columns(table_name) & TIMESTAMP_COLUMN_NAMES
+    end
+
+    def columns(table_name)
+      column_info(table_name).keys
+    end
+
+    def table_info(schema: "public")
+      recs = @connection.all <<~SQL, schema, into: Hash
+        SELECT table_schema || '.' || table_name AS name, *
+        FROM information_schema.tables
+        WHERE table_schema=$1
+      SQL
+      records_by_attr(recs, :name)
+    end
+
+    def column_info(table_name)
+      @column_info ||= {}
+      @column_info[table_name] ||= _column_info(table_name)
+    end
+
+    private
+
+    def _column_info(table_name)
+      schema, table_name = parse_table_name(table_name)
+      recs = @connection.all <<~SQL, schema, table_name, into: Hash
+        SELECT
+          column_name AS name,
+          *
+        FROM information_schema.columns
+        WHERE table_schema=$1 AND table_name=$2
+      SQL
+
+      records_by_attr(recs, :column_name)
+    end
+
+    def parse_table_name(table_name)
+      p1, p2 = table_name.split(".", 2)
+      if p2
+        [p1, p2]
+      else
+        ["public", p1]
+      end
+    end
+
+    def records_by_attr(records, attr)
+      records.inject({}) do |hsh, record|
+        record.reject! { |_k, v| v.nil? }
+        hsh.update record[attr] => OpenStruct.new(record)
+      end
+    end
+
+    public
+
+    def looked_up_pg_classes
+      @looked_up_pg_classes ||= Hash.new { |hsh, key| hsh[key] = _lookup_pg_class(key) }
+    end
+
+    def lookup_pg_class(oid)
+      looked_up_pg_classes[oid]
+    end
+
+    private
+
+    def _lookup_pg_class(oid)
+      ::Simple::SQL.ask <<~SQL, oid
+        SELECT nspname AS schema, relname AS host_table
+        FROM pg_class
+        JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace
+        WHERE pg_class.oid=$1
+      SQL
+    end
+  end
+end

data/lib/simple/sql/connection.rb

@@ -1,3 +1,13 @@
+class Simple::SQL::Connection
+end
+
+require_relative "connection/raw_connection"
+require_relative "connection/active_record_connection"
+
+require_relative "connection/reflection"
+require_relative "connection/insert"
+require_relative "connection/duplicate"
+
 # A Connection object.
 #
 # A Connection object is built around a raw connection (as created from the pg
@@ -26,6 +36,3 @@ class Simple::SQL::Connection
   extend Forwardable
   delegate [:wait_for_notify] => :raw_connection
 end
-
-require_relative "connection/raw_connection"
-require_relative "connection/active_record_connection"

data/lib/simple/sql/connection_adapter.rb

@@ -90,6 +90,16 @@ module Simple::SQL::ConnectionAdapter
     end
   end
 
+  # returns an Array [min_cost, max_cost] based on the database's estimation
+  def costs(sql, *args)
+    explanation_first = Simple::SQL.ask "EXPLAIN #{sql}", *args
+    unless explanation_first =~ /cost=(\d+(\.\d+))\.+(\d+(\.\d+))/
+      raise "Cannot determine cost"
+    end
+
+    [Float($1), Float($3)]
+  end
+
   # Executes a block, usually of db insert code, while holding an
   # advisory lock.
   #
@@ -145,7 +155,7 @@ module Simple::SQL::ConnectionAdapter
   end
 
   def convert_rows_to_result(rows, into:, pg_source_oid:)
-    Result.build(rows, target_type: into, pg_source_oid: pg_source_oid)
+    Result.build(self, rows, target_type: into, pg_source_oid: pg_source_oid)
   end
 
   public

data/lib/simple/sql/fragment.rb

@@ -0,0 +1,11 @@
+# rubocop:disable Style/StructInheritance
+module Simple
+  module SQL
+    class Fragment < Struct.new(:to_sql)
+    end
+
+    def fragment(str)
+      Fragment.new(str)
+    end
+  end
+end

data/lib/simple/sql/logging.rb

@@ -50,7 +50,7 @@ module Simple
       end
 
       def slow_query_treshold=(slow_query_treshold)
-        expect! slow_query_treshold > 0
+        expect! slow_query_treshold.nil? || slow_query_treshold > 0
         @slow_query_treshold = slow_query_treshold
       end
 

data/lib/simple/sql/result/association_loader.rb

@@ -12,7 +12,6 @@ require "active_support/core_ext/string/inflections"
 module ::Simple::SQL::Result::AssociationLoader # :nodoc:
   extend self
 
-  SQL = ::Simple::SQL
   H = ::Simple::SQL::Helpers
 
   private
@@ -23,10 +22,10 @@ module ::Simple::SQL::Result::AssociationLoader # :nodoc:
   # "foo.users", if such a table exists.
   #
   # Raises an ArgumentError if no matching table can be found.
-  def find_associated_table(association, schema:)
+  def find_associated_table(connection, association, schema:)
     fq_association = "#{schema}.#{association}"
 
-    tables_in_schema = ::Simple::SQL::Reflection.table_info(schema: schema).keys
+    tables_in_schema = connection.reflection.table_info(schema: schema).keys
 
     return fq_association              if tables_in_schema.include?(fq_association)
     return fq_association.singularize  if tables_in_schema.include?(fq_association.singularize)
@@ -49,7 +48,7 @@ module ::Simple::SQL::Result::AssociationLoader # :nodoc:
   #
   # Raises an ArgumentError if no association can be found between these tables.
   #
-  def find_matching_relation(host_table, associated_table)
+  def find_matching_relation(connection, host_table, associated_table)
     expect! host_table => /^[^.]+.[^.]+$/
     expect! associated_table => /^[^.]+.[^.]+$/
 
@@ -73,7 +72,7 @@ module ::Simple::SQL::Result::AssociationLoader # :nodoc:
         OR (belonging_table=$2 AND having_table=$1)
     SQL
 
-    relations = SQL.all(sql, host_table, associated_table, into: :struct)
+    relations = connection.all(sql, host_table, associated_table, into: :struct)
 
     return relations.first if relations.length == 1
 
@@ -150,7 +149,7 @@ module ::Simple::SQL::Result::AssociationLoader # :nodoc:
   # - host_table: the name of the table \a records has been loaded from.
   # - schema: the schema name in the database.
   # - as: the name to sue for the association. Defaults to +association+
-  def preload(records, association, host_table:, schema:, as:, order_by:, limit:)
+  def preload(connection, records, association, host_table:, schema:, as:, order_by:, limit:)
     return records if records.empty?
 
     expect! records.first => Hash
@@ -158,8 +157,8 @@ module ::Simple::SQL::Result::AssociationLoader # :nodoc:
     as = association if as.nil?
     fq_host_table = "#{schema}.#{host_table}"
 
-    associated_table = find_associated_table(association, schema: schema)
-    relation         = find_matching_relation(fq_host_table, associated_table)
+    associated_table = find_associated_table(connection, association, schema: schema)
+    relation         = find_matching_relation(connection, fq_host_table, associated_table)
 
     if fq_host_table == relation.belonging_table
       if order_by || limit

data/lib/simple/sql/result/records.rb

@@ -1,15 +1,12 @@
 # rubocop:disable Naming/UncommunicativeMethodParamName
 
 require_relative "association_loader"
-require "simple/sql/reflection"
 
 class ::Simple::SQL::Result::Records < ::Simple::SQL::Result
-  Reflection = ::Simple::SQL::Reflection
-
-  def initialize(records, target_type:, pg_source_oid:) # :nodoc:
+  def initialize(connection, records, target_type:, pg_source_oid:) # :nodoc:
     # expect! records.first => Hash unless records.empty?
 
-    super(records)
+    super(connection, records)
 
     @hash_records   = records
     @target_type    = target_type
@@ -50,9 +47,10 @@ class ::Simple::SQL::Result::Records < ::Simple::SQL::Result
     # resolve oid into table and schema name.
     #
     # [TODO] is this still correct?
-    schema, host_table = Reflection.lookup_pg_class @pg_source_oid
+    schema, host_table = connection.reflection.lookup_pg_class @pg_source_oid
 
-    AssociationLoader.preload @hash_records, association,
+    AssociationLoader.preload connection,
+                              @hash_records, association,
                               host_table: host_table, schema: schema, as: as,
                               order_by: order_by, limit: limit
 
@@ -69,7 +67,7 @@ class ::Simple::SQL::Result::Records < ::Simple::SQL::Result
   def materialize
     records = @hash_records
     if @target_type != Hash
-      schema, host_table = Reflection.lookup_pg_class(@pg_source_oid)
+      schema, host_table = connection.reflection.lookup_pg_class(@pg_source_oid)
       records = RowConverter.convert_row(records, associations: @associations,
                                                   into: @target_type,
                                                   fq_table_name: "#{schema}.#{host_table}")

data/lib/simple/sql/result.rb

@@ -1,5 +1,6 @@
-# rubocop:disable Metrics/AbcSize
 # rubocop:disable Naming/AccessorMethodName
+# rubocop:disable Style/DoubleNegation
+# rubocop:disable Style/GuardClause
 
 require_relative "helpers"
 
@@ -19,51 +20,82 @@ class ::Simple::SQL::Result < Array
   # A Result object is requested via ::Simple::SQL::Result.build, which then
   # chooses the correct implementation, based on the <tt>target_type:</tt>
   # parameter.
-  def self.build(records, target_type:, pg_source_oid:) # :nodoc:
+  def self.build(connection, records, target_type:, pg_source_oid:) # :nodoc:
     if target_type.nil?
-      new(records)
+      new(connection, records)
     else
-      Records.new(records, target_type: target_type, pg_source_oid: pg_source_oid)
+      Records.new(connection, records, target_type: target_type, pg_source_oid: pg_source_oid)
     end
   end
 
-  def initialize(records) # :nodoc:
+  attr_reader :connection
+
+  def initialize(connection, records) # :nodoc:
+    @connection = connection
     replace(records)
   end
 
-  # returns the total_count of search hits
+  # returns the (potentialy estimated) total count of results
+  #
+  # This is only available for paginated scopes
+  def total_fast_count
+    @total_fast_count ||= pagination_scope.fast_count
+  end
+
+  # returns the (potentialy estimated) total number of pages
+  #
+  # This is only available for paginated scopes
+  def total_fast_pages
+    @total_fast_pages ||= (total_fast_count * 1.0 / pagination_scope.per).ceil
+  end
+
+  # returns the (potentialy slow) exact total count of results
   #
-  # This is filled in when resolving a paginated scope.
-  attr_reader :total_count
+  # This is only available for paginated scopes
+  def total_count
+    @total_count ||= pagination_scope.count
+  end
 
-  # returns the total number of pages of search hits
+  # returns the (potentialy estimated) total number of pages
   #
-  # This is filled in when resolving a paginated scope. It takes
-  # into account the scope's "per" option.
-  attr_reader :total_pages
+  # This is only available for paginated scopes
+  def total_pages
+    @total_pages ||= (total_count * 1.0 / pagination_scope.per).ceil
+  end
 
   # returns the current page number in a paginated search
   #
-  # This is filled in when resolving a paginated scope.
-  attr_reader :current_page
+  # This is only available for paginated scopes
+  def current_page
+    @current_page ||= pagination_scope.page
+  end
+
+  def paginated?
+    !!@pagination_scope
+  end
 
   private
 
+  def pagination_scope
+    raise "Only available only for paginated scopes" unless paginated?
+
+    @pagination_scope
+  end
+
   def set_pagination_info(scope)
     raise ArgumentError, "per must be > 0" unless scope.per > 0
 
+    @pagination_scope = scope
+
+    # This branch is an optimization: the call to the database to count is
+    # not necessary if we know that there are not even any results on the
+    # first page.
     if scope.page <= 1 && empty?
-      # This branch is an optimization: the call to the database to count is
-      # not necessary if we know that there are not even any results on the
-      # first page.
-      @total_count  = 0
       @current_page = 1
-    else
-      sql = "SELECT COUNT(*) FROM (#{scope.order_by(nil).to_sql(pagination: false)}) simple_sql_count"
-      @total_count = ::Simple::SQL.ask(sql, *scope.args)
-      @current_page = scope.page
+      @total_count  = 0
+      @total_pages  = 1
+      @total_fast_count  = 0
+      @total_fast_pages  = 1
     end
-
-    @total_pages = (@total_count * 1.0 / scope.per).ceil
   end
 end

data/lib/simple/sql/scope/count.rb

@@ -0,0 +1,33 @@
+class Simple::SQL::Scope
+  EXACT_COUNT_THRESHOLD = 10_000
+
+  # Returns the exact count of matching records
+  def count
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
+  end
+
+  # Returns the fast count of matching records
+  #
+  # For counts larger than EXACT_COUNT_THRESHOLD this returns an estimate
+  def fast_count
+    estimate = estimated_count
+    return estimate if estimate > EXACT_COUNT_THRESHOLD
+
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.ask("SELECT COUNT(*) FROM (#{sql}) _total_count", *args)
+  end
+
+  private
+
+  def estimated_count
+    sql = order_by(nil).to_sql(pagination: false)
+    ::Simple::SQL.each("EXPLAIN #{sql}", *args) do |line|
+      next unless line =~ /\brows=(\d+)/
+
+      return Integer($1)
+    end
+
+    -1
+  end
+end

data/lib/simple/sql/scope/count_by_groups.rb

@@ -0,0 +1,79 @@
+# rubocop:disable Metrics/AbcSize
+# rubocop:disable Metrics/MethodLength
+
+class Simple::SQL::Scope
+  # Potentially fast implementation of returning all different values for a specific group.
+  #
+  # For example:
+  #
+  #   Scope.new("SELECT * FROM users").enumerate_groups("gender") -> [ "female", "male" ]
+  #
+  # It is possible to enumerate over multiple attributes, for example:
+  #
+  #   scope.enumerate_groups fragment: "ARRAY[workflow, queue]"
+  #
+  # In any case it is important that an index exists that the database can use to group
+  # by the +sql_fragment+, for example:
+  #
+  #   CREATE INDEX ix3 ON table((ARRAY[workflow, queue]));
+  #
+  def enumerate_groups(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    _, max_cost = ::Simple::SQL.costs "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+    raise "enumerate_groups: takes too much time. Make sure to create a suitable index" if max_cost > 10_000
+
+    groups = []
+    var_name = "$#{@args.count + 1}"
+    cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq", *args
+
+    while cur
+      groups << cur
+      cur = ::Simple::SQL.ask "SELECT MIN(#{sql_fragment}) FROM (#{sql}) sq"" WHERE #{sql_fragment} > #{var_name}", *args, cur
+    end
+
+    groups
+  end
+
+  def count_by(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    recs = ::Simple::SQL.all "SELECT #{sql_fragment} AS group, COUNT(*) AS count FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+    Hash[recs]
+  end
+
+  def fast_count_by(sql_fragment)
+    sql = order_by(nil).to_sql(pagination: false)
+
+    _, max_cost = ::Simple::SQL.costs "SELECT COUNT(*) FROM (#{sql}) sq GROUP BY #{sql_fragment}", *args
+
+    return count_by(sql_fragment) if max_cost < 10_000
+
+    # iterate over all groups, estimating the count for each. If the count is
+    # less than EXACT_COUNT_THRESHOLD we ask for the exact count in that and
+    # similarily sparse groups.
+    var_name = "$#{@args.count + 1}"
+
+    counts = {}
+    sparse_groups = []
+    enumerate_groups(sql_fragment).each do |group|
+      scope = ::Simple::SQL::Scope.new("SELECT * FROM (#{sql}) sq WHERE #{sql_fragment}=#{var_name}", *args, group)
+      counts[group] = scope.send(:estimated_count)
+      sparse_groups << group if estimated_count < EXACT_COUNT_THRESHOLD
+    end
+
+    # fetch exact counts in all sparse_groups
+    unless sparse_groups.empty?
+      sparse_counts = ::Simple::SQL.all <<~SQL, *args, sparse_groups
+        SELECT #{sql_fragment} AS group, COUNT(*) AS count
+        FROM (#{sql}) sq
+        WHERE #{sql_fragment} = ANY(#{var_name})
+        GROUP BY #{sql_fragment}
+      SQL
+
+      counts.update Hash[sparse_counts]
+    end
+
+    counts
+  end
+end

data/lib/simple/sql/scope.rb

@@ -3,6 +3,8 @@
 require_relative "scope/filters.rb"
 require_relative "scope/order.rb"
 require_relative "scope/pagination.rb"
+require_relative "scope/count.rb"
+require_relative "scope/count_by_groups.rb"
 
 # The Simple::SQL::Scope class helps building scopes; i.e. objects
 # that start as a quite basic SQL query, and allow one to add
@@ -24,11 +26,11 @@ class Simple::SQL::Scope
   #
   #     Simple::SQL::Scope.new(table: "mytable", select: "*", where: { id: 1, foo: "bar" }, order_by: "id desc")
   #
-  def initialize(sql)
+  def initialize(sql, args = [])
     expect! sql => [String, Hash]
 
     @sql     = nil
-    @args    = []
+    @args    = args
     @filters = []
 
     case sql

data/lib/simple/sql/version.rb

@@ -1,5 +1,5 @@
 module Simple
   module SQL
-    VERSION = "0.5.0"
+    VERSION = "0.5.2"
   end
 end

data/lib/simple/sql.rb

@@ -3,17 +3,14 @@ require "logger"
 require "expectation"
 
 require_relative "sql/version"
+require_relative "sql/fragment"
 require_relative "sql/helpers"
-
 require_relative "sql/result"
 require_relative "sql/config"
 require_relative "sql/logging"
 require_relative "sql/scope"
 require_relative "sql/connection_adapter"
 require_relative "sql/connection"
-require_relative "sql/reflection"
-require_relative "sql/insert"
-require_relative "sql/duplicate"
 
 module Simple
   # The Simple::SQL module
@@ -21,7 +18,10 @@ module Simple
     extend self
 
     extend Forwardable
-    delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify] => :default_connection
+    delegate [:ask, :all, :each, :exec, :locked, :print, :transaction, :wait_for_notify, :costs] => :default_connection
+    delegate [:reflection] => :default_connection
+    delegate [:duplicate] => :default_connection
+    delegate [:insert] => :default_connection
 
     delegate [:logger, :logger=] => ::Simple::SQL::Logging
 
@@ -35,6 +35,11 @@ module Simple
       Connection.create(database_url)
     end
 
+    # deprecated
+    def configuration
+      Config.parse_url(Config.determine_url)
+    end
+
     # -- default connection ---------------------------------------------------
 
     DEFAULT_CONNECTION_KEY = :"Simple::SQL.default_connection"

data/spec/simple/sql/count_by_groups_spec.rb

@@ -0,0 +1,44 @@
+require "spec_helper"
+
+describe "Simple::SQL::Scope#count_by" do
+  let!(:users)        { 1.upto(10).map { |i| create(:user, role_id: i) } }
+  let(:all_role_ids)  { SQL.all("SELECT DISTINCT role_id FROM users") }
+  let(:scope)         { SQL::Scope.new("SELECT * FROM users") }
+
+  describe "enumerate_groups" do
+    it "returns all groups" do
+      expect(scope.enumerate_groups("role_id")).to contain_exactly(*all_role_ids)
+      expect(scope.where("role_id < 4").enumerate_groups("role_id")).to contain_exactly(*(1.upto(3).to_a))
+    end
+  end
+
+  describe "count_by" do
+    it "counts all groups" do
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+
+      expect(scope.count_by("role_id")).to include(1 => 4)
+      expect(scope.count_by("role_id")).to include(2 => 1)
+      expect(scope.count_by("role_id").keys).to contain_exactly(*all_role_ids)
+    end
+  end
+
+  describe "fast_count_by" do
+    before do
+      # 10_000 is chosen "magically". It is large enough to switch to the fast algorithm,
+      # but 
+      allow(::Simple::SQL).to receive(:costs).and_return([0, 10_000])
+    end
+    
+    it "counts all groups" do
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+      create(:user, role_id: 1)
+
+      expect(scope.fast_count_by("role_id")).to include(1 => 4)
+      expect(scope.fast_count_by("role_id")).to include(2 => 1)
+      expect(scope.fast_count_by("role_id").keys).to contain_exactly(*all_role_ids)
+    end
+  end
+end

data/spec/simple/sql/count_spec.rb

@@ -0,0 +1,29 @@
+require "spec_helper"
+
+describe "Simple::SQL::Scope#count" do
+  let!(:users)      { 1.upto(USER_COUNT).map { create(:user) } }
+  let(:min_user_id) { SQL.ask "SELECT min(id) FROM users" }
+  let(:scope)       { SQL::Scope.new("SELECT * FROM users") }
+
+  describe "exact count" do
+    it "counts" do
+      expect(scope.count).to eq(USER_COUNT)
+    end
+
+    it "evaluates conditions" do
+      expect(scope.where("id < $1", min_user_id).count).to eq(0)
+      expect(scope.where("id <= $1", min_user_id).count).to eq(1)
+    end
+  end
+
+  describe "fast count" do
+    it "counts" do
+      expect(scope.fast_count).to eq(USER_COUNT)
+    end
+
+    it "evaluates conditions" do
+      expect(scope.where("id < $1", min_user_id).fast_count).to eq(0)
+      expect(scope.where("id <= $1", min_user_id).fast_count).to eq(1)
+    end
+  end
+end

data/spec/simple/sql/logging_spec.rb

@@ -0,0 +1,16 @@
+require "spec_helper"
+
+describe "Simple::SQL logging" do
+  context 'when running a slow query' do
+    before do
+      SQL::Logging.slow_query_treshold = 0.05
+    end
+    after do
+      SQL::Logging.slow_query_treshold = nil
+    end
+
+    it "does not crash" do
+      SQL.ask "SELECT pg_sleep(0.1)"
+    end
+  end
+end

data/spec/simple/sql/reflection_spec.rb

@@ -1,23 +1,23 @@
 require "spec_helper"
 
-describe "Simple::SQL::Reflection" do
+describe "Simple::SQL.reflection" do
   describe ".columns" do
     it "returns the columns of a table in the public schema" do
-      expect(SQL::Reflection.columns("users")).to include("first_name")
+      expect(SQL.reflection.columns("users")).to include("first_name")
     end
 
     it "returns the columns of a table in a non-'public' schema" do
-      expect(SQL::Reflection.columns("information_schema.tables")).to include("table_name")
+      expect(SQL.reflection.columns("information_schema.tables")).to include("table_name")
     end
   end
 
   describe ".tables" do
     it "returns the tables in the public schema" do
-      expect(SQL::Reflection.tables).to include("public.users")
+      expect(SQL.reflection.tables).to include("public.users")
     end
 
     it "returns tables in a non-'public' schema" do
-      expect(SQL::Reflection.tables(schema: "information_schema")).to include("information_schema.tables")
+      expect(SQL.reflection.tables(schema: "information_schema")).to include("information_schema.tables")
     end
   end
 end

data/spec/simple/sql/result_count_spec.rb

@@ -0,0 +1,57 @@
+require "spec_helper"
+
+describe "Simple::SQL::Result counts" do
+  let!(:users)          { 1.upto(USER_COUNT).map { create(:user) } }
+  let(:min_user_id)     { SQL.ask "SELECT min(id) FROM users" }
+  let(:scope)           { SQL::Scope.new("SELECT * FROM users") }
+  let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
+
+  describe "exact counting" do
+    it "counts" do
+      result = SQL.all(paginated_scope)
+      expect(result.total_count).to eq(USER_COUNT)
+      expect(result.total_pages).to eq(USER_COUNT)
+      expect(result.current_page).to eq(1)
+    end
+  end
+  
+  describe "fast counting" do
+    it "counts fast" do
+      result = SQL.all(paginated_scope)
+
+      expect(result.total_fast_count).to eq(USER_COUNT)
+      expect(result.total_fast_pages).to eq(USER_COUNT)
+      expect(result.current_page).to eq(1)
+    end
+  end
+
+  context 'when running with a non-paginated paginated_scope' do
+    it "raises errors" do
+      result = SQL.all(scope)
+
+      expect { result.total_count }.to raise_error(RuntimeError)
+      expect { result.total_pages }.to raise_error(RuntimeError)
+      expect { result.current_page }.to raise_error(RuntimeError)
+      expect { result.total_fast_count }.to raise_error(RuntimeError)
+      expect { result.total_fast_pages }.to raise_error(RuntimeError)
+    end
+  end
+
+
+  context 'when running with an empty, paginated paginated_scope' do
+    let(:scope)           { SQL::Scope.new("SELECT * FROM users WHERE FALSE") }
+    let(:paginated_scope) { scope.paginate(per: 1, page: 1) }
+
+    it "returns correct results" do
+      result = SQL.all(paginated_scope)
+
+      expect(result.total_count).to eq(0)
+      expect(result.total_pages).to eq(1)
+
+      expect(result.total_fast_count).to eq(0)
+      expect(result.total_fast_pages).to eq(1)
+
+      expect(result.current_page).to eq(1)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -14,7 +14,9 @@ Dir.glob("./spec/support/**/*.rb").sort.each { |path| load path }
 require "simple/sql"
 
 unless ENV["USE_ACTIVE_RECORD"]
-  Simple::SQL.connect!
+  database_url = Simple::SQL::Config.determine_url
+
+  Simple::SQL.connect! database_url
   Simple::SQL.ask "DELETE FROM users"
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple-sql
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.2
 platform: ruby
 authors:
 - radiospiel
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-04 00:00:00.000000000 Z
+date: 2019-03-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg_array_parser
@@ -202,22 +202,25 @@ files:
 - lib/simple/sql/config.rb
 - lib/simple/sql/connection.rb
 - lib/simple/sql/connection/active_record_connection.rb
+- lib/simple/sql/connection/duplicate.rb
+- lib/simple/sql/connection/insert.rb
 - lib/simple/sql/connection/raw_connection.rb
+- lib/simple/sql/connection/reflection.rb
 - lib/simple/sql/connection_adapter.rb
-- lib/simple/sql/duplicate.rb
 - lib/simple/sql/formatting.rb
+- lib/simple/sql/fragment.rb
 - lib/simple/sql/helpers.rb
 - lib/simple/sql/helpers/decoder.rb
 - lib/simple/sql/helpers/encoder.rb
 - lib/simple/sql/helpers/immutable.rb
 - lib/simple/sql/helpers/row_converter.rb
-- lib/simple/sql/insert.rb
 - lib/simple/sql/logging.rb
-- lib/simple/sql/reflection.rb
 - lib/simple/sql/result.rb
 - lib/simple/sql/result/association_loader.rb
 - lib/simple/sql/result/records.rb
 - lib/simple/sql/scope.rb
+- lib/simple/sql/scope/count.rb
+- lib/simple/sql/scope/count_by_groups.rb
 - lib/simple/sql/scope/filters.rb
 - lib/simple/sql/scope/order.rb
 - lib/simple/sql/scope/pagination.rb
@@ -232,11 +235,15 @@ files:
 - spec/simple/sql/associations_spec.rb
 - spec/simple/sql/config_spec.rb
 - spec/simple/sql/conversion_spec.rb
+- spec/simple/sql/count_by_groups_spec.rb
+- spec/simple/sql/count_spec.rb
 - spec/simple/sql/duplicate_spec.rb
 - spec/simple/sql/duplicate_unique_spec.rb
 - spec/simple/sql/each_spec.rb
 - spec/simple/sql/insert_spec.rb
+- spec/simple/sql/logging_spec.rb
 - spec/simple/sql/reflection_spec.rb
+- spec/simple/sql/result_count_spec.rb
 - spec/simple/sql/scope_spec.rb
 - spec/simple/sql/version_spec.rb
 - spec/simple/sql_locked_spec.rb
@@ -267,8 +274,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: SQL with a simple interface
@@ -279,11 +285,15 @@ test_files:
 - spec/simple/sql/associations_spec.rb
 - spec/simple/sql/config_spec.rb
 - spec/simple/sql/conversion_spec.rb
+- spec/simple/sql/count_by_groups_spec.rb
+- spec/simple/sql/count_spec.rb
 - spec/simple/sql/duplicate_spec.rb
 - spec/simple/sql/duplicate_unique_spec.rb
 - spec/simple/sql/each_spec.rb
 - spec/simple/sql/insert_spec.rb
+- spec/simple/sql/logging_spec.rb
 - spec/simple/sql/reflection_spec.rb
+- spec/simple/sql/result_count_spec.rb
 - spec/simple/sql/scope_spec.rb
 - spec/simple/sql/version_spec.rb
 - spec/simple/sql_locked_spec.rb

data/lib/simple/sql/duplicate.rb

@@ -1,96 +0,0 @@
-# rubocop:disable Style/StructInheritance
-
-module Simple
-  module SQL
-    unless defined?(Fragment)
-
-      class Fragment < Struct.new(:to_sql)
-      end
-
-    end
-
-    def fragment(str)
-      Fragment.new(str)
-    end
-
-    #
-    # Creates duplicates of record in a table.
-    #
-    # This method handles timestamp columns (these will be set to the current
-    # time) and primary keys (will be set to NULL.) You can pass in overrides
-    # as a third argument for specific columns.
-    #
-    # Parameters:
-    #
-    # - ids: (Integer, Array<Integer>) primary key ids
-    # - overrides: Hash[column_names => SQL::Fragment]
-    #
-    def duplicate(table, ids, overrides = {})
-      ids = Array(ids)
-      return [] if ids.empty?
-
-      Duplicator.new(table, overrides).call(ids)
-    end
-
-    class Duplicator
-      attr_reader :table_name, :custom_overrides
-
-      def initialize(table_name, overrides)
-        @table_name = table_name
-        @custom_overrides = validated_overrides(overrides)
-      end
-
-      def call(ids)
-        Simple::SQL.all query, ids
-      rescue PG::UndefinedColumn => e
-        raise ArgumentError, e.message
-      end
-
-      private
-
-      # This stringify all keys of the overrides hash, and verifies that
-      # all values in there are SQL::Fragments
-      def validated_overrides(overrides)
-        overrides.inject({}) do |hsh, (key, value)|
-          unless value.is_a?(Fragment)
-            raise ArgumentError, "Pass in override values via SQL.fragment (for #{value.inspect})"
-          end
-
-          hsh.update key.to_s => value.to_sql
-        end
-      end
-
-      def timestamp_overrides
-        Reflection.timestamp_columns(table_name).inject({}) do |hsh, column|
-          hsh.update column => "now() AS #{column}"
-        end
-      end
-
-      def copy_columns
-        (Reflection.columns(table_name) - Reflection.primary_key_columns(table_name)).inject({}) do |hsh, column|
-          hsh.update column => column
-        end
-      end
-
-      # build SQL query
-      def query
-        sources = {}
-
-        sources.update copy_columns
-        sources.update timestamp_overrides
-        sources.update custom_overrides
-
-        # convert into an Array, to make sure that keys and values aka firsts
-        # and lasts are always in the correct order.
-        sources = sources.to_a
-
-        <<~SQL
-          INSERT INTO #{table_name}(#{sources.map(&:first).join(', ')})
-            SELECT #{sources.map(&:last).join(', ')}
-            FROM #{table_name}
-            WHERE id = ANY($1) RETURNING id
-        SQL
-      end
-    end
-  end
-end

data/lib/simple/sql/insert.rb

@@ -1,87 +0,0 @@
-# rubocop:disable Metrics/AbcSize
-# rubocop:disable Metrics/LineLength
-# rubocop:disable Layout/AlignHash
-
-module Simple
-  module SQL
-    #
-    # - table_name - the name of the table
-    # - records - a single hash of attributes or an array of hashes of attributes
-    # - on_conflict - uses a postgres ON CONFLICT clause to ignore insert conflicts if true
-    #
-    def insert(table, records, on_conflict: nil, into: nil)
-      if records.is_a?(Hash)
-        insert_many(table, [records], on_conflict, into).first
-      else
-        insert_many(table, records, on_conflict, into)
-      end
-    end
-
-    private
-
-    def insert_many(table, records, on_conflict, into)
-      return [] if records.empty?
-
-      inserter = Inserter.create(table_name: table.to_s, columns: records.first.keys, on_conflict: on_conflict, into: into)
-      inserter.insert(records: records)
-    end
-
-    class Inserter
-      SQL = ::Simple::SQL
-
-      @@inserters = {}
-
-      def self.create(table_name:, columns:, on_conflict:, into:)
-        expect! on_conflict => CONFICT_HANDLING.keys
-
-        @@inserters[[table_name, columns, on_conflict, into]] ||= new(table_name: table_name, columns: columns, on_conflict: on_conflict, into: into)
-      end
-
-      #
-      # - table_name - the name of the table
-      # - columns - name of columns, as Array[String] or Array[Symbol]
-      #
-      def initialize(table_name:, columns:, on_conflict:, into:)
-        raise ArgumentError, "Cannot insert a record without attributes" if columns.empty?
-
-        @columns = columns
-        @into = into
-
-        cols = []
-        vals = []
-
-        cols += columns
-        vals += columns.each_with_index.map { |_, idx| "$#{idx + 1}" }
-
-        timestamp_columns = SQL::Reflection.timestamp_columns(table_name) - columns.map(&:to_s)
-
-        cols += timestamp_columns
-        vals += timestamp_columns.map { "now()" }
-
-        returning = into ? "*" : "id"
-
-        @sql = "INSERT INTO #{table_name} (#{cols.join(',')}) VALUES(#{vals.join(',')}) #{confict_handling(on_conflict)} RETURNING #{returning}"
-      end
-
-      CONFICT_HANDLING = {
-        nil      => "",
-        :nothing => "ON CONFLICT DO NOTHING",
-        :ignore  => "ON CONFLICT DO NOTHING"
-      }
-
-      def confict_handling(on_conflict)
-        CONFICT_HANDLING.fetch(on_conflict) do
-          raise(ArgumentError, "Invalid on_conflict value #{on_conflict.inspect}")
-        end
-      end
-
-      def insert(records:)
-        SQL.transaction do
-          records.map do |record|
-            SQL.ask @sql, *record.values_at(*@columns), into: @into
-          end
-        end
-      end
-    end
-  end
-end

data/lib/simple/sql/reflection.rb

@@ -1,106 +0,0 @@
-module Simple
-  module SQL
-    module Reflection
-      extend self
-
-      extend Forwardable
-      delegate [:ask, :all, :records, :record] => ::Simple::SQL
-
-      def tables(schema: "public")
-        table_info(schema: schema).keys
-      end
-
-      def primary_key_columns(table_name)
-        all <<~SQL, table_name
-          SELECT pg_attribute.attname
-          FROM   pg_index
-          JOIN   pg_attribute ON pg_attribute.attrelid = pg_index.indrelid AND pg_attribute.attnum = ANY(pg_index.indkey)
-          WHERE  pg_index.indrelid = $1::regclass
-          AND    pg_index.indisprimary;
-        SQL
-      end
-
-      TIMESTAMP_COLUMN_NAMES = %w(inserted_at created_at updated_at)
-
-      # timestamp_columns are columns that will be set automatically after
-      # inserting or updating a record. This includes:
-      #
-      # - inserted_at (for Ecto)
-      # - created_at (for ActiveRecord)
-      # - updated_at (for Ecto and ActiveRecord)
-      def timestamp_columns(table_name)
-        columns(table_name) & TIMESTAMP_COLUMN_NAMES
-      end
-
-      def columns(table_name)
-        column_info(table_name).keys
-      end
-
-      def table_info(schema: "public")
-        recs = all <<~SQL, schema, into: Hash
-          SELECT table_schema || '.' || table_name AS name, *
-          FROM information_schema.tables
-          WHERE table_schema=$1
-        SQL
-        records_by_attr(recs, :name)
-      end
-
-      def column_info(table_name)
-        @column_info ||= {}
-        @column_info[table_name] ||= _column_info(table_name)
-      end
-
-      private
-
-      def _column_info(table_name)
-        schema, table_name = parse_table_name(table_name)
-        recs = all <<~SQL, schema, table_name, into: Hash
-          SELECT
-            column_name AS name,
-            *
-          FROM information_schema.columns
-          WHERE table_schema=$1 AND table_name=$2
-        SQL
-
-        records_by_attr(recs, :column_name)
-      end
-
-      def parse_table_name(table_name)
-        p1, p2 = table_name.split(".", 2)
-        if p2
-          [p1, p2]
-        else
-          ["public", p1]
-        end
-      end
-
-      def records_by_attr(records, attr)
-        records.inject({}) do |hsh, record|
-          record.reject! { |_k, v| v.nil? }
-          hsh.update record[attr] => OpenStruct.new(record)
-        end
-      end
-
-      public
-
-      def looked_up_pg_classes
-        @looked_up_pg_classes ||= Hash.new { |hsh, key| hsh[key] = _lookup_pg_class(key) }
-      end
-
-      def lookup_pg_class(oid)
-        looked_up_pg_classes[oid]
-      end
-
-      private
-
-      def _lookup_pg_class(oid)
-        ::Simple::SQL.ask <<~SQL, oid
-          SELECT nspname AS schema, relname AS host_table
-          FROM pg_class
-          JOIN pg_namespace ON pg_namespace.oid=pg_class.relnamespace
-          WHERE pg_class.oid=$1
-        SQL
-      end
-    end
-  end
-end