mini_sql 0.2.3 → 1.0.1

data/Gemfile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 source "https://rubygems.org"
 
-git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
 # Specify your gem's dependencies in mini_sql.gemspec
 gemspec

data/Guardfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 guard :minitest do
   watch(%r{^test/(.*)_test\.rb$})
   watch(%r{^lib/(.*/)?([^/]+)\.rb$})     { |m| "test/#{m[1]}#{m[2]}_test.rb" }

data/README.md

@@ -32,11 +32,28 @@ conn.query("select 1 id, 'bob' name").each do |user|
   puts user.id # 1
 end
 
+# extend result objects with additional method
+module ProductDecorator
+  def amount_price
+    price * quantity
+  end
+end
+
+conn.query_decorator(ProductDecorator, "select 20 price, 3 quantity").each do |user|
+  puts user.amount_price # 60
+end
+
 p conn.query_single('select 1 union select 2')
 # [1,2]
 
 p conn.query_hash('select 1 as a, 2 as b union select 3, 4')
 # [{"a" => 1, "b"=> 1},{"a" => 3, "b" => 4}
+ 
+p conn.query_array("select 1 as a, '2' as b union select 3, 'e'")
+# [[1, '2'], [3, 'e']]
+ 
+p conn.query_array("select 1 as a, '2' as b union select 3, 'e'").to_h
+# {1 => '2', 3 => 'e'}
 ```
 
 ## The query builder
@@ -63,9 +80,18 @@ end
 The builder allows for `order_by`, `where`, `select`, `set`, `limit`, `join`, `left_join` and `offset`.
 
 ## Is it fast?
-
 Yes, it is very fast. See benchmarks in [the bench directory](https://github.com/discourse/mini_sql/tree/master/bench).
 
+**Comparison mini_sql methods**
+```
+query_array     1351.6 i/s
+      query      963.8 i/s - 1.40x  slower
+ query_hash      787.4 i/s - 1.72x  slower
+
+query_single('select id from topics limit 1000')             2368.9 i/s
+ query_array('select id from topics limit 1000').flatten     1350.1 i/s - 1.75x  slower
+```
+
 As a rule it will outperform similar naive PG code while remaining safe.
 
 ```ruby
@@ -98,6 +124,68 @@ MiniSql is careful to always clear results as soon as possible.
 
 MiniSql's default type mapper prefers treating `timestamp without time zone` columns as utc. This is done to ensure widest amount of compatability and is a departure from the default in the PG 1.0 gem. If you wish to amend behavior feel free to pass in a custom type_map.
 
+## Custom type maps
+
+When using Postgres, native type mapping implementation is used. This is roughly
+implemented as:
+
+```ruby
+type_map ||= PG::BasicTypeMapForResults.new(conn)
+# additional specific decoders
+```
+
+The type mapper instansitated once on-demand at boot and reused by all mini_sql connections.
+
+Initializing the basic type map for Postgres can be a costly operation. You may
+wish to amend the type mapper so for example you only return strings:
+
+```
+# maybe you do not want Integer
+p cnn.query("select a 1").first.a
+"1"
+```
+
+To specify a different type mapper for your results use:
+
+```
+MiniSql::Connections.get(pg_connection, type_map: custom_type_map)
+```
+
+In the case of Rails you can opt to use the type mapper Rails uses with:
+
+```
+pg_cnn = ActiveRecord::Base.connection.raw_connection
+mini_sql_cnn = MiniSql::Connection.get(pg_cnn, type_map: pg_cnn.type_map_for_results)
+```
+
+Note the type mapper for Rails may miss some of the mapping MiniSql ships with such as `IPAddr`, MiniSql is also careful to use the very efficient TimestampUtc decoders where available.
+
+## Streaming support
+
+In some exceptional cases you may want to stream results directly from the database. This enables selection of 100s of thousands of rows with limited memory impact.
+
+Two interfaces exists for this:
+
+`query_each` : which can be used to get materialized objects  
+`query_each_hash` : which can be used to iterate through Hash objects
+
+Usage:
+
+```ruby
+mini_sql_cnn.query_each("SELECT * FROM tons_of_cows limit :limit", limit: 1_000_000) do |row|
+  puts row.cow_name
+  puts row.cow_age
+end
+
+mini_sql_cnn.query_each_hash("SELECT * FROM one_million_cows") do |row|
+  puts row["cow_name"]
+  puts row["cow_age"]
+end
+```
+
+Note, in Postgres streaming is going to be slower than non-streaming options due to internal implementation in the pq gem, each row gets a full result object and additional bookkeeping is needed. Only use it if you need to optimize memory usage.
+
+Streaming support is only implemented in the postgres backend at the moment, PRs welcome to add to other backends.
 
 ## I want more features!
 

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rake/testtask"
 
@@ -13,4 +15,4 @@ Rake::TestTask.new(:test) do |t|
   t.test_files = FileList[test_glob]
 end
 
-task :default => :test
+task default: :test

data/bench/timestamp_perf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/inline'
 
 gemfile do
@@ -21,8 +23,8 @@ require 'benchmark/ips'
 require 'mini_sql'
 
 ActiveRecord::Base.establish_connection(
-  :adapter => "postgresql",
-  :database => "test_db"
+  adapter: "postgresql",
+  database: "test_db"
 )
 
 Sequel.default_timezone = :utc
@@ -47,20 +49,20 @@ SQL
 class Timestamp < ActiveRecord::Base
 end
 
-class TimestampSequel< Sequel::Model(:timestamps)
+class TimestampSequel < Sequel::Model(:timestamps)
 end
 
-
 Timestamp.transaction do
   stamps = {
   }
   Timestamp.columns.each do |c|
-    stamps[c.name.to_sym] = case c.type
-                           when :integer then 1
-                           when :datetime then Time.now
-                           when :boolean then false
-                           else "HELLO WORLD" * 2
-                           end
+    stamps[c.name.to_sym] =
+      case c.type
+      when :integer then 1
+      when :datetime then Time.now
+      when :boolean then false
+      else "HELLO WORLD" * 2
+      end
   end
 
   1000.times do |id|
@@ -71,7 +73,7 @@ end
 
 $conn = ActiveRecord::Base.connection.raw_connection
 
-def ar_pluck_times(l=1000)
+def ar_pluck_times(l = 1000)
   s = +""
   Timestamp.limit(l).order(:id).pluck(:time1, :time2).each do |time1, time2|
     s << time1.to_f.to_s
@@ -80,7 +82,7 @@ def ar_pluck_times(l=1000)
   s
 end
 
-def ar_select_times(l=1000)
+def ar_select_times(l = 1000)
   s = +""
   Timestamp.limit(l).order(:id).select(:time1, :time2).each do |t|
     s << t.time1.to_f.to_s
@@ -91,7 +93,7 @@ end
 
 $mini_sql = MiniSql::Connection.new($conn)
 
-def pg_times_params(l=1000)
+def pg_times_params(l = 1000)
   s = +""
   # use the safe pattern here
   r = $conn.async_exec_params(-"select time1, time2 from timestamps order by id limit $1", [l])
@@ -110,7 +112,7 @@ def pg_times_params(l=1000)
   s
 end
 
-def pg_times(l=1000)
+def pg_times(l = 1000)
   s = +""
   # use the safe pattern here
   r = $conn.async_exec("select time1, time2 from timestamps order by id limit #{l}")
@@ -129,7 +131,7 @@ def pg_times(l=1000)
   s
 end
 
-def mini_sql_times(l=1000)
+def mini_sql_times(l = 1000)
   s = +""
   $mini_sql.query(-"select time1, time2 from timestamps order by id limit ?", l).each do |t|
     s << t.time1.to_f.to_s
@@ -138,7 +140,7 @@ def mini_sql_times(l=1000)
   s
 end
 
-def sequel_times(l=1000)
+def sequel_times(l = 1000)
   s = +""
   TimestampSequel.limit(l).order(:id).select(:time1, :time2).each do |t|
     s << t.time1.to_f.to_s
@@ -147,7 +149,7 @@ def sequel_times(l=1000)
   s
 end
 
-def sequel_pluck_times(l=1000)
+def sequel_pluck_times(l = 1000)
   s = +""
   TimestampSequel.limit(l).order(:id).select_map([:time1, :time2]).each do |t|
     s << t[0].to_f.to_s
@@ -156,7 +158,7 @@ def sequel_pluck_times(l=1000)
   s
 end
 
-def sequel_raw_times(l=1000)
+def sequel_raw_times(l = 1000)
   s = +""
   DB[-"select time1, time2 from timestamps order by id limit ?", l].map([:time1, :time2]).each do |time1, time2|
     s << time1.to_f.to_s
@@ -166,13 +168,13 @@ def sequel_raw_times(l=1000)
 end
 
 # usage is not really recommended but just to compare to pluck lets have it
-def mini_sql_times_single(l=1000)
+def mini_sql_times_single(l = 1000)
   s = +""
   i = 0
   r = $mini_sql.query_single(-"select time1, time2 from timestamps order by id limit ?", l)
   while i < r.length
     s << r[i].to_f.to_s
-    s << r[i+1].to_f.to_s
+    s << r[i + 1].to_f.to_s
     i += 2
   end
   s
@@ -190,7 +192,6 @@ end
 #   s
 # end
 
-
 results = [
   ar_select_times,
   ar_pluck_times,

data/bench/topic_mysql_perf.rb

@@ -91,7 +91,6 @@ end
 class TopicSequel < Sequel::Model(:topics)
 end
 
-
 Topic.transaction do
   topic = {
   }
@@ -179,7 +178,7 @@ def mini_sql_title_id_query_single
   r = $mini_sql.query_single(-"select id, title from topics order by id limit 1000")
   while i < r.length
     s << r[i].to_s
-    s << r[i+1]
+    s << r[i + 1]
     i += 2
   end
   s
@@ -197,7 +196,6 @@ results = [
 
 exit(-1) unless results.uniq.length == 1
 
-
 Benchmark.ips do |r|
   r.report("ar select title id") do |n|
     while n > 0
@@ -244,8 +242,6 @@ Benchmark.ips do |r|
   r.compare!
 end
 
-
-
 def wide_topic_ar
   Topic.first
 end
@@ -301,10 +297,8 @@ end
 # ar select title id pluck:      317.1 i/s - 1.53x  slower
 #   ar select title id:      102.3 i/s - 4.74x  slower
 
-
 # Comparison:
 #  wide topic mini sql:     6768.7 i/s
 #     wide topic mysql:     6063.9 i/s - same-ish: difference falls within error
 #    wide topic sequel:     4908.6 i/s - same-ish: difference falls within error
 #        wide topic ar:     2630.2 i/s - 2.57x  slower
-

data/bench/topic_perf.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'bundler/inline'
 
 gemfile do
@@ -12,6 +14,7 @@ gemfile do
   gem 'sequel', github: 'jeremyevans/sequel'
   gem 'sequel_pg', github: 'jeremyevans/sequel_pg', require: 'sequel'
   gem 'swift-db-postgres', github: 'deepfryed/swift-db-postgres'
+  gem 'draper'
 end
 
 require 'sequel'
@@ -89,7 +92,6 @@ end
 class TopicSequel < Sequel::Model(:topics)
 end
 
-
 Topic.transaction do
   topic = {
   }
@@ -184,7 +186,7 @@ def mini_sql_title_id_query_single
   r = $mini_sql.query_single(-"select id, title from topics order by id limit 1000")
   while i < r.length
     s << r[i].to_s
-    s << r[i+1]
+    s << r[i + 1]
     i += 2
   end
   s
@@ -193,8 +195,8 @@ end
 # connects over unix socket
 $swift = Swift::DB::Postgres.new(db: "test_db")
 
-def swift_select_title_id(l=1000)
-  s = ""
+def swift_select_title_id(l = 1000)
+  s = +''
   i = 0
   r = $swift.execute("select id, title from topics order by id limit 1000")
   while i < r.selected_rows
@@ -218,6 +220,174 @@ results = [
 
 exit(-1) unless results.uniq.length == 1
 
+# https://github.com/drapergem/draper
+class TopicDraper < Draper::Decorator
+  delegate :id
+
+  def title_bang
+    object.title + '!!!'
+  end
+end
+
+# https://ruby-doc.org/stdlib-2.5.1/libdoc/delegate/rdoc/SimpleDelegator.html
+class TopicSimpleDelegator < SimpleDelegator
+  def title_bang
+    title + '!!!'
+  end
+end
+
+class TopicDecoratorSequel < TopicSequel
+  def title_bang
+    title + '!!!'
+  end
+end
+
+class TopicArModel < Topic
+  def title_bang
+    title + '!!!'
+  end
+end
+
+module TopicDecorator
+  def title_bang
+    title + '!!!'
+  end
+end
+
+Benchmark.ips do |r|
+  r.report('query_decorator') do |n|
+    while n > 0
+      $mini_sql.query_decorator(TopicDecorator, 'select id, title from topics order by id limit 1000').each do |obj|
+        obj.title_bang
+        obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('extend') do |n|
+    while n > 0
+      $mini_sql.query('select id, title from topics order by id limit 1000').each do |obj|
+        d_obj = obj.extend(TopicDecorator)
+        d_obj.title_bang
+        d_obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('draper') do |n|
+    while n > 0
+      $mini_sql.query('select id, title from topics order by id limit 1000').each do |obj|
+        d_obj = TopicDraper.new(obj)
+        d_obj.title_bang
+        d_obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('simple_delegator') do |n|
+    while n > 0
+      $mini_sql.query('select id, title from topics order by id limit 1000').each do |obj|
+        d_obj = TopicSimpleDelegator.new(obj)
+        d_obj.title_bang
+        d_obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('query') do |n|
+    while n > 0
+      $mini_sql.query('select id, title from topics order by id limit 1000').each do |obj|
+        obj.title + '!!!'
+        obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('ar model') do |n|
+    while n > 0
+      TopicArModel.limit(1000).order(:id).select(:id, :title).each do |obj|
+        obj.title_bang
+        obj.id
+      end
+      n -= 1
+    end
+  end
+  r.report('sequel model') do |n|
+    while n > 0
+      TopicDecoratorSequel.limit(1000).order(:id).select(:id, :title).each do |obj|
+        obj.title_bang
+        obj.id
+      end
+      n -= 1
+    end
+  end
+
+  r.compare!
+end
+
+# Comparison:
+#                query:      828.4 i/s
+#      query_decorator:      819.3 i/s - same-ish: difference falls within error
+#         sequel model:      672.4 i/s - 1.23x  slower
+#               extend:      519.4 i/s - 1.59x  slower
+#     simple_delegator:      496.8 i/s - 1.67x  slower
+#               draper:      416.2 i/s - 1.99x  slower
+#             ar model:      113.4 i/s - 7.30x  slower
+
+Benchmark.ips do |r|
+  r.report('query_hash') do |n|
+    while n > 0
+      $mini_sql.query_hash('select id, title from topics order by id limit 1000').each do |hash|
+        [hash['id'], hash['title']]
+      end
+      n -= 1
+    end
+  end
+  r.report('query_array') do |n|
+    while n > 0
+      $mini_sql.query_array('select id, title from topics order by id limit 1000').each do |id, title|
+        [id, title]
+      end
+      n -= 1
+    end
+  end
+  r.report('query') do |n|
+    while n > 0
+      $mini_sql.query('select id, title from topics order by id limit 1000').each do |obj|
+        [obj.id, obj.title]
+      end
+      n -= 1
+    end
+  end
+
+  r.compare!
+end
+
+# Comparison:
+#         query_array:     1351.6 i/s
+#               query:      963.8 i/s - 1.40x  slower
+#          query_hash:      787.4 i/s - 1.72x  slower
+
+Benchmark.ips do |r|
+  r.report('query_single') do |n|
+    while n > 0
+      $mini_sql.query_single('select id from topics order by id limit 1000')
+      n -= 1
+    end
+  end
+  r.report('query_array') do |n|
+    while n > 0
+      $mini_sql.query_array('select id from topics order by id limit 1000').flatten
+      n -= 1
+    end
+  end
+
+  r.compare!
+end
+
+# Comparison:
+#        query_single:     2368.9 i/s
+#         query_array:     1350.1 i/s - 1.75x  slower
 
 Benchmark.ips do |r|
   r.report("ar select title id") do |n|
@@ -271,8 +441,6 @@ Benchmark.ips do |r|
   r.compare!
 end
 
-
-
 def wide_topic_ar
   Topic.first
 end
@@ -320,7 +488,6 @@ Benchmark.ips do |r|
   r.compare!
 end
 
-
 # Comparison:
 #   pg select title id:     1519.7 i/s
 # mini_sql query_single title id:     1335.0 i/s - 1.14x  slower
@@ -338,10 +505,7 @@ end
 #    wide topic sequel:     5696.8 i/s - 1.31x  slower
 #        wide topic ar:     2515.0 i/s - 2.97x  slower
 
-
-
 # to run deep analysis run
 # MemoryProfiler.report do
 #   ar
 # end.pretty_print
-