mini_sql 0.2.5 → 0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c8f5bcd09064cc2353fb1c2f172845cb738c72f9da2ff44786fdf1f119f7c36c
-  data.tar.gz: 1d7abcda16fa7d972aa0ae6b64bc1f23532feda03daf0098dcbff00f1e2e0d49
+  metadata.gz: 3ea4564f53848c314c76c45db0c2af652c1e128ebfc1ba52d6ae11b3799149f4
+  data.tar.gz: ecdfdc07fdf11e60338bc2ceb0415cbcc027720c5bba645281217a51befc2402
 SHA512:
-  metadata.gz: a84dca1674849011cd77e796046d0ece93e7952603470a51b959332180a1533fb1d15f1d3e3b69cb6f5bcc2f43e4f80c6902d85c0709443bc9004a15bc721921
-  data.tar.gz: 4433947c712bcf6c9346fa2b0df792bc95c7fdaa453e5ae19b649241c24ba7ab3b57397cb66114b68ba4e622dfcc6242e6631beed129f5a8cd08bc618e13381d
+  metadata.gz: 76a040187f4932dc9fbc80ae3323633a38462753caec47233dbfad3e276e90d8eeb0fbddcbfe974ade3b0b99c18a487d5ef865ebdb1effc67048352ecd35d99d
+  data.tar.gz: 73c3519bf51a7934af89827830e3e7c45366eec01b1d65dc0c61afb2b87c12b093e9f3db99b1c4db081e6a6a1e983fd24d88db3aa13f63b9e546af0dce36beae

data/CHANGELOG

@@ -1,3 +1,7 @@
+2020-06-25 - 0.3
+
+- Added support for query_each and query_each_hash, which lazily queries rows and enables selecting large result sets by streaming
+
 2020-04-07 - 0.2.5
 
 - Added support for custom type maps with Postgres connections

data/README.md

@@ -130,10 +130,12 @@ When using Postgres, native type mapping implementation is used. This is roughly
 implemented as:
 
 ```ruby
-type_map = PG::BasicTypeMapForResults.new(conn)
+type_map ||= PG::BasicTypeMapForResults.new(conn)
 # additional specific decoders
 ```
 
+The type mapper instansitated once on-demand at boot and reused by all mini_sql connections.
+
 Initializing the basic type map for Postgres can be a costly operation. You may
 wish to amend the type mapper so for example you only return strings:
 
@@ -158,6 +160,33 @@ mini_sql_cnn = MiniSql::Connection.get(pg_cnn, type_map: pg_cnn.type_map_for_res
 
 Note the type mapper for Rails may miss some of the mapping MiniSql ships with such as `IPAddr`, MiniSql is also careful to use the very efficient TimestampUtc decoders where available.
 
+## Streaming support
+
+In some exceptional cases you may want to stream results directly from the database. This enables selection of 100s of thousands of rows with limited memory impact.
+
+Two interfaces exists for this:
+
+`query_each` : which can be used to get materialized objects  
+`query_each_hash` : which can be used to iterate through Hash objects
+
+Usage:
+
+```ruby
+mini_sql_cnn.query_each("SELECT * FROM tons_of_cows limit :limit", limit: 1_000_000) do |row|
+  puts row.cow_name
+  puts row.cow_age
+end
+
+mini_sql_cnn.query_each_hash("SELECT * FROM one_million_cows") do |row|
+  puts row["cow_name"]
+  puts row["cow_age"]
+end
+```
+
+Note, in Postgres streaming is going to be slower than non-streaming options due to internal implementation in the pq gem, each row gets a full result object and additional bookkeeping is needed. Only use it if you need to optimize memory usage.
+
+Streaming support is only implemented in the postgres backend at the moment, PRs welcome to add to other backends.
+
 ## I want more features!
 
 MiniSql is designed to be very minimal. Even though the query builder and type materializer give you a lot of mileage, it is not intended to be a fully fledged ORM. If you are looking for an ORM I recommend investigating ActiveRecord or Sequel which provide significantly more features.

data/lib/mini_sql/connection.rb

@@ -31,11 +31,23 @@ module MiniSql
       raise NotImplementedError, "must be implemented by child connection"
     end
 
-    def exec(sql, *params)
+    def query_hash(sql, *params)
       raise NotImplementedError, "must be implemented by child connection"
     end
 
-    def query_hash(sql, *params)
+    def query_decorator(sql, *params)
+      raise NotImplementedError, "must be implemented by child connection"
+    end
+
+    def query_each(sql, *params)
+      raise NotImplementedError, "must be implemented by child connection"
+    end
+
+    def query_each_hash(sql, *params)
+      raise NotImplementedError, "must be implemented by child connection"
+    end
+
+    def exec(sql, *params)
       raise NotImplementedError, "must be implemented by child connection"
     end
 

data/lib/mini_sql/postgres/connection.rb

@@ -83,7 +83,7 @@ module MiniSql
         result = run(sql, params)
         result.type_map = type_map
         result.values
-       ensure
+      ensure
          result.clear if result
       end
 
@@ -95,6 +95,72 @@ module MiniSql
         result.clear if result
       end
 
+      def query_each(sql, *params)
+        raise StandardError, "Please supply a block when calling query_each" if !block_given?
+        if params && params.length > 0
+          sql = param_encoder.encode(sql, *params)
+        end
+
+        raw_connection.send_query(sql)
+        raw_connection.set_single_row_mode
+
+        loop do
+          result = raw_connection.get_result
+          break if !result
+
+          result.check
+
+          if result.ntuples == 0
+            # skip, this happens at the end when we get totals
+          else
+            materializer ||= @deserializer_cache.materializer(result)
+            result.type_map = type_map
+            i = 0
+            # technically we should only get 1 row here
+            # but protect against future batching changes
+            while i < result.ntuples
+              yield materializer.materialize(result, i)
+              i += 1
+            end
+          end
+
+          result.clear
+        end
+      end
+
+      def query_each_hash(sql, *params)
+        raise StandardError, "Please supply a block when calling query_each_hash" if !block_given?
+        if params && params.length > 0
+          sql = param_encoder.encode(sql, *params)
+        end
+
+        raw_connection.send_query(sql)
+        raw_connection.set_single_row_mode
+
+        loop do
+          result = raw_connection.get_result
+          break if !result
+
+          result.check
+
+          if result.ntuples == 0
+            # skip, this happens at the end when we get totals
+          else
+            result.type_map = type_map
+            i = 0
+
+            # technically we should only get 1 row here
+            # but protect against future batching changes
+            while i < result.ntuples
+              yield result[i]
+              i += 1
+            end
+          end
+
+          result.clear
+        end
+      end
+
       def query_decorator(decorator, sql, *params)
         result = run(sql, params)
         result.type_map = type_map

data/lib/mini_sql/postgres/deserializer_cache.rb

@@ -11,6 +11,20 @@ module MiniSql
         @max_size = max_size || DEFAULT_MAX_SIZE
       end
 
+      def materializer(result)
+        key = result.fields
+
+        materializer = @cache.delete(key)
+        if materializer
+          @cache[key] = materializer
+        else
+          materializer = @cache[key] = new_row_matrializer(result)
+          @cache.shift if @cache.length > @max_size
+        end
+
+        materializer
+      end
+
       def materialize(result, decorator_module = nil)
         return [] if result.ntuples == 0
 
@@ -42,6 +56,15 @@ module MiniSql
       def new_row_matrializer(result)
         fields = result.fields
 
+        i = 0
+        while i < fields.length
+          # special handling for unamed column
+          if fields[i] == "?column?"
+            fields[i] = "column#{i}"
+          end
+          i += 1
+        end
+
         Class.new do
           attr_accessor(*fields)
 

data/lib/mini_sql/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module MiniSql
-  VERSION = "0.2.5"
+  VERSION = "0.3"
 end

data/mini_sql.gemspec

@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "> 10"
   spec.add_development_dependency "minitest", "~> 5.0"
   spec.add_development_dependency "guard", "~> 2.14"
   spec.add_development_dependency "guard-minitest", "~> 2.4"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mini_sql
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: '0.3'
 platform: ruby
 authors:
 - Sam Saffron
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-04-07 00:00:00.000000000 Z
+date: 2020-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -210,7 +210,7 @@ metadata:
   bug_tracker_uri: https://github.com/discourse/mini_sql/issues
   source_code_uri: https://github.com/discourse/mini_sql
   changelog_uri: https://github.com/discourse/mini_sql/blob/master/CHANGELOG
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -226,7 +226,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.0.3
-signing_key: 
+signing_key:
 specification_version: 4
 summary: A fast, safe, simple direct SQL executor
 test_files: []