querrel 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a221606ea4c6afe6ec28d674c8a3991bb221d15
-  data.tar.gz: 3c1bb8daff36108b3addd087d3769cbe3b5cefab
+  metadata.gz: ac00fa1731878f9644eeef4cba1df54454b70d3c
+  data.tar.gz: 3333ac82975cc0ab9a9359ea254db41c97330ded
 SHA512:
-  metadata.gz: 971c487ef162a42b1874616ea1f267a13836d0671766fe12f83dc359fef3df071d0b4b9c6cf8e39d9cd85cf896184b806f3bfb6881c4f9fb5d102942c7c9e744
-  data.tar.gz: d2aa19d3e6cb412bee5210107b616607847870baf61bc5d4c8948a13d756c771c278610aa578f5f5e4f1f0c6a225cb2af61a3b384a30a85751e00e52602681cd
+  metadata.gz: fa8d8b12c96c966a0b5e755749a175902f4ba5b1808f94d3cdb70179250d6f624eba73d69f754a9e0e642bc10c02960455ac1bafb25fccb69dd44e8980daaef3
+  data.tar.gz: a65135e0097d7ccdad1d7f276ab77adc0cc48f1787db0a413c12afa93920795558157ccda86737eb51a85e589ab86c6f2ed7910b771cb0155a89063e567f41e6

data/README.md

@@ -18,6 +18,8 @@ Or install it yourself as:
 
 ## Usage
 
+### Basic
+
 You can use with Querrel directly via it's top level namespace, or by creating an instance:
 
 ```ruby
@@ -28,6 +30,35 @@ all_brands = Querrel.new(['db1', 'db2', 'db3']).query(Brand.all)
 
 Either of the above will give you an array of all the Brand objects from all the given databases. The records will be marked as readonly.
 
+### Advanced
+
+`query` will yield a block with the passed in `ActiveRecord::Relation`, this allows you to do additional operations on the results before they are merged, for example you could use pluck:
+
+```ruby
+all_brand_names = q.query(Brand.all) do |s|
+  s.pluck(:name)
+end
+```
+
+There is also a `run` method which instead of running a preprescribed scope and merging, will yield the `ConnectedModelFactory` class which allows you to wrap any ActiveRecord class so that you can query it in the thread, for instance:
+
+```ruby
+require 'thread'
+all_brands = Brand.all
+b_s = Mutex.new
+all_products = Product.all
+p_s = Mutex.new
+
+Querrel.run(on: dbs) do |q|
+  b_s.synchronize { all_brands += q[Brand].all.to_a }
+  p_s.synchronize { all_products += q[Product].all.to_a }
+end
+```
+
+At the moment, any queries using `run` and the ConnectedModelFactory won't return objects of the class passed in, but an anonymous class instead. However, `query` can marshall the results back into the original class.
+
+### Databases
+
 There are three ways in which you can instruct Querrel which databases to use:
 
 1. Pass in an array of environments, e.g. `Querrel.new([:customer1, :customer2])`
@@ -47,6 +78,14 @@ There are three ways in which you can instruct Querrel which databases to use:
     })
     ```
 
+### Configuration
+
+By default Querrel will use a maximum of 20 threads, but you can adjust this using the `:threads` option:
+
+```ruby
+q = Querrel.new(dbs, threads: 50)
+```
+
 ## Contributing
 
 1. Fork it ( https://github.com/meritec/querrel/fork )

data/lib/querrel/connected_model_factory.rb

@@ -6,6 +6,8 @@ module Querrel
       Class.new(model).tap do |m|
         m.send(:define_singleton_method, :name) { dynamic_class_name }
         m.establish_connection(con_spec.config)
+
+        Thread.current[:querrel_connected_models] << m
       end
     end
   end

data/lib/querrel/map_reduce.rb

@@ -6,6 +6,7 @@ module Querrel
     end
 
     def map(scope, options = {}, &blk)
+      options = @options.merge(options)
       if options.key?(:on)
         resolver = ConnectionResolver.new(options[:on], !!options[:db_names])
         dbs = resolver.configurations.keys
@@ -16,29 +17,38 @@ module Querrel
 
       query_model = scope.model
       results = {}
+      results_semaphore = Mutex.new
 
-      threads = []
+      pool = StaticPool.new(options[:threads] || 20)
       dbs.each do |db|
-        threads << Thread.new do
+        pool.enqueue do
+          Thread.current[:querrel_connected_models] = []
           con_spec = retrieve_connection_spec(db, resolver)
           Thread.current[:querrel_con_spec] = con_spec
           dynamic_class = ConnectedModelFactory[query_model, con_spec]
 
           begin
             local_scope = dynamic_class.all.merge(scope)
-            results[db] = if block_given?
-              res = yield(local_scope)
+            local_results = if block_given?
+              res = yield(local_scope, ConnectedModelFactory)
               res.to_a.each(&:readonly!) if res.is_a?(ActiveRecord::Relation)
               res
             else
-              local_scope.to_a.each(&:readonly!)
+              local_scope.to_a.map do |r|
+                query_model.instantiate(r.attributes, {}).tap(&:readonly!)
+              end
             end
+
+            results_semaphore.synchronize { results[db] = local_results }
           ensure
-            dynamic_class.connection_pool.release_connection
+            Thread.current[:querrel_connected_models].each do |m|
+              m.connection_pool.release_connection
+            end
+            Thread.current[:querrel_connected_models] = nil
           end
         end
       end
-      threads.each(&:join)
+      pool.do_your_thang!
 
       results
     end

data/lib/querrel/querreller.rb

@@ -1,5 +1,6 @@
 require 'querrel/connected_model_factory'
 require 'querrel/connection_resolver'
+require 'querrel/static_pool'
 require 'querrel/map_reduce'
 
 module Querrel
@@ -8,10 +9,12 @@ module Querrel
     attr_accessor :connection_resolver
 
     def initialize(dbs, options = {})
-      @connection_resolver = ConnectionResolver.new(dbs, options[:db_names])
+      @connection_resolver = ConnectionResolver.new(dbs, options.delete(:db_names))
+      @options = options
     end
 
     def run(options = {}, &blk)
+      options = @options.merge(options)
       if options.key?(:on)
         resolver = ConnectionResolver.new(options[:on], !!options[:db_names])
         dbs = resolver.configurations.keys
@@ -20,15 +23,23 @@ module Querrel
         dbs = @connection_resolver.configurations.keys
       end
 
-      threads = []
+      pool = StaticPool.new(options[:threads] || 20)
       dbs.each do |db|
-        threads << Thread.new do
-          con_spec = retrieve_connection_spec(db, resolver)
-          Thread.current[:querrel_con_spec] = con_spec
-          yield(ConnectedModelFactory)
+        pool.enqueue do
+          begin
+            Thread.current[:querrel_connected_models] = []
+            con_spec = retrieve_connection_spec(db, resolver)
+            Thread.current[:querrel_con_spec] = con_spec
+            yield(ConnectedModelFactory)
+          ensure
+            Thread.current[:querrel_connected_models].each do |m|
+              m.connection_pool.release_connection
+            end
+            Thread.current[:querrel_connected_models] = nil
+          end
         end
       end
-      threads.each(&:join)
+      pool.do_your_thang!
     end
 
     def retrieve_connection_spec(db, resolver)

data/lib/querrel/static_pool.rb

@@ -0,0 +1,26 @@
+require 'thread'
+
+module Querrel
+  class StaticPool
+    def initialize(size)
+      @size = size
+      @jobs = Queue.new
+    end
+
+    def enqueue(&job)
+      @jobs.push(job)
+    end
+
+    def do_your_thang!
+      threads = Array.new(@size) do
+        Thread.new do
+          while job = @jobs.pop(true) rescue nil
+            job.call
+          end
+        end
+      end
+
+      threads.each(&:join)
+    end
+  end
+end

data/lib/querrel/version.rb

@@ -1,3 +1,3 @@
 module Querrel
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

data/test/non_instance_test.rb

@@ -14,6 +14,7 @@ class NonInstanceTest < Querrel::Test
 
     assert_equal num_brands * @dbs.length, res.length,
       "Not returning the correct number of results"
+    assert res.all?{ |r| r.is_a?(Brand) }
   end
 
   def test_map

data/test/static_pool_test.rb

@@ -0,0 +1,23 @@
+require 'thread'
+require_relative 'setup/test_helper'
+
+class StaticPoolTest < Querrel::Test
+  def test_never_more_than_max_threads
+    max_threads = 10
+    p = Querrel::StaticPool.new(max_threads)
+
+    thread_counts = []
+    thread_semaphore = Mutex.new
+
+    50.times do
+      p.enqueue do
+        thread_semaphore.synchronize do
+          thread_counts << Thread.list.count{ |t| t.status == "run" }
+        end
+      end
+    end
+    p.do_your_thang!
+
+    assert thread_counts.all?{ |c| c <= max_threads }
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: querrel
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Mike Campbell
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-21 00:00:00.000000000 Z
+date: 2015-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -97,6 +97,7 @@ files:
 - lib/querrel/connection_resolver.rb
 - lib/querrel/map_reduce.rb
 - lib/querrel/querreller.rb
+- lib/querrel/static_pool.rb
 - lib/querrel/version.rb
 - querrel.gemspec
 - test/instance_test.rb
@@ -110,6 +111,7 @@ files:
 - test/setup/querrel_test_class.rb
 - test/setup/schema.rb
 - test/setup/test_helper.rb
+- test/static_pool_test.rb
 - test/test_test.rb
 homepage: https://github.com/meritec/querrel
 licenses:
@@ -147,4 +149,5 @@ test_files:
 - test/setup/querrel_test_class.rb
 - test/setup/schema.rb
 - test/setup/test_helper.rb
+- test/static_pool_test.rb
 - test/test_test.rb