metacrunch 3.1.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f740db9ee343380c368f28d0b5da470070f089a1
-  data.tar.gz: 0ba256e6de54b5147e2680ae047af3f91d136201
+  metadata.gz: cc8d5679bad10cb1cc6452cdc377f15845a3cd97
+  data.tar.gz: 9505173130302cecfb30fbac670f3ccfede37513
 SHA512:
-  metadata.gz: 3f8a630085cb3423ce1ac4752f5a44986c11961ffefa003135611ade832ae91beece7b77c744262629548eb131fcca990b06e6995821179d90d94fc1db650e25
-  data.tar.gz: 3b463bce9b8e1a5ba5a1c060963f9e0c2c3ceb5bdd80fa5497b885ebc54a96e2368288aacbebc281cb28cb4a49e7d84d2502ce357381b05e8577e4ec18a7a901
+  metadata.gz: db2d938c58fe6ceb62301208dc4394978a073abe7010c4bd90bc906b085f431fa01484baf208bff36b5134326d6baf3f1ec1800cae7841e38ac1813c538dfeb2
+  data.tar.gz: 38f1dff772a76812d3c4c34ec918d4ece4b03f72f7b136485e23efc0cb9a66be085d81c6c0b05510c482225fd757836e95b17964146d68530ba59433590c242e

data/lib/metacrunch/db/reader.rb

@@ -14,7 +14,6 @@ module Metacrunch
       end
 
       @dataset = dataset_proc.call(@db).unlimited
-      @total_numbers_of_records = @dataset.count
 
       unless @dataset.opts[:order]
         raise ArgumentError, "Metacrunch::Db::Reader requires the dataset be ordered."
@@ -24,18 +23,8 @@ module Metacrunch
     def each(&block)
       return enum_for(__method__) unless block_given?
 
-      @db.transaction do
-        offset = (-number_of_processes * @rows_per_fetch) + (process_index * @rows_per_fetch)
-
-        loop do
-          offset = offset + (number_of_processes * @rows_per_fetch)
-
-          @dataset.limit(@rows_per_fetch).offset(offset).each do |row|
-            yield(row)
-          end
-
-          break if offset + @rows_per_fetch >= @total_numbers_of_records
-        end
+      @dataset.paged_each(rows_per_fetch: @rows_per_fetch, strategy: :filter) do |row|
+        yield(row)
       end
 
       self

data/lib/metacrunch/redis/queue_reader.rb

@@ -2,6 +2,7 @@ require "metacrunch/redis"
 
 module Metacrunch
   class Redis::QueueReader
+    include Metacrunch::ParallelProcessableReader
 
     def initialize(redis_connection_or_url, queue_name, options = {})
       @queue_name = queue_name
@@ -21,8 +22,8 @@ module Metacrunch
 
       if @blocking_mode
         while true
-          result = @redis.blpop(@queue_name)
-          yield JSON.parse(result[1]) if result
+          list, result = @redis.blpop(@queue_name)
+          yield JSON.parse(result) if result
         end
       else
         while result = @redis.lpop(@queue_name)

data/lib/metacrunch/redis/queue_writer.rb

@@ -7,6 +7,8 @@ module Metacrunch
       @queue_name = queue_name
       raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
 
+      @save_on_close = options.delete(:save_on_close) || true
+
       @redis = if redis_connection_or_url.is_a?(String)
         ::Redis.new(url: redis_connection_or_url)
       else
@@ -19,7 +21,10 @@ module Metacrunch
     end
 
     def close
-      @redis.close if @redis
+      if @redis
+        @redis.bgsave if @save_on_close
+        @redis.close
+      end
     end
 
   end

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "3.1.1"
+  VERSION = "3.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.1.2
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.7
 signing_key: 
 specification_version: 4
 summary: Data processing and ETL toolkit for Ruby