metacrunch 3.1.1 → 3.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/metacrunch/db/reader.rb +2 -13
- data/lib/metacrunch/redis/queue_reader.rb +3 -2
- data/lib/metacrunch/redis/queue_writer.rb +6 -1
- data/lib/metacrunch/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc8d5679bad10cb1cc6452cdc377f15845a3cd97
|
4
|
+
data.tar.gz: 9505173130302cecfb30fbac670f3ccfede37513
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db2d938c58fe6ceb62301208dc4394978a073abe7010c4bd90bc906b085f431fa01484baf208bff36b5134326d6baf3f1ec1800cae7841e38ac1813c538dfeb2
|
7
|
+
data.tar.gz: 38f1dff772a76812d3c4c34ec918d4ece4b03f72f7b136485e23efc0cb9a66be085d81c6c0b05510c482225fd757836e95b17964146d68530ba59433590c242e
|
data/lib/metacrunch/db/reader.rb
CHANGED
@@ -14,7 +14,6 @@ module Metacrunch
|
|
14
14
|
end
|
15
15
|
|
16
16
|
@dataset = dataset_proc.call(@db).unlimited
|
17
|
-
@total_numbers_of_records = @dataset.count
|
18
17
|
|
19
18
|
unless @dataset.opts[:order]
|
20
19
|
raise ArgumentError, "Metacrunch::Db::Reader requires the dataset be ordered."
|
@@ -24,18 +23,8 @@ module Metacrunch
|
|
24
23
|
def each(&block)
|
25
24
|
return enum_for(__method__) unless block_given?
|
26
25
|
|
27
|
-
@
|
28
|
-
|
29
|
-
|
30
|
-
loop do
|
31
|
-
offset = offset + (number_of_processes * @rows_per_fetch)
|
32
|
-
|
33
|
-
@dataset.limit(@rows_per_fetch).offset(offset).each do |row|
|
34
|
-
yield(row)
|
35
|
-
end
|
36
|
-
|
37
|
-
break if offset + @rows_per_fetch >= @total_numbers_of_records
|
38
|
-
end
|
26
|
+
@dataset.paged_each(rows_per_fetch: @rows_per_fetch, strategy: :filter) do |row|
|
27
|
+
yield(row)
|
39
28
|
end
|
40
29
|
|
41
30
|
self
|
@@ -2,6 +2,7 @@ require "metacrunch/redis"
|
|
2
2
|
|
3
3
|
module Metacrunch
|
4
4
|
class Redis::QueueReader
|
5
|
+
include Metacrunch::ParallelProcessableReader
|
5
6
|
|
6
7
|
def initialize(redis_connection_or_url, queue_name, options = {})
|
7
8
|
@queue_name = queue_name
|
@@ -21,8 +22,8 @@ module Metacrunch
|
|
21
22
|
|
22
23
|
if @blocking_mode
|
23
24
|
while true
|
24
|
-
result = @redis.blpop(@queue_name)
|
25
|
-
yield JSON.parse(result
|
25
|
+
list, result = @redis.blpop(@queue_name)
|
26
|
+
yield JSON.parse(result) if result
|
26
27
|
end
|
27
28
|
else
|
28
29
|
while result = @redis.lpop(@queue_name)
|
@@ -7,6 +7,8 @@ module Metacrunch
|
|
7
7
|
@queue_name = queue_name
|
8
8
|
raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
|
9
9
|
|
10
|
+
@save_on_close = options.delete(:save_on_close) || true
|
11
|
+
|
10
12
|
@redis = if redis_connection_or_url.is_a?(String)
|
11
13
|
::Redis.new(url: redis_connection_or_url)
|
12
14
|
else
|
@@ -19,7 +21,10 @@ module Metacrunch
|
|
19
21
|
end
|
20
22
|
|
21
23
|
def close
|
22
|
-
|
24
|
+
if @redis
|
25
|
+
@redis.bgsave if @save_on_close
|
26
|
+
@redis.close
|
27
|
+
end
|
23
28
|
end
|
24
29
|
|
25
30
|
end
|
data/lib/metacrunch/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2016-
|
13
|
+
date: 2016-10-28 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport
|
@@ -146,7 +146,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
146
|
version: '0'
|
147
147
|
requirements: []
|
148
148
|
rubyforge_project:
|
149
|
-
rubygems_version: 2.
|
149
|
+
rubygems_version: 2.6.7
|
150
150
|
signing_key:
|
151
151
|
specification_version: 4
|
152
152
|
summary: Data processing and ETL toolkit for Ruby
|