metacrunch 3.1.3 → 3.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/metacrunch/job.rb +6 -2
- data/lib/metacrunch/redis/queue_reader.rb +7 -2
- data/lib/metacrunch/redis/queue_writer.rb +9 -1
- data/lib/metacrunch/redis/writer.rb +33 -0
- data/lib/metacrunch/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2ce37e928a0b2b84de0067604a3b56fe0206bda6
|
4
|
+
data.tar.gz: 42cf1bc2c321c35e34b716331300fcdaaf6f282b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2e870644b9c78a4dd8590776f913ae712ebd8dedf381d1cbd6541e9a3c5a51311b1126c8af1a47b1d39d496675a8176076f5f99352741eeb4caa58ae29078dff
|
7
|
+
data.tar.gz: 6ba5ade2739a26d424f14add74c49fcadd7d03868a2f4c0e8651aff45d833b636f9e00a5176d5a1f1705703a6d99a6383b657c56d0902b6fcc97e840da98727d
|
data/lib/metacrunch/job.rb
CHANGED
@@ -139,8 +139,12 @@ module Metacrunch
|
|
139
139
|
def run_transformations_and_write_destinations(data, flush_buffers: false)
|
140
140
|
transformations.each do |transformation|
|
141
141
|
if transformation.is_a?(Buffer)
|
142
|
-
|
143
|
-
|
142
|
+
if data.present?
|
143
|
+
data = transformation.buffer(data)
|
144
|
+
data = transformation.flush if flush_buffers
|
145
|
+
else
|
146
|
+
data = transformation.flush
|
147
|
+
end
|
144
148
|
else
|
145
149
|
data = transformation.call(data) if data.present?
|
146
150
|
end
|
@@ -9,6 +9,7 @@ module Metacrunch
|
|
9
9
|
raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
|
10
10
|
|
11
11
|
@blocking_mode = options.delete(:blocking) || false
|
12
|
+
@blocking_timeout = options.delete(:blocking_timeout) || 0
|
12
13
|
|
13
14
|
@redis = if redis_connection_or_url.is_a?(String)
|
14
15
|
::Redis.new(url: redis_connection_or_url)
|
@@ -22,8 +23,12 @@ module Metacrunch
|
|
22
23
|
|
23
24
|
if @blocking_mode
|
24
25
|
while true
|
25
|
-
list, result = @redis.blpop(@queue_name)
|
26
|
-
|
26
|
+
list, result = @redis.blpop(@queue_name, timeout: @blocking_timeout)
|
27
|
+
if result.present?
|
28
|
+
yield JSON.parse(result)
|
29
|
+
else
|
30
|
+
yield nil
|
31
|
+
end
|
27
32
|
end
|
28
33
|
else
|
29
34
|
while result = @redis.lpop(@queue_name)
|
@@ -17,7 +17,15 @@ module Metacrunch
|
|
17
17
|
end
|
18
18
|
|
19
19
|
def write(data)
|
20
|
-
@redis.rpush(@queue_name, data
|
20
|
+
@redis.rpush(@queue_name, data)
|
21
|
+
rescue RuntimeError => e
|
22
|
+
if e.message =~ /maxmemory/
|
23
|
+
puts "Redis has reached maxmemory. Waiting 10 seconds and trying again..."
|
24
|
+
sleep(10)
|
25
|
+
retry
|
26
|
+
else
|
27
|
+
raise e
|
28
|
+
end
|
21
29
|
end
|
22
30
|
|
23
31
|
def close
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require "metacrunch/redis"
|
2
|
+
|
3
|
+
module Metacrunch
|
4
|
+
class Redis::Writer
|
5
|
+
|
6
|
+
def initialize(redis_connection_or_url, options = {})
|
7
|
+
@save_on_close = options.delete(:save_on_close) || true
|
8
|
+
|
9
|
+
@key = options.delete(:key) || :key
|
10
|
+
|
11
|
+
@redis = if redis_connection_or_url.is_a?(String)
|
12
|
+
::Redis.new(url: redis_connection_or_url)
|
13
|
+
else
|
14
|
+
redis_connection_or_url
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def write(data)
|
19
|
+
key = data[@key]
|
20
|
+
raise ArgumentError, "No key found in data. Tried '#{@key}' but didn't found a value." unless key
|
21
|
+
|
22
|
+
@redis.set(key.to_s, data.to_json)
|
23
|
+
end
|
24
|
+
|
25
|
+
def close
|
26
|
+
if @redis
|
27
|
+
@redis.bgsave if @save_on_close
|
28
|
+
@redis.close
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
data/lib/metacrunch/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.1.
|
4
|
+
version: 3.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2017-02-09 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport
|
@@ -120,6 +120,7 @@ files:
|
|
120
120
|
- lib/metacrunch/redis.rb
|
121
121
|
- lib/metacrunch/redis/queue_reader.rb
|
122
122
|
- lib/metacrunch/redis/queue_writer.rb
|
123
|
+
- lib/metacrunch/redis/writer.rb
|
123
124
|
- lib/metacrunch/test_utils.rb
|
124
125
|
- lib/metacrunch/test_utils/dummy_callable.rb
|
125
126
|
- lib/metacrunch/test_utils/dummy_destination.rb
|
@@ -146,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
147
|
version: '0'
|
147
148
|
requirements: []
|
148
149
|
rubyforge_project:
|
149
|
-
rubygems_version: 2.
|
150
|
+
rubygems_version: 2.5.2
|
150
151
|
signing_key:
|
151
152
|
specification_version: 4
|
152
153
|
summary: Data processing and ETL toolkit for Ruby
|