metacrunch 3.1.3 → 3.1.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 717b40c307d7a613d071f5d4017dc110e6dd60da
4
- data.tar.gz: 142f77f8a029ff508571397b1581e05263a76d96
3
+ metadata.gz: 2ce37e928a0b2b84de0067604a3b56fe0206bda6
4
+ data.tar.gz: 42cf1bc2c321c35e34b716331300fcdaaf6f282b
5
5
  SHA512:
6
- metadata.gz: 9a11b97548c5eeee9c7870e5f3fd6bf157985b5f0a03d0ebeafd7199604f5d0dd0c6ac40d47fe469993f70e61e4a7f40a2edc690c5e6c0913799bb661c3297ad
7
- data.tar.gz: 12a67ca48bd4ee52cc58765b750c389566aaed7f4bbbd413c3e63c6e482bbbe30f8e05ef46c0c19752e16b28b43ade460b6b787a76f881e2eaf0646d67142c65
6
+ metadata.gz: 2e870644b9c78a4dd8590776f913ae712ebd8dedf381d1cbd6541e9a3c5a51311b1126c8af1a47b1d39d496675a8176076f5f99352741eeb4caa58ae29078dff
7
+ data.tar.gz: 6ba5ade2739a26d424f14add74c49fcadd7d03868a2f4c0e8651aff45d833b636f9e00a5176d5a1f1705703a6d99a6383b657c56d0902b6fcc97e840da98727d
@@ -139,8 +139,12 @@ module Metacrunch
139
139
  def run_transformations_and_write_destinations(data, flush_buffers: false)
140
140
  transformations.each do |transformation|
141
141
  if transformation.is_a?(Buffer)
142
- data = transformation.buffer(data) if data.present?
143
- data = transformation.flush if flush_buffers
142
+ if data.present?
143
+ data = transformation.buffer(data)
144
+ data = transformation.flush if flush_buffers
145
+ else
146
+ data = transformation.flush
147
+ end
144
148
  else
145
149
  data = transformation.call(data) if data.present?
146
150
  end
@@ -9,6 +9,7 @@ module Metacrunch
9
9
  raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
10
10
 
11
11
  @blocking_mode = options.delete(:blocking) || false
12
+ @blocking_timeout = options.delete(:blocking_timeout) || 0
12
13
 
13
14
  @redis = if redis_connection_or_url.is_a?(String)
14
15
  ::Redis.new(url: redis_connection_or_url)
@@ -22,8 +23,12 @@ module Metacrunch
22
23
 
23
24
  if @blocking_mode
24
25
  while true
25
- list, result = @redis.blpop(@queue_name)
26
- yield JSON.parse(result) if result
26
+ list, result = @redis.blpop(@queue_name, timeout: @blocking_timeout)
27
+ if result.present?
28
+ yield JSON.parse(result)
29
+ else
30
+ yield nil
31
+ end
27
32
  end
28
33
  else
29
34
  while result = @redis.lpop(@queue_name)
@@ -17,7 +17,15 @@ module Metacrunch
17
17
  end
18
18
 
19
19
  def write(data)
20
- @redis.rpush(@queue_name, data.to_json)
20
+ @redis.rpush(@queue_name, data)
21
+ rescue RuntimeError => e
22
+ if e.message =~ /maxmemory/
23
+ puts "Redis has reached maxmemory. Waiting 10 seconds and trying again..."
24
+ sleep(10)
25
+ retry
26
+ else
27
+ raise e
28
+ end
21
29
  end
22
30
 
23
31
  def close
@@ -0,0 +1,33 @@
1
+ require "metacrunch/redis"
2
+
3
+ module Metacrunch
4
+ class Redis::Writer
5
+
6
+ def initialize(redis_connection_or_url, options = {})
7
+ @save_on_close = options.delete(:save_on_close) || true
8
+
9
+ @key = options.delete(:key) || :key
10
+
11
+ @redis = if redis_connection_or_url.is_a?(String)
12
+ ::Redis.new(url: redis_connection_or_url)
13
+ else
14
+ redis_connection_or_url
15
+ end
16
+ end
17
+
18
+ def write(data)
19
+ key = data[@key]
20
+ raise ArgumentError, "No key found in data. Tried '#{@key}' but didn't found a value." unless key
21
+
22
+ @redis.set(key.to_s, data.to_json)
23
+ end
24
+
25
+ def close
26
+ if @redis
27
+ @redis.bgsave if @save_on_close
28
+ @redis.close
29
+ end
30
+ end
31
+
32
+ end
33
+ end
@@ -1,3 +1,3 @@
1
1
  module Metacrunch
2
- VERSION = "3.1.3"
2
+ VERSION = "3.1.4"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: metacrunch
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.3
4
+ version: 3.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - René Sprotte
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2016-10-28 00:00:00.000000000 Z
13
+ date: 2017-02-09 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: activesupport
@@ -120,6 +120,7 @@ files:
120
120
  - lib/metacrunch/redis.rb
121
121
  - lib/metacrunch/redis/queue_reader.rb
122
122
  - lib/metacrunch/redis/queue_writer.rb
123
+ - lib/metacrunch/redis/writer.rb
123
124
  - lib/metacrunch/test_utils.rb
124
125
  - lib/metacrunch/test_utils/dummy_callable.rb
125
126
  - lib/metacrunch/test_utils/dummy_destination.rb
@@ -146,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
146
147
  version: '0'
147
148
  requirements: []
148
149
  rubyforge_project:
149
- rubygems_version: 2.6.7
150
+ rubygems_version: 2.5.2
150
151
  signing_key:
151
152
  specification_version: 4
152
153
  summary: Data processing and ETL toolkit for Ruby