metacrunch 3.1.3 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 717b40c307d7a613d071f5d4017dc110e6dd60da
-  data.tar.gz: 142f77f8a029ff508571397b1581e05263a76d96
+  metadata.gz: 2ce37e928a0b2b84de0067604a3b56fe0206bda6
+  data.tar.gz: 42cf1bc2c321c35e34b716331300fcdaaf6f282b
 SHA512:
-  metadata.gz: 9a11b97548c5eeee9c7870e5f3fd6bf157985b5f0a03d0ebeafd7199604f5d0dd0c6ac40d47fe469993f70e61e4a7f40a2edc690c5e6c0913799bb661c3297ad
-  data.tar.gz: 12a67ca48bd4ee52cc58765b750c389566aaed7f4bbbd413c3e63c6e482bbbe30f8e05ef46c0c19752e16b28b43ade460b6b787a76f881e2eaf0646d67142c65
+  metadata.gz: 2e870644b9c78a4dd8590776f913ae712ebd8dedf381d1cbd6541e9a3c5a51311b1126c8af1a47b1d39d496675a8176076f5f99352741eeb4caa58ae29078dff
+  data.tar.gz: 6ba5ade2739a26d424f14add74c49fcadd7d03868a2f4c0e8651aff45d833b636f9e00a5176d5a1f1705703a6d99a6383b657c56d0902b6fcc97e840da98727d

data/lib/metacrunch/job.rb

@@ -139,8 +139,12 @@ module Metacrunch
     def run_transformations_and_write_destinations(data, flush_buffers: false)
       transformations.each do |transformation|
         if transformation.is_a?(Buffer)
-          data = transformation.buffer(data) if data.present?
-          data = transformation.flush if flush_buffers
+          if data.present?
+            data = transformation.buffer(data)
+            data = transformation.flush if flush_buffers
+          else
+            data = transformation.flush
+          end
         else
           data = transformation.call(data) if data.present?
         end

data/lib/metacrunch/redis/queue_reader.rb

@@ -9,6 +9,7 @@ module Metacrunch
       raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
 
       @blocking_mode = options.delete(:blocking) || false
+      @blocking_timeout = options.delete(:blocking_timeout) || 0
 
       @redis = if redis_connection_or_url.is_a?(String)
         ::Redis.new(url: redis_connection_or_url)
@@ -22,8 +23,12 @@ module Metacrunch
 
       if @blocking_mode
         while true
-          list, result = @redis.blpop(@queue_name)
-          yield JSON.parse(result) if result
+          list, result = @redis.blpop(@queue_name, timeout: @blocking_timeout)
+          if result.present?
+            yield JSON.parse(result)
+          else
+            yield nil
+          end
         end
       else
         while result = @redis.lpop(@queue_name)

data/lib/metacrunch/redis/queue_writer.rb

@@ -17,7 +17,15 @@ module Metacrunch
     end
 
     def write(data)
-      @redis.rpush(@queue_name, data.to_json)
+      @redis.rpush(@queue_name, data)
+    rescue RuntimeError => e
+      if e.message =~ /maxmemory/
+        puts "Redis has reached maxmemory. Waiting 10 seconds and trying again..."
+        sleep(10)
+        retry
+      else
+        raise e
+      end
     end
 
     def close

data/lib/metacrunch/redis/writer.rb

@@ -0,0 +1,33 @@
+require "metacrunch/redis"
+
+module Metacrunch
+  class Redis::Writer
+
+    def initialize(redis_connection_or_url, options = {})
+      @save_on_close = options.delete(:save_on_close) || true
+
+      @key = options.delete(:key) || :key
+
+      @redis = if redis_connection_or_url.is_a?(String)
+        ::Redis.new(url: redis_connection_or_url)
+      else
+        redis_connection_or_url
+      end
+    end
+
+    def write(data)
+      key = data[@key]
+      raise ArgumentError, "No key found in data. Tried '#{@key}' but didn't found a value." unless key
+
+      @redis.set(key.to_s, data.to_json)
+    end
+
+    def close
+      if @redis
+        @redis.bgsave if @save_on_close
+        @redis.close
+      end
+    end
+
+  end
+end

data/lib/metacrunch/version.rb

@@ -1,3 +1,3 @@
 module Metacrunch
-  VERSION = "3.1.3"
+  VERSION = "3.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.1.4
 platform: ruby
 authors:
 - René Sprotte
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-10-28 00:00:00.000000000 Z
+date: 2017-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -120,6 +120,7 @@ files:
 - lib/metacrunch/redis.rb
 - lib/metacrunch/redis/queue_reader.rb
 - lib/metacrunch/redis/queue_writer.rb
+- lib/metacrunch/redis/writer.rb
 - lib/metacrunch/test_utils.rb
 - lib/metacrunch/test_utils/dummy_callable.rb
 - lib/metacrunch/test_utils/dummy_destination.rb
@@ -146,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.7
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Data processing and ETL toolkit for Ruby