metacrunch-redis 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2966f98bbe7578886cd8062a85adbdcdfac6506
-  data.tar.gz: 93b03ff34459f6dda1fa41e4a13e5080ba832c62
+  metadata.gz: 4516e6b7a64acaad451fd16d0613a5feb3c703fb
+  data.tar.gz: 2b99d46beb74ba1fd681a10a1d21d434245d96cc
 SHA512:
-  metadata.gz: bff4e20ae228229b11e4dac0da275e47d8a850a88659da98b6f4225ce08de5391a08d8e383bf8b4c67182d90eb90e94c771f4359a75d862f7d4230fe7958540d
-  data.tar.gz: 663a3cf3d7749621fe0ea09634f1ddb0c3604aeca9fae4accd9354d066ad0d3d29b1d6dc633af4d1d02f3310d3570ce08b76ccce0b8794aee6f0af6ed2b464a7
+  metadata.gz: '088ce8b7cc439843103adac1d1a0a4134b45b88e202b9113193b2ada078bbe1f300d5ccc1fa15a6c065557235343da431ce662a6755380a8cdc4af2b9d91cc50'
+  data.tar.gz: 88603596a4bac2c83b3718a58d02fe732a621d32200321ded23b42a30c8ce6e3ccd105e6e4f422d0bcc1c58d3f340c54d33c9325813a31e8d27dc81de3854997

data/Readme.md

@@ -0,0 +1,74 @@
+metacrunch-redis
+===============
+
+[![Gem Version](https://badge.fury.io/rb/metacrunch-redis.svg)](http://badge.fury.io/rb/metacrunch-redis)
+[![Code Climate](https://codeclimate.com/github/ubpb/metacrunch-redis/badges/gpa.svg)](https://codeclimate.com/github/ubpb/metacrunch-redis)
+[![Build Status](https://travis-ci.org/ubpb/metacrunch-redis.svg)](https://travis-ci.org/ubpb/metacrunch-redis)
+
+This is the official [Redis](https://redis.io) package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
+
+Installation
+------------
+
+Include the gem in your `Gemfile`
+
+```ruby
+gem "metacrunch-redis", "~> 1.0.1"
+```
+
+and run `$ bundle install` to install it.
+
+Or install it manually
+
+```
+$ gem install metacrunch-redis
+```
+
+Usage
+-----
+
+*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
+
+### `Metacrunch::Redis::QueueSource`
+
+This class provides a metacrunch `source` implementation that can be used to read data from a Redis queue/list into a metacrunch job.
+
+```ruby
+# my_job.metacrunch
+
+# connect to redis
+redis = ::Redis.new(url: "redis://localhost:6379/my-db")
+
+# Set the source
+source Metacrunch::Redis::QueueSource.new(redis, "my-list" [, OPTIONS])
+```
+
+**Options**
+
+* `:blocking`: When set to `true` the source will block and waits for new data if the redis list is empty. Defaults to `false`.
+
+
+### `Metacrunch::Redis::QueueDestination`
+
+This class provides a metacrunch `destination` implementation that can be used to write data from a metacrunch job into a redis queue/list.
+
+Redis only stores strings as values. If you want to store an object, you can use a serialization mechanism such as JSON. You can use a transformation to convert your data into JSON format before your data reaching the destination.
+
+In case Redis reaches it's `maxmemory` limit during write, the implementation will wait for 10 seconds and tries to write the data again. That means you can set a proper `maxmemory` limit for your Redis instance and don't need to worry about your metacrunch jobs getting aborted.
+
+```ruby
+# my_job.metacrunch
+
+# Prepare the data to be stored. 
+transformation ->(data) do
+  # Return json for the destination
+  data.to_json
+end
+
+# Write data into redis queue
+destination Metacrunch::Redis::QueueDestination.new(redis, "my-list" [, OPTIONS])
+```
+
+**Options**
+
+* `:save_on_close`: When set to `true` a Redis `bgsave` will be performed when the destination is closed. Defaults to `false`. 

data/lib/metacrunch/redis/queue_destination.rb

@@ -3,20 +3,19 @@ require "metacrunch/redis"
 module Metacrunch
   class Redis::QueueDestination
 
-    def initialize(redis_connection_or_url, queue_name, options = {})
-      @queue_name = queue_name
-      raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
-
-      @save_on_close = options.delete(:save_on_close) || true
+    DEFAULT_OPTIONS = {
+      save_on_close: false
+    }
 
-      @redis = if redis_connection_or_url.is_a?(String)
-        ::Redis.new(url: redis_connection_or_url)
-      else
-        redis_connection_or_url
-      end
+    def initialize(redis, queue_name, options = {})
+      @redis = redis
+      @queue_name = queue_name
+      @options = DEFAULT_OPTIONS.merge(options)
     end
 
     def write(data)
+      return if data.blank?
+
       @redis.rpush(@queue_name, data)
     rescue RuntimeError => e
       if e.message =~ /maxmemory/
@@ -30,7 +29,9 @@ module Metacrunch
 
     def close
       if @redis
-        @redis.bgsave if @save_on_close
+        begin
+          @redis.bgsave if @options[:save_on_close]
+        rescue Redis::CommandError ; end
         @redis.close
       end
     end

data/lib/metacrunch/redis/queue_source.rb

@@ -3,26 +3,22 @@ require "metacrunch/redis"
 module Metacrunch
   class Redis::QueueSource
 
-    def initialize(redis_connection_or_url, queue_name, options = {})
-      @queue_name = queue_name
-      raise ArgumentError, "queue_name must be a string" unless queue_name.is_a?(String)
-
-      @blocking_mode = options.delete(:blocking) || false
-      @blocking_timeout = options.delete(:blocking_timeout) || 0
+    DEFAULT_OPTIONS = {
+      blocking_mode: false
+    }
 
-      @redis = if redis_connection_or_url.is_a?(String)
-        ::Redis.new(url: redis_connection_or_url)
-      else
-        redis_connection_or_url
-      end
+    def initialize(redis, queue_name, options = {})
+      @redis = redis
+      @queue_name = queue_name
+      @options = DEFAULT_OPTIONS.merge(options)
     end
 
     def each(&block)
       return enum_for(__method__) unless block_given?
 
-      if @blocking_mode
+      if @options[:blocking_mode]
         while true
-          list, result = @redis.blpop(@queue_name, timeout: @blocking_timeout)
+          list, result = @redis.blpop(@queue_name, timeout: 0)
           if result.present?
             yield result
           else

data/lib/metacrunch/redis/version.rb

@@ -1,5 +1,5 @@
 module Metacrunch
   module Redis
-    VERSION = "1.0.1"
+    VERSION = "1.1.0"
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metacrunch-redis
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.0
 platform: ruby
 authors:
 - René Sprotte
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-25 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -49,6 +49,7 @@ files:
 - Gemfile
 - License.txt
 - Rakefile
+- Readme.md
 - bin/console
 - lib/metacrunch/redis.rb
 - lib/metacrunch/redis/queue_destination.rb