metacrunch-redis 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Readme.md +74 -0
- data/lib/metacrunch/redis/queue_destination.rb +12 -11
- data/lib/metacrunch/redis/queue_source.rb +9 -13
- data/lib/metacrunch/redis/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4516e6b7a64acaad451fd16d0613a5feb3c703fb
|
4
|
+
data.tar.gz: 2b99d46beb74ba1fd681a10a1d21d434245d96cc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '088ce8b7cc439843103adac1d1a0a4134b45b88e202b9113193b2ada078bbe1f300d5ccc1fa15a6c065557235343da431ce662a6755380a8cdc4af2b9d91cc50'
|
7
|
+
data.tar.gz: 88603596a4bac2c83b3718a58d02fe732a621d32200321ded23b42a30c8ce6e3ccd105e6e4f422d0bcc1c58d3f340c54d33c9325813a31e8d27dc81de3854997
|
data/Readme.md
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
metacrunch-redis
|
2
|
+
===============
|
3
|
+
|
4
|
+
[](http://badge.fury.io/rb/metacrunch-redis)
|
5
|
+
[](https://codeclimate.com/github/ubpb/metacrunch-redis)
|
6
|
+
[](https://travis-ci.org/ubpb/metacrunch-redis)
|
7
|
+
|
8
|
+
This is the official [Redis](https://redis.io) package for the [metacrunch ETL toolkit](https://github.com/ubpb/metacrunch).
|
9
|
+
|
10
|
+
Installation
|
11
|
+
------------
|
12
|
+
|
13
|
+
Include the gem in your `Gemfile`
|
14
|
+
|
15
|
+
```ruby
|
16
|
+
gem "metacrunch-redis", "~> 1.0.1"
|
17
|
+
```
|
18
|
+
|
19
|
+
and run `$ bundle install` to install it.
|
20
|
+
|
21
|
+
Or install it manually
|
22
|
+
|
23
|
+
```
|
24
|
+
$ gem install metacrunch-redis
|
25
|
+
```
|
26
|
+
|
27
|
+
Usage
|
28
|
+
-----
|
29
|
+
|
30
|
+
*Note: For working examples on how to use this package check out our [demo repository](https://github.com/ubpb/metacrunch-demo).*
|
31
|
+
|
32
|
+
### `Metacrunch::Redis::QueueSource`
|
33
|
+
|
34
|
+
This class provides a metacrunch `source` implementation that can be used to read data from a Redis queue/list into a metacrunch job.
|
35
|
+
|
36
|
+
```ruby
|
37
|
+
# my_job.metacrunch
|
38
|
+
|
39
|
+
# connect to redis
|
40
|
+
redis = ::Redis.new(url: "redis://localhost:6379/my-db")
|
41
|
+
|
42
|
+
# Set the source
|
43
|
+
source Metacrunch::Redis::QueueSource.new(redis, "my-list" [, OPTIONS])
|
44
|
+
```
|
45
|
+
|
46
|
+
**Options**
|
47
|
+
|
48
|
+
* `:blocking`: When set to `true` the source will block and waits for new data if the redis list is empty. Defaults to `false`.
|
49
|
+
|
50
|
+
|
51
|
+
### `Metacrunch::Redis::QueueDestination`
|
52
|
+
|
53
|
+
This class provides a metacrunch `destination` implementation that can be used to write data from a metacrunch job into a redis queue/list.
|
54
|
+
|
55
|
+
Redis only stores strings as values. If you want to store an object, you can use a serialization mechanism such as JSON. You can use a transformation to convert your data into JSON format before your data reaching the destination.
|
56
|
+
|
57
|
+
In case Redis reaches it's `maxmemory` limit during write, the implementation will wait for 10 seconds and tries to write the data again. That means you can set a proper `maxmemory` limit for your Redis instance and don't need to worry about your metacrunch jobs getting aborted.
|
58
|
+
|
59
|
+
```ruby
|
60
|
+
# my_job.metacrunch
|
61
|
+
|
62
|
+
# Prepare the data to be stored.
|
63
|
+
transformation ->(data) do
|
64
|
+
# Return json for the destination
|
65
|
+
data.to_json
|
66
|
+
end
|
67
|
+
|
68
|
+
# Write data into redis queue
|
69
|
+
destination Metacrunch::Redis::QueueDestination.new(redis, "my-list" [, OPTIONS])
|
70
|
+
```
|
71
|
+
|
72
|
+
**Options**
|
73
|
+
|
74
|
+
* `:save_on_close`: When set to `true` a Redis `bgsave` will be performed when the destination is closed. Defaults to `false`.
|
@@ -3,20 +3,19 @@ require "metacrunch/redis"
|
|
3
3
|
module Metacrunch
|
4
4
|
class Redis::QueueDestination
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
@save_on_close = options.delete(:save_on_close) || true
|
6
|
+
DEFAULT_OPTIONS = {
|
7
|
+
save_on_close: false
|
8
|
+
}
|
11
9
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
end
|
10
|
+
def initialize(redis, queue_name, options = {})
|
11
|
+
@redis = redis
|
12
|
+
@queue_name = queue_name
|
13
|
+
@options = DEFAULT_OPTIONS.merge(options)
|
17
14
|
end
|
18
15
|
|
19
16
|
def write(data)
|
17
|
+
return if data.blank?
|
18
|
+
|
20
19
|
@redis.rpush(@queue_name, data)
|
21
20
|
rescue RuntimeError => e
|
22
21
|
if e.message =~ /maxmemory/
|
@@ -30,7 +29,9 @@ module Metacrunch
|
|
30
29
|
|
31
30
|
def close
|
32
31
|
if @redis
|
33
|
-
|
32
|
+
begin
|
33
|
+
@redis.bgsave if @options[:save_on_close]
|
34
|
+
rescue Redis::CommandError ; end
|
34
35
|
@redis.close
|
35
36
|
end
|
36
37
|
end
|
@@ -3,26 +3,22 @@ require "metacrunch/redis"
|
|
3
3
|
module Metacrunch
|
4
4
|
class Redis::QueueSource
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
@blocking_mode = options.delete(:blocking) || false
|
11
|
-
@blocking_timeout = options.delete(:blocking_timeout) || 0
|
6
|
+
DEFAULT_OPTIONS = {
|
7
|
+
blocking_mode: false
|
8
|
+
}
|
12
9
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
10
|
+
def initialize(redis, queue_name, options = {})
|
11
|
+
@redis = redis
|
12
|
+
@queue_name = queue_name
|
13
|
+
@options = DEFAULT_OPTIONS.merge(options)
|
18
14
|
end
|
19
15
|
|
20
16
|
def each(&block)
|
21
17
|
return enum_for(__method__) unless block_given?
|
22
18
|
|
23
|
-
if @blocking_mode
|
19
|
+
if @options[:blocking_mode]
|
24
20
|
while true
|
25
|
-
list, result = @redis.blpop(@queue_name, timeout:
|
21
|
+
list, result = @redis.blpop(@queue_name, timeout: 0)
|
26
22
|
if result.present?
|
27
23
|
yield result
|
28
24
|
else
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch-redis
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-09
|
11
|
+
date: 2017-10-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -49,6 +49,7 @@ files:
|
|
49
49
|
- Gemfile
|
50
50
|
- License.txt
|
51
51
|
- Rakefile
|
52
|
+
- Readme.md
|
52
53
|
- bin/console
|
53
54
|
- lib/metacrunch/redis.rb
|
54
55
|
- lib/metacrunch/redis/queue_destination.rb
|