punchline 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 7cf29f420c42097206b7df8314facad3f2f777c6
4
+ data.tar.gz: 73c9c8d9036f745b3055551ab0de2e40204ae954
5
+ SHA512:
6
+ metadata.gz: aea0964fe5cf076e7c7cc645997877f1cb056437c18f2f3a332c5932f80c8106ce628197e8fe6bef2f70d417e7a21e162adeee2e1eccfd2149185d0ad077e4fa
7
+ data.tar.gz: 504d8ffd5b9c953d50e3e60629e5a0edab8da91a56629dd583c61965b67ffd8c4240122a0053ff81b029ec66ebd1b923abc470707bcdd5353d835b5535d24240
data/.coveralls.yml ADDED
@@ -0,0 +1 @@
1
+ service_name: travis-ci
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --color --format doc
data/.travis.yml ADDED
@@ -0,0 +1,10 @@
1
+ language: ruby
2
+ cache: bundler
3
+ services:
4
+ - redis-server
5
+ rvm:
6
+ - jruby
7
+ - 2.0.0
8
+ - 2.1.0
9
+
10
+ script: 'bundle exec rake'
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in punchline.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Chris Atkins
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,71 @@
1
+ [![Build Status](https://travis-ci.org/catkins/punchline.svg)](https://travis-ci.org/catkins/punchline) [![Dependency Status](https://gemnasium.com/catkins/punchline.svg)](https://gemnasium.com/catkins/punchline) [![Coverage Status](https://img.shields.io/coveralls/catkins/punchline.svg)](https://coveralls.io/r/catkins/punchline) [![Code Climate](https://codeclimate.com/github/catkins/punchline/badges/gpa.svg)](https://codeclimate.com/github/catkins/punchline)
2
+
3
+ # Punchline
4
+
5
+ Punchline is a Redis backed Minimum Priority Queue with enforced uniqueness and atomicity fuelled by lua scripts.
6
+
7
+ ## Motivation
8
+
9
+ At Doceo, we needed a way to atomically keep track of dirty records that needed reprocessing, whilst also avoiding doing extra work if records are marked as dirty and haven't been processed yet.
10
+
11
+ ## Prerequisites
12
+
13
+ - Redis 2.6+
14
+
15
+ Currently tested against Ruby 2.0.0, 2.1.0 and JRuby
16
+
17
+ ## Installation
18
+
19
+ Add this line to your application's Gemfile:
20
+
21
+ ```ruby
22
+ gem 'punchline', github: 'catkins/punchline'
23
+ ```
24
+
25
+ And then execute:
26
+
27
+ ```bash
28
+ $ bundle
29
+ ```
30
+
31
+ ## Usage
32
+
33
+ ```ruby
34
+ require 'punchline'
35
+
36
+ # optionally override Punchline with your own Redis client, otherwise defaults to Redis.new
37
+ Punchline.config.redis = Redis.new host: "10.0.1.1", port: 6830
38
+
39
+ # create a queue
40
+ queue = Punchline::MinQueue.new
41
+ queue.length # => 0
42
+
43
+ # add a key
44
+ queue.enqueue priority: Time.now.to_i, value: 'hello!' # => true
45
+ queue.length # => 1
46
+
47
+ # shortly after... higher priority score is rejected
48
+ queue.enqueue priority: Time.now.to_i, value: 'hello!' # => false
49
+ queue.length # => 1
50
+
51
+ # original key is retrieved
52
+ queue.dequeue # => { :priority => 1411405014, :value => "hello!" }
53
+
54
+ # queue is now empty
55
+ queue.length # => 0
56
+
57
+ ```
58
+
59
+ ## TODO
60
+
61
+ - Add support for Redis::Namespace
62
+ - Come up with a gem name that isn't taken...
63
+ - Push to rubygems
64
+
65
+ ## Contributing
66
+
67
+ 1. Fork it ( http://github.com/catkins/punchline/fork )
68
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
69
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
70
+ 4. Push to the branch (`git push origin my-new-feature`)
71
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,10 @@
1
+ require 'rspec/core/rake_task'
2
+ require 'bundler/gem_tasks'
3
+
4
+ # Default directory to look in is `/specs`
5
+ # Run with `rake spec`
6
+ RSpec::Core::RakeTask.new(:spec) do |task|
7
+ task.rspec_opts = ['--color', '--format', 'doc']
8
+ end
9
+
10
+ task :default => :spec
@@ -0,0 +1,13 @@
1
+ # Encoding: utf-8
2
+
3
+ require 'redis'
4
+
5
+ module Punchline
6
+ class Configuration
7
+ attr_accessor :redis
8
+
9
+ def initialize(redis = nil)
10
+ @redis = redis || Redis.new
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,9 @@
1
+ local key = KEYS[1]
2
+
3
+ -- fetch the lowest priority value
4
+ local score = redis.call("ZRANGE", key, 0, 0, "WITHSCORES")
5
+
6
+ -- delete the smallest value
7
+ redis.call("ZREMRANGEBYRANK", key, 0, 0)
8
+
9
+ return score
@@ -0,0 +1,19 @@
1
+ local key = KEYS[1]
2
+ local priority = tonumber(ARGV[1])
3
+ local value = ARGV[2]
4
+
5
+ local current_score = tonumber(redis.call("ZSCORE", key, value))
6
+
7
+ -- first time key has been added
8
+ if current_score == nil then
9
+ redis.call("ZADD", key, priority, value)
10
+ return true
11
+ end
12
+
13
+ -- only add if it's a lower priority
14
+ if priority < current_score then
15
+ redis.call("ZADD", key, priority, value)
16
+ return true
17
+ end
18
+
19
+ return false
@@ -0,0 +1,81 @@
1
+ # Encoding: utf-8
2
+
3
+ module Punchline
4
+ class MinQueue
5
+
6
+ attr_accessor :key
7
+
8
+ def initialize(key)
9
+ @key = key
10
+ load_scripts!
11
+ end
12
+
13
+ def config
14
+ @config ||= Punchline.config.dup
15
+ end
16
+
17
+ def length
18
+ redis.zcard key
19
+ end
20
+
21
+ def all
22
+ redis.zrange(key, 0, -1, with_scores: true).map do |pair|
23
+ { value: pair.first, priority: pair.last.to_i }
24
+ end
25
+ end
26
+
27
+ def enqueue(options = {})
28
+ priority = options.fetch :priority
29
+ value = options.fetch :value
30
+ @enqueue.call([key], [priority, value]) == 1
31
+ end
32
+
33
+ def dequeue
34
+ value, priority = @dequeue.call [key]
35
+
36
+ { value: value, priority: priority.to_i } unless value.nil?
37
+ end
38
+
39
+ def redis
40
+ config.redis
41
+ end
42
+
43
+ def load_scripts!
44
+ @enqueue = Script.new redis, 'enqueue.lua'
45
+ @dequeue = Script.new redis, 'dequeue.lua'
46
+ end
47
+
48
+ def clear!
49
+ redis.del key
50
+ end
51
+
52
+ def reset_scripts!
53
+ redis.script :flush
54
+ end
55
+
56
+
57
+ class Script
58
+ SCRIPT_BASE_PATH = File.expand_path('../lua', __FILE__)
59
+
60
+ attr_accessor :redis, :body, :sha, :script_name
61
+
62
+ def initialize(redis, script_name)
63
+ @redis = redis
64
+ @script_name = script_name
65
+ end
66
+
67
+ def call(keys = [], argv = [])
68
+ load! unless @body
69
+ @redis.evalsha sha, keys, argv
70
+ end
71
+
72
+ private
73
+
74
+ def load!
75
+ path = File.join SCRIPT_BASE_PATH, script_name
76
+ @body = File.read path
77
+ @sha = redis.script :load, body
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,5 @@
1
+ # Encoding: utf-8
2
+
3
+ module Punchline
4
+ VERSION = "0.0.1"
5
+ end
data/lib/punchline.rb ADDED
@@ -0,0 +1,11 @@
1
+ require 'punchline/version'
2
+ require 'punchline/configuration'
3
+ require 'punchline/min_queue'
4
+
5
+ module Punchline
6
+ class << self
7
+ def config
8
+ @config ||= Configuration.new
9
+ end
10
+ end
11
+ end
data/punchline.gemspec ADDED
@@ -0,0 +1,28 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'punchline/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'punchline'
8
+ spec.version = Punchline::VERSION
9
+ spec.authors = ['Chris Atkins']
10
+ spec.email = ['christopherlionelatkins@gmail.com']
11
+ spec.summary = %q{Persistent redis based min-priority queue}
12
+ spec.description = %q{Persistent redis based min-priority queue.}
13
+ spec.homepage = 'http://github.com/catkins/punchline'
14
+ spec.license = 'MIT'
15
+
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ['lib']
20
+
21
+ spec.add_runtime_dependency 'redis', '>= 3.0.0'
22
+
23
+ spec.add_development_dependency 'bundler', '~> 1.5'
24
+ spec.add_development_dependency 'rake'
25
+ spec.add_development_dependency 'rspec', '~> 3.1.0'
26
+ spec.add_development_dependency 'pry'
27
+ spec.add_development_dependency 'coveralls'
28
+ end
@@ -0,0 +1,14 @@
1
+ require 'spec_helper'
2
+
3
+ module Punchline
4
+ describe Configuration do
5
+ subject(:config) { Configuration.new }
6
+
7
+ it { should respond_to :redis }
8
+
9
+ describe '#redis' do
10
+ subject(:redis) { config.redis }
11
+ it { should_not be_nil }
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,169 @@
1
+ require 'spec_helper'
2
+
3
+ module Punchline
4
+ describe MinQueue do
5
+ TEST_KEY = 'punchline:test:queue'
6
+
7
+ def clear_queue!
8
+ Redis.new.del TEST_KEY
9
+ end
10
+
11
+ before(:all) { clear_queue! }
12
+ after(:each) { clear_queue! }
13
+
14
+ let(:some_key) { TEST_KEY }
15
+ subject(:min_queue) { MinQueue.new some_key }
16
+
17
+
18
+ it { should_not be_nil }
19
+ it { should respond_to :config }
20
+
21
+ describe '#redis' do
22
+ it { should respond_to :redis }
23
+
24
+ it 'should not be nil' do
25
+ expect(subject.redis).not_to be_nil
26
+ end
27
+
28
+ it 'should be equal to config#redis' do
29
+ config = min_queue.config
30
+ expect(min_queue.redis).to eq config.redis
31
+ end
32
+ end
33
+
34
+ describe '#key' do
35
+ it { should respond_to :key }
36
+
37
+ it 'matches the constructor params' do
38
+ expect(min_queue.key).to eq some_key
39
+ end
40
+ end
41
+
42
+ describe '#length' do
43
+ it { should respond_to :length }
44
+
45
+ let(:mock_redis) { double 'redis', zcard: 5, del: true }
46
+
47
+ it 'is initially zero' do
48
+ expect(subject.length).to eq 0
49
+ end
50
+
51
+ it 'checks the cardinality of sorted set in redis' do
52
+ subject.config.redis = mock_redis
53
+ expect(mock_redis).to receive(:zcard).with(some_key)
54
+ subject.length
55
+ end
56
+
57
+ it 'returns the length of the sorted set' do
58
+ subject.config.redis = mock_redis
59
+ expect(subject.length).to eq 5
60
+ end
61
+ end
62
+
63
+ describe '#all' do
64
+ it { should respond_to :all }
65
+
66
+ it 'delegates to reading redis range' do
67
+ expect(subject.redis).to receive(:zrange).with(some_key, 0, -1, with_scores: true)
68
+ .and_return([])
69
+ subject.all
70
+ end
71
+
72
+ it 'returns an array' do
73
+ expect(subject.all).to be_kind_of Array
74
+ end
75
+
76
+ it 'returns elements as hashes' do
77
+ subject.enqueue priority: 123, value: 'hello'
78
+
79
+ hash = subject.all.first
80
+ expect(hash).not_to be_nil
81
+ expect(hash[:priority]).to eq 123
82
+ expect(hash[:value]).to eq 'hello'
83
+ end
84
+ end
85
+
86
+ describe '#enqueue' do
87
+ after(:each) do
88
+ subject.reset_scripts!
89
+ end
90
+
91
+ it { should respond_to :enqueue }
92
+
93
+ it 'increases the length by 1' do
94
+ expect {
95
+ subject.enqueue priority: 123, value: 'hello'
96
+ }.to change {
97
+ subject.length
98
+ }.by 1
99
+ end
100
+
101
+ it 'returns true when key is written' do
102
+ result = subject.enqueue priority: 123, value: 'hello'
103
+ expect(result).to eq true
104
+ end
105
+
106
+ it 'returns false when key is not written' do
107
+ result = subject.enqueue priority: 123, value: 'hello'
108
+ result = subject.enqueue priority: 456, value: 'hello'
109
+ expect(result).to eq false
110
+ end
111
+
112
+ describe 'duplicates' do
113
+ it 'ignores duplicate values' do
114
+ subject.enqueue priority: 123, value: 'hello'
115
+
116
+ expect {
117
+ subject.enqueue priority: 567, value: 'hello'
118
+ }.to change {
119
+ subject.length
120
+ }.by(0)
121
+ end
122
+
123
+ it 'retains only the lowest priority score' do
124
+ subject.enqueue priority: 123, value: 'hello'
125
+ subject.enqueue priority: 567, value: 'hello'
126
+ subject.enqueue priority: 789, value: 'hello'
127
+
128
+ pair = subject.dequeue
129
+ expect(pair[:priority]).to eq 123
130
+ end
131
+ end
132
+ end
133
+
134
+ describe '#dequeue' do
135
+ after(:each) do
136
+ subject.reset_scripts!
137
+ subject.clear!
138
+ end
139
+
140
+ it { should respond_to :dequeue }
141
+
142
+ it 'decreases the length by 1' do
143
+ subject.enqueue priority: 123, value: 'hello'
144
+
145
+ expect{ subject.dequeue }.to change { subject.length }.by -1
146
+ end
147
+
148
+ it 'returns the pair with the lowest score' do
149
+ subject.enqueue priority: 123, value: 'hello'
150
+ subject.enqueue priority: 567, value: 'world'
151
+ expect(subject.dequeue).to eq({ value: 'hello', priority: 123 })
152
+ end
153
+ end
154
+
155
+ describe '#clear!' do
156
+ it { should respond_to :clear! }
157
+
158
+ it 'deletes the key on redis' do
159
+ expect(subject.redis).to receive(:del).with(some_key)
160
+ subject.clear!
161
+ end
162
+
163
+ it 'resets the length' do
164
+ subject.enqueue priority: 123, value: 'hello'
165
+ expect { subject.clear! }.to change { subject.length }.by -1
166
+ end
167
+ end
168
+ end
169
+ end
@@ -0,0 +1,5 @@
1
+ require 'coveralls'
2
+ Coveralls.wear!
3
+
4
+ require 'pry'
5
+ require 'punchline'
metadata ADDED
@@ -0,0 +1,149 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: punchline
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Chris Atkins
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-09-23 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: redis
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 3.0.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '>='
25
+ - !ruby/object:Gem::Version
26
+ version: 3.0.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '1.5'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: '1.5'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '>='
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rspec
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ~>
60
+ - !ruby/object:Gem::Version
61
+ version: 3.1.0
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ~>
67
+ - !ruby/object:Gem::Version
68
+ version: 3.1.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: pry
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '>='
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '>='
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: coveralls
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '>='
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '>='
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
97
+ description: Persistent redis based min-priority queue.
98
+ email:
99
+ - christopherlionelatkins@gmail.com
100
+ executables: []
101
+ extensions: []
102
+ extra_rdoc_files: []
103
+ files:
104
+ - .coveralls.yml
105
+ - .gitignore
106
+ - .rspec
107
+ - .travis.yml
108
+ - Gemfile
109
+ - LICENSE.txt
110
+ - README.md
111
+ - Rakefile
112
+ - lib/punchline.rb
113
+ - lib/punchline/configuration.rb
114
+ - lib/punchline/lua/dequeue.lua
115
+ - lib/punchline/lua/enqueue.lua
116
+ - lib/punchline/min_queue.rb
117
+ - lib/punchline/version.rb
118
+ - punchline.gemspec
119
+ - spec/mindy/configuration_spec.rb
120
+ - spec/mindy/min_queue_spec.rb
121
+ - spec/spec_helper.rb
122
+ homepage: http://github.com/catkins/punchline
123
+ licenses:
124
+ - MIT
125
+ metadata: {}
126
+ post_install_message:
127
+ rdoc_options: []
128
+ require_paths:
129
+ - lib
130
+ required_ruby_version: !ruby/object:Gem::Requirement
131
+ requirements:
132
+ - - '>='
133
+ - !ruby/object:Gem::Version
134
+ version: '0'
135
+ required_rubygems_version: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - '>='
138
+ - !ruby/object:Gem::Version
139
+ version: '0'
140
+ requirements: []
141
+ rubyforge_project:
142
+ rubygems_version: 2.3.0
143
+ signing_key:
144
+ specification_version: 4
145
+ summary: Persistent redis based min-priority queue
146
+ test_files:
147
+ - spec/mindy/configuration_spec.rb
148
+ - spec/mindy/min_queue_spec.rb
149
+ - spec/spec_helper.rb