reliable-queue-rb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e4da5c91348c8e894a37d4c8d260160cb5d7bbca55a30fe1f69f99c333f349c8
+  data.tar.gz: ddde902f582ec54278f70ca149b98574476fd6e39b40e9755c1d87c8bbbe7af2
+SHA512:
+  metadata.gz: 282b4a5e12f6dcbf3039c6402356f53f3c3a2eaa6fedfa8102a535c5ef75a6b38e9d7296e47dad9b66c686ff8c232d34bf2d20c83bbfc143fb070314d244501a
+  data.tar.gz: 041ffe4a0f9fa508b4e059c461efe2dffbc5d216c8f56e72c1afc54e36e863c60f275e3086e2df964e1fa101cef8d70716379be84f2fb354d4f1f4d6853c615a

data/CHANGELOG.md

@@ -0,0 +1,7 @@
+# Change Log
+All notable changes to this project will be documented in this file. This
+project adheres to [Semantic Versioning](http://semver.org/).
+
+## [0.1.0] - 2020-12-14
+### Added
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020-2021 Altmetric LLP
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# Reliable Queue [![Build Status](https://api.travis-ci.com/altmetric/reliable-queue-rb.svg?branch=main)](https://travis-ci.com/github/altmetric/reliable-queue-rb)
+
+Ruby reliable queue implementation on top of Redis. It makes sure that message is not lost between popping it from Redis queue and compeleting the task.
+
+## Installation
+
+Add this line to your application's `Gemfile`:
+
+```ruby
+gem 'reliable-queue-rb', '~> 0.1.0'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install reliable-queue-rb
+
+## Usage
+Reliable Queue
+```ruby
+queue = ReliableQueue.new(redis_queue, redis_client)
+
+queue.each do |message|
+  # process message
+end
+```
+
+ChunkedReliableQueue
+```ruby
+queue = ChunkedReliableQueue.new(working_on_queue_suffix, redis_queue, redis_client)
+
+queue.each_slice(batch_size) do |messages|
+  # process array of messages
+end
+```
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/altmetric/reliable-queue-rb.
+
+## License
+
+Copyright © 2020-2021 Altmetric LLP
+
+Distributed under the [MIT License](http://opensource.org/licenses/MIT).

data/lib/chunked_reliable_queue.rb

@@ -0,0 +1,43 @@
+class ChunkedReliableQueue
+  DEFAULT_SIZE = 100
+
+  attr_reader :name, :queue, :size, :working_queue, :redis
+
+  def initialize(name, queue, redis)
+    @name = name
+    @queue = queue
+    @redis = redis
+    @working_queue = "#{queue}.working_on.#{name}"
+
+    requeue_unfinished_work
+  end
+
+  def each_slice(size = DEFAULT_SIZE)
+    return enum_for(:each_slice, size) unless block_given?
+
+    loop do
+      blocking_reply = redis.brpoplpush(queue, working_queue, 30)
+      next unless blocking_reply
+
+      replies = [blocking_reply]
+      replies += redis.multi { |multi|
+        (size - 1).times do
+          multi.rpoplpush(queue, working_queue)
+        end
+      }.compact
+
+      yield replies
+      redis.multi do |multi|
+        replies.each do |reply|
+          multi.lrem(working_queue, 0, reply)
+        end
+      end
+    end
+  end
+
+  private
+
+  def requeue_unfinished_work
+    loop while redis.rpoplpush(working_queue, queue)
+  end
+end

data/lib/reliable_queue.rb

@@ -0,0 +1,31 @@
+class ReliableQueue
+  include Enumerable
+
+  attr_reader :queue, :working_queue, :redis
+
+  def initialize(queue, redis)
+    @queue = queue
+    @redis = redis
+    @working_queue = "#{queue}.working_on"
+
+    requeue_unfinished_work
+  end
+
+  def each
+    return enum_for(:each) unless block_given?
+
+    loop do
+      reply = redis.brpoplpush(queue, working_queue, 30)
+      next unless reply
+
+      yield reply
+      redis.lrem(working_queue, 0, reply)
+    end
+  end
+
+  private
+
+  def requeue_unfinished_work
+    loop while redis.rpoplpush(working_queue, queue)
+  end
+end

data/spec/chunked_reliable_queue_spec.rb

@@ -0,0 +1,55 @@
+require 'chunked_reliable_queue'
+require "redis"
+
+RSpec.describe ChunkedReliableQueue do
+  let(:redis) { Redis.new }
+
+  after do
+    redis.flushall
+  end
+
+  describe '.new' do
+    it 'requeues any unfinished work for the given queue' do
+      redis.lpush('foo.working_on.name1', '1')
+      queue = described_class.new('name1', 'foo', redis)
+      enum = queue.each_slice(1)
+
+      expect(enum.next).to eq(['1'])
+    end
+
+    it 'does not block if there is no unfinished work' do
+      queue = described_class.new('name1', 'foo', redis)
+
+      expect(queue).to be_a(described_class)
+    end
+  end
+
+  describe '#each_slice' do
+    it 'returns work from the queue in order' do
+      redis.lpush('foo', '1')
+      redis.lpush('foo', '2')
+      queue = described_class.new('name1', 'foo', redis)
+      enum = queue.each_slice(2)
+
+      expect(enum.next).to eq(['1', '2'])
+    end
+
+    it 'return chunks using a block' do
+      redis.lpush('foo', '1')
+      redis.lpush('foo', '2')
+      redis.lpush('foo', '3')
+      redis.lpush('foo', '4')
+      queue = described_class.new('name1', 'foo', redis)
+
+      result = []
+      iteration = 0
+      queue.each_slice(2) do |slice|
+        result += slice
+        iteration += 1
+        break if iteration >= 2
+      end
+
+      expect(result).to eq(['1', '2', '3', '4'])
+    end
+  end
+end

data/spec/reliable_queue_spec.rb

@@ -0,0 +1,35 @@
+require 'reliable_queue'
+require "redis"
+
+RSpec.describe ReliableQueue do
+  let(:redis) { Redis.new }
+
+  after do
+    redis.del('foo')
+  end
+
+  describe '.new' do
+    it 'requeues any unfinished work for the given queue' do
+      redis.rpush('foo.working_on', '1')
+      queue = described_class.new('foo', redis)
+
+      expect(queue.first).to eq('1')
+    end
+
+    it 'does not block if there is no unfinished work' do
+      queue = described_class.new('foo', redis)
+
+      expect(queue).to be_a(described_class)
+    end
+  end
+
+  describe '#each' do
+    it 'returns work from the queue in order' do
+      redis.lpush('foo', '1')
+      redis.lpush('foo', '2')
+      queue = described_class.new('foo', redis)
+
+      expect(queue.take(2)).to eq(['1', '2'])
+    end
+  end
+end

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: reliable-queue-rb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Anna Klimas
+- Jonathan Hernandez
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-12-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+description:
+email:
+- support@altmetric.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- lib/chunked_reliable_queue.rb
+- lib/reliable_queue.rb
+- spec/chunked_reliable_queue_spec.rb
+- spec/reliable_queue_spec.rb
+homepage: https://github.com/altmetric/reliable-queue-rb
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.6
+signing_key:
+specification_version: 4
+summary: Ruby library for reliable queue processing.
+test_files:
+- spec/reliable_queue_spec.rb
+- spec/chunked_reliable_queue_spec.rb