reliable-queue 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef20e5154d605fc6c1469cb10e9a3cdfc656c0c2
+  data.tar.gz: aa28b4ee3dbd541441f83f317aab458ac146182a
+SHA512:
+  metadata.gz: 0bde4b76e29f9b12c67f2e37be07401955b32e0ea0ed949fcf3273a4d7c46bb3783c8545f3a078050d781ababcc673ad6a9e3ac0ca1d581d34909c8a67ff16db
+  data.tar.gz: 205b8d56659f268522b14bfbdc246ec94b4d865e0606ce605c67bf0a0676f1d8ec4d16450f7c982a2e5f79a6c960a3efb1d2f7e0c13d299a6e02dff1233aaed2

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 2.2.2
+  - 1.9.3
+services:
+  - redis
+before_install: gem install bundler -v 1.10.5

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in reliable.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 myobie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,154 @@
+[![Build
+Status](https://travis-ci.org/wunderlist/reliable.svg?branch=master)](https://travis-ci.org/wunderlist/reliable)
+
+# Reliable is.
+
+Redis is a great storage service for building a reliable queue. That's
+what this is for.
+
+## Is this like [Ost](https://github.com/soveran/ost)?
+
+Ost was the inspiration for this project. We love ost, but it lacks a few of the nice things we want (retry, failures count, etc) and will will implement those extra features here. We also wanted parallelism baked in.
+
+## Configuring redis
+
+One must set the `REDIS_URL` environment variable.
+
+## Enqueueing messages
+
+The developer is responsible for enqueuing `String`s or string-like
+objects.
+
+```ruby
+Reliable[:messages].push(JSON.generate({
+  id: 123,
+  title: "Hello"
+}))
+```
+
+In this example `:messages` is the queue name. The developer can make as
+many queues as they like.
+
+## Processing messages
+
+### Processing all messages as they arrive
+
+```ruby
+Reliable[:messages].each do |message|
+  hash = JSON.generate(message)
+  DatabaseTable.find(hash["id"]).do_something_awesome(message)
+end
+```
+
+or
+
+```ruby
+Reliable[:emails].each do |message|
+  hash = JSON.generate(message)
+  Emailer.new(hash).deliver
+end
+```
+
+Calling `#each` will block the main thread and sleep forever.
+
+### Processing messages in parallel
+
+If the processing code is thread-safe, the developer can spawn any
+number of threads with `#peach`:
+
+```ruby
+Reliable[:touches].peach(concurrency: 12) { |id| Model.find(id).touch }
+```
+
+In this example 12 threads will be created and all are joined with the
+main thread.
+
+### Processing some messages
+
+It's also possible to process only some message by `#take`-ing as many
+as necessary:
+
+```ruby
+Reliable[:urls].take(2) do |url|
+  content = open(url)
+  PersistentStore.store(content)
+end
+```
+
+Or if you just want the urls themselves as an array:
+
+```ruby
+urls = Reliable[:urls].take(2)
+
+urls.each do |url|
+  content = open(url)
+  PersistentStore.store(content)
+end
+```
+
+And if the developer wants, they can get an enumerator object and
+interact with it as necessary:
+
+```ruby
+enumerator = Reliable[:ids].to_enum { |id| notify(id) }
+assert_equal 0, notifications.length
+4.times { enumerator.next }
+assert_equal 4, notifications.length
+```
+
+## Time
+
+Make sure the distributed clock starts moving before you lock the main thread.
+The clock makes it possible to re-enqueue stale items.
+
+Here is a full example:
+
+```ruby
+Reliable[:emails].periodically_move_time_forward
+Reliable[:emails].peach(concurrency: 6) do |message|
+  hash = JSON.generate(message)
+  Emailer.new(hash).deliver
+end
+```
+
+## Rails
+
+Probably best to create an initializer:
+
+```ruby
+# config/initializers/reliable.rb
+Reliable[:emails].periodically_move_time_forward
+```
+
+Then in a controller one might:
+
+```ruby
+# app/controllers/users_controller.rb
+class UsersController < ApplicationController
+  def create
+    @user = User.create!(params.require(:email))
+    Reliable[:emails].push JSON.generate({
+      user_id: @user.id
+    })
+    redirect_to root_url
+  end
+end
+```
+
+Then, in a worker file:
+
+```ruby
+# app/workers/emails_worker.rb
+Reliable[:emails].peach(concurrency: 6) do |message|
+  hash = JSON.parse(message)
+  user = User.find(hash[:user_id])
+  Emailer.welcome_email(user).deliver
+end
+```
+
+And maybe one would have a `Procfile` like this:
+
+```
+web: bin/rails s -p$PORT
+worker: bin/rails r app/workers/emails_worker.rb
+```

data/Rakefile

@@ -0,0 +1,17 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec
+
+task :console do
+  $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+  ENV["RELIABLE_TIMEOUT"] = "1"
+  ENV["RELIABLE_TIME_TRAVEL_DELAY"] = "1"
+  ENV["REDIS_URI"] = "redis://127.0.0.1:6379/0"
+  require 'reliable'
+  require 'irb'
+  ARGV.clear
+  IRB.start
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "reliable"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/reliable.rb

@@ -0,0 +1,33 @@
+require_relative 'reliable/version'
+require_relative 'reliable/queue'
+require_relative 'reliable/redis'
+
+module Reliable
+  POP_TIMEOUT = ENV.fetch("RELIABLE_TIMEOUT", "2").to_i
+  TIME_TRAVEL_DELAY = ENV.fetch("RELIABLE_TIME_TRAVEL_DELAY", "60").to_i
+  PROCESSING_TIMEOUT = ENV.fetch("PROCESSING_TIMEOUT", "120").to_i
+
+  @queues = Hash.new do |h, k|
+    h[k] = Queue.new(k)
+  end
+
+  def self.[](queue)
+    @queues[queue]
+  end
+
+  class NullLogger
+    def debug(*); end
+    def info(*); end
+    def error(*); end
+    def warn(*); end
+    def fatal(*); end
+  end
+
+  def self.logger
+    @logger ||= NullLogger.new
+  end
+
+  def self.logger=(new_logger)
+    @logger = new_logger
+  end
+end

data/lib/reliable/list.rb

@@ -0,0 +1,14 @@
+module Reliable
+  class List
+    attr_reader :key
+
+    def initialize(key, redis)
+      @key = key
+      @redis = redis
+    end
+
+    def llen
+      @redis.llen(key)
+    end
+  end
+end

data/lib/reliable/queue.rb

@@ -0,0 +1,111 @@
+require 'logger'
+require 'securerandom'
+
+require_relative '../reliable'
+require_relative 'list'
+require_relative 'uuid'
+require_relative 'worker'
+
+module Reliable
+  class Queue
+    FatalError = Class.new(StandardError)
+
+    attr_accessor :base_key, :uuid
+
+    def initialize(name)
+      @name = name
+      @base_key = "reliable:queues:#{name}"
+      @redis = Redis.new
+      @pending_key = @base_key + ":pending"
+      @failed_key = @base_key + ":failed"
+    end
+
+    def current_time
+      @redis.time
+    end
+
+    def push(value)
+      UUID.new(current_time) do |uuid|
+        @redis.set_and_lpush(pending.key, uuid.to_s, value)
+      end
+    end
+    alias_method :<<, :push
+
+    def pending
+      @pending ||= List.new(@pending_key, @redis)
+    end
+
+    def failed
+      @failed ||= List.new(@failed_key, @redis)
+    end
+
+    def create_worker(&work)
+      Worker.new(self, &work)
+    end
+
+    def to_enum(&work)
+      work ||= ->(item) { item }
+      worker = create_worker(&work)
+      Enumerator.new do |y|
+        loop do                   # forever
+          result = worker.next    # do work
+          y.yield result          # then release control
+        end
+      end
+    end
+
+    def take(number, &block)
+      to_enum(&block).take(number)
+    end
+
+    def each(&block)
+      if block_given?
+        # This is because we confuse iteration with work here
+        # So we need to front-load the work, then fake the iteration
+        to_enum(&block).each { |item| item }
+      else
+        to_enum
+      end
+    end
+
+    def peach(opts = {}, &block)
+      raise "must supply a block" unless block_given?
+
+      concurrency = opts.fetch(:concurrency)
+
+      threads = concurrency.times.map do
+        Thread.new { each(&block) }
+      end
+
+      threads.map { |t| t.abort_on_exception = true }
+      threads.map(&:join)
+    end
+
+    def total_processing
+      keys = @redis.scan "reliable:queues:*:workers:*:processing"
+
+      lengths = @redis.pipeline do |pipe|
+        keys.each do |key|
+          pipe.llen key
+        end
+      end
+
+      lengths.map(&:to_i).reduce(0, &:+)
+    end
+
+    def total_items
+      @redis.scan("reliable:items:*").length
+    end
+
+    def logger
+      Reliable.logger
+    end
+
+    def notify(e, other = {})
+      # TODO: make configurable
+      logger.info e.inspect
+      logger.info e.backtrace
+      logger.info other.inspect
+    end
+  end
+end

data/lib/reliable/redis.rb

@@ -0,0 +1,121 @@
+require 'redic'
+
+module Reliable
+  class Redis
+    def initialize
+      url = ENV.fetch("REDIS_URI") { ENV.fetch("REDIS_URL") }
+      @connection = Redic.new(url)
+      @mutex = Mutex.new
+    end
+
+    def synchronize
+      @mutex.synchronize { yield }
+    end
+
+    def command(*args)
+      @connection.call!(*args)
+    end
+
+    def scommand(*args)
+      synchronize { command(*args) }
+    end
+
+    def time
+      scommand("TIME").join(".")
+    end
+
+    def get(key)
+      scommand "GET", key
+    end
+
+    def set(key, value)
+      scommand "SET", key, value
+    end
+
+    def incr(key)
+      scommand "INCR", key
+    end
+
+    def llen(key)
+      scommand "LLEN", key
+    end
+
+    def brpoplpush(pop_key, push_key, timeout = POP_TIMEOUT)
+      scommand "BRPOPLPUSH", pop_key, push_key, timeout
+    end
+
+    def rpoplpush(pop_key, push_key)
+      scommand "RPOPLPUSH", pop_key, push_key
+    end
+
+    def lpush(key, value)
+      scommand "LPUSH", key, value
+    end
+
+    def lpop(key)
+      scommand "LPOP", key
+    end
+
+    class Pipeline
+      def initialize(conn)
+        @connection = conn
+      end
+
+      def queue(*args)
+        @connection.queue(*args)
+      end
+      alias_method :q, :queue
+
+      def llen(key)
+        queue "LLEN", key
+      end
+    end
+
+    def pipeline
+      synchronize do
+        @connection.reset
+        pipe = Pipeline.new(@connection)
+        yield(pipe)
+        @connection.commit
+      end
+    end
+
+    def multi
+      synchronize do
+        begin
+          command "MULTI"
+          yield
+          command "EXEC"
+        rescue StandardError => e
+          command "DISCARD"
+          raise
+        end
+      end
+    end
+
+    def set_and_lpush(list_key, key, value)
+      multi do
+        command "SET", key, value
+        command "LPUSH", list_key, key
+      end
+    end
+
+    def lpop_and_del(list_key, key)
+      multi do
+        command "LPOP", list_key
+        command "DEL", key
+      end
+    end
+
+    def scan(pattern)
+      keys = []
+      cursor = "0"
+      loop do
+        cursor, list = scommand "SCAN", cursor, "MATCH", pattern
+        keys << list
+        break if cursor == "0"
+      end
+      keys.flatten.compact.uniq
+    end
+  end
+end

data/lib/reliable/uuid.rb

@@ -0,0 +1,38 @@
+require 'securerandom'
+
+module Reliable
+  MalformedUUID = Class.new(StandardError)
+
+  class UUID
+    def self.parse(key)
+      _, _, time, random, tries = key.split(":")
+      raise MalformedUUID unless time && random && tries
+      new(time, random, tries)
+    end
+
+    attr_reader :random
+
+    def initialize(time, random = SecureRandom.uuid, tries = 0)
+      @time = time
+      @random = random
+      @tries = tries || 0
+      yield(self) if block_given?
+    end
+
+    def time
+      @time.to_i
+    end
+
+    def tries
+      @tries.to_i
+    end
+
+    def incr
+      @tries = tries + 1
+    end
+
+    def to_s
+      "reliable:items:#{time}:#{random}:#{tries}"
+    end
+  end
+end

data/lib/reliable/version.rb

@@ -0,0 +1,3 @@
+module Reliable
+  VERSION = "0.1.0"
+end

data/lib/reliable/worker.rb

@@ -0,0 +1,62 @@
+require 'securerandom'
+require 'delegate'
+
+module Reliable
+  class Worker
+    extend Forwardable
+
+    delegate [:pending, :failed] => :@queue
+
+    def initialize(queue, &block)
+      @queue = queue
+      @block = block
+      @processing_key = "#{queue.base_key}:workers:#{SecureRandom.uuid}:processing"
+      @redis = Redis.new
+    end
+
+    def processing
+      @processing ||= List.new(@processing_key, @redis)
+    end
+
+    def next
+      uuid = @redis.brpoplpush pending.key, processing.key
+      return if uuid.nil?
+
+      item = @redis.get uuid
+
+      if item
+        catch(:failed) do
+          process item, &@block
+          @redis.lpop_and_del processing.key, uuid
+          logger.info "Processed #{uuid}"
+        end
+      end
+
+      item
+    rescue StandardError => e
+      @redis.rpoplpush processing.key, failed.key
+      notify(e, uuid: uuid, worker: processing.key)
+      nil
+    end
+
+    def process(item, &block)
+      block.yield item
+    rescue StandardError => e
+      @redis.rpoplpush processing.key, failed.key
+      notify(e, worker: processing.key)
+      throw :failed
+    end
+    private :process
+
+    def logger
+      Reliable.logger
+    end
+
+    def notify(e, other = {})
+      # TODO: make configurable
+      logger.info e.inspect
+      logger.info e.backtrace
+      logger.info other.inspect
+    end
+  end
+end

data/reliable.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'reliable/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "reliable-queue"
+  spec.version       = Reliable::VERSION
+  spec.authors       = ["myobie"]
+  spec.email         = ["me@nathanherald.com"]
+
+  spec.summary       = %q{A reliable queue using redis}
+  spec.description   = %q{A relialbe queue library for enqueuing and creating workers.}
+  spec.homepage      = "https://github.com/wunderlist/reliable"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+
+  spec.add_dependency "redic"
+end

metadata

@@ -0,0 +1,119 @@
+--- !ruby/object:Gem::Specification
+name: reliable-queue
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- myobie
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-12-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A relialbe queue library for enqueuing and creating workers.
+email:
+- me@nathanherald.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/reliable.rb
+- lib/reliable/list.rb
+- lib/reliable/queue.rb
+- lib/reliable/redis.rb
+- lib/reliable/uuid.rb
+- lib/reliable/version.rb
+- lib/reliable/worker.rb
+- reliable.gemspec
+homepage: https://github.com/wunderlist/reliable
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.0
+signing_key: 
+specification_version: 4
+summary: A reliable queue using redis
+test_files: []