shoryuken-later 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzYxOWE4MDhiZmVkMjc5YjBjNTE4ZjJiY2VmOTY1OGI1MDhkYWU3NA==
+  data.tar.gz: !binary |-
+    ODMzY2ZkNGExNzlmNjYwZWI3NjcxMWI4MWE3YWQ0NDE4MmZkZDBiYw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    Yjg1OGYxNzRiOWUxNDg5NTVkZGQxNGRmNzM5Njc5YzQzNTcxZTUxM2U3MTRj
+    NDU1YmRiYzQxYTgzMDQ2MWNlNDcyZGU1MDMyOGMzZTFiMTg2MjA0ZTQzNmIz
+    MzlkZGZmYmM2N2EyMTI1MzI1MGYwNTg1ZjUwOTk0ZTRhNThkOGY=
+  data.tar.gz: !binary |-
+    MjJjY2Q5MWViNTBlMDFjNjc4Y2RiNWYwZDE0MTYwNThiZTMyMzI2ZWMyNjdi
+    MjhkMzA0ZjAzNGMxMWE0OTU5NzAzNzE1NGY2NTdkMDUyOTVjYjAwNDdmY2Y3
+    ZDRlNTlmNjI2ZGQ5MDJlOWIwZjIxMTJlMWI5ZWUxNjZiYzVmZTc=

data/.gitignore

@@ -0,0 +1,22 @@
+*.rbc
+.rspec
+/pkg
+/log
+/tmp
+/spec/tmp
+**.orig
+/.buildpath
+/.project
+/.settings
+shoryuken.yml
+
+## Environment normalisation:
+/.bundle
+/vendor/bundle
+
+# these should all be checked in to normalise the environment:
+# Gemfile.lock, .ruby-version, .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+/Gemfile.lock

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in shoryuken-later.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,166 @@
+GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.
+

data/README.md

@@ -0,0 +1,63 @@
+# shoryuken-later
+
+A scheduling plugin for [Shoryuken](https://github.com/phstc/shoryuken) that uses [Dynamo DB](https://aws.amazon.com/dynamodb/)
+to schedule messages arbitrarily far into the future.
+
+## Features
+
+### Integration with Shoryuken::Worker
+
+A new method named `perform_later` is added to `Shoryuken::Worker` allowing messages to be delayed arbitrarily far into the future. If the delay is 15 minutes or less, then the message is enqueued into the specified SQS `:queue` as usual.  Otherwise, the message is inserted into the specified DynamoDB `:schedule_table`.
+
+```ruby
+ require 'shoryuken-later'
+
+ class MyWorker
+   include Shoryuken::Worker
+  
+   shoryuken_options queue: 'default', schedule_table: 'default_schedule'
+ end
+
+ # Schedules a message to be processed 30 minutes from now.
+ MyWorker.perform_later(Time.now + 30 * 60, 'Foobar')
+```
+
+### One or more schedule tables
+
+Supports polling one or more DynamoDB tables for messages.
+
+```yaml
+later:
+  tables:
+    - default_schedule
+    - other_schedule
+```
+
+### Namespaced configuration
+
+You can use the same configuration file for both `Shoryuken` and `Shoryuken::Later`, because the new configuration options are namespaced.
+
+```yaml
+# This key is used by both Shoryuken and Shoryuken::Later
+aws:
+  access_key_id:      ...       # or <%= ENV['AWS_ACCESS_KEY_ID'] %>
+  secret_access_key:  ...       # or <%= ENV['AWS_SECRET_ACCESS_KEY'] %>
+  region:             us-east-1 # or <%= ENV['AWS_REGION'] %>
+  
+# This key is only used by Shoryuken::Later
+later:
+  delay: 5 * 60   # How frequently to poll the schedule table, in seconds.
+  tables:
+    - table1
+    
+# These keys are used by both Shoryuken and Shoryuken::Later
+logfile: some/path/to/file.log
+pidfile: some/path/to/file.pid
+
+# These keys are only used by Shoryuken
+concurrency: 3
+delay: 0
+queues:
+  - [queue1, 1]
+  - [queue2, 2]
+```

data/Rakefile

@@ -0,0 +1,13 @@
+require 'bundler/gem_tasks'
+
+$stdout.sync = true
+
+begin
+  require 'rspec/core/rake_task'
+
+  RSpec::Core::RakeTask.new(:spec)
+
+  task :default => :spec
+rescue LoadError
+  # no rspec available
+end

data/bin/shoryuken-later

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/shoryuken/later/cli'
+
+begin
+  Shoryuken::Later::CLI.instance.run(ARGV)
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/lib/shoryuken/later/active_job_adapter.rb

@@ -0,0 +1,73 @@
+# Practically all of this has been "borrowed" from Shoryuken.
+
+# @see ActiveJob::QueueAdapter::ShoryukenAdapter
+
+# ActiveJob docs: http://edgeguides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+
+require 'shoryuken-later'
+
+module ActiveJob
+  module QueueAdapters
+    # == Shoryuken::Later adapter for Active Job
+    #
+    # Shoryuken ("sho-ryu-ken") is a super-efficient AWS SQS thread based message processor.
+    # Shoryuken::Later allows messages to be delayed arbitrarily far into the future.
+    #
+    # Read more about Shoryuken {here}[https://github.com/phstc/shoryuken].
+    # Read more about Shoryuken::Later {here}[https://github.com/joekhoobyar/shoryuken-later].
+    #
+    # To use Shoryuken::Later set the queue_adapter config to +:shoryuken_later+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :shoryuken_later
+    class ShoryukenLaterAdapter
+      class << self
+        def enqueue(job) #:nodoc:
+          register_worker!(job)
+
+          Shoryuken::Client.send_message(job.queue_name, job.serialize, message_attributes: message_attributes)
+        end
+
+        def enqueue_at(job, timestamp) #:nodoc:
+          register_worker!(job)
+
+          delay = (timestamp - Time.current.to_f).round
+          if delay > 15.minutes
+            Shoryuken::Later::Client.put_item(Shoryuken::Later.default_table, perform_at: delay.to_i,
+                                                                              shoryuken_queue: job.queue_name, shoryuken_class: JobWrapper.to_s,
+                                                                              shoryuken_args: JSON.dump(body: job.serialize, options: {}))
+          else
+            Shoryuken::Client.send_message(job.queue_name, job.serialize, delay_seconds: delay,
+                                                                          message_attributes: message_attributes)
+          end
+        end
+
+
+        private
+
+        def register_worker!(job)
+          Shoryuken.register_worker(job.queue_name, JobWrapper)
+        end
+
+        def message_attributes
+          @message_attributes ||= {
+            'shoryuken_class' => {
+              string_value: JobWrapper.to_s,
+              data_type: 'String'
+            }
+          }
+        end
+      end
+
+      class JobWrapper #:nodoc:
+        include Shoryuken::Worker
+
+        shoryuken_options body_parser: :json, auto_delete: true
+
+        def perform(sqs_msg, hash)
+          Base.execute hash
+        end
+      end
+    end
+  end
+end

data/lib/shoryuken/later/cli.rb

@@ -0,0 +1,281 @@
+# Most of this has been "borrowed" from Shoryuken.
+
+# @see Shoryuken::CLI
+$stdout.sync = true
+
+require 'singleton'
+require 'optparse'
+require 'erb'
+require 'shoryuken/later'
+
+module Shoryuken
+  module Later
+    class CLI
+      include Shoryuken::Util
+      include Singleton
+
+      attr_accessor :launcher
+      
+      def run(args)
+        self_read, self_write = IO.pipe
+
+        %w[INT TERM USR1 USR2 TTIN].each do |sig|
+          trap sig do
+            self_write.puts(sig)
+          end
+        end
+
+        setup_options(args) do |cli_options|
+          # this needs to happen before configuration is parsed, since it may depend on Rails env
+          load_rails if cli_options[:rails]
+        end
+        initialize_logger
+        require_workers
+        validate!
+        daemonize
+        write_pid
+        load_celluloid
+
+        require 'shoryuken/later/launcher'
+        @launcher = Shoryuken::Later::Launcher.new
+
+        begin
+          launcher.run
+
+          while readable_io = IO.select([self_read])
+            signal = readable_io.first[0].gets.strip
+            handle_signal(signal)
+          end
+        rescue Interrupt
+          launcher.stop(shutdown: true)
+          exit 0
+        end
+      end
+
+      private
+
+      def load_celluloid
+        raise "Celluloid cannot be required until here, or it will break Shoryuken::Later's daemonization" if defined?(::Celluloid) && Shoryuken::Later.options[:daemon]
+
+        # Celluloid can't be loaded until after we've daemonized
+        # because it spins up threads and creates locks which get
+        # into a very bad state if forked.
+        require 'celluloid/autostart'
+        Celluloid.logger = (Shoryuken::Later.options[:verbose] ? Shoryuken::Later.logger : nil)
+
+        require 'shoryuken/later/manager'
+      end
+
+      def load_rails
+        # Adapted from: https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/cli.rb
+
+        require 'rails'
+        if ::Rails::VERSION::MAJOR < 4
+          require File.expand_path("config/environment.rb")
+          ::Rails.application.eager_load!
+        else
+          # Painful contortions, see 1791 for discussion
+          require File.expand_path("config/application.rb")
+          ::Rails::Application.initializer "shoryuken-later.eager_load" do
+            ::Rails.application.config.eager_load = true
+          end
+          require File.expand_path("config/environment.rb")
+        end
+
+        logger.info "Rails environment loaded"
+      end
+
+      def daemonize
+        return unless Shoryuken::Later.options[:daemon]
+
+        raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless Shoryuken::Later.options[:logfile]
+
+        files_to_reopen = []
+        ObjectSpace.each_object(File) do |file|
+          files_to_reopen << file unless file.closed?
+        end
+
+        Process.daemon(true, true)
+
+        files_to_reopen.each do |file|
+          begin
+            file.reopen file.path, "a+"
+            file.sync = true
+          rescue ::Exception
+          end
+        end
+
+        [$stdout, $stderr].each do |io|
+          File.open(Shoryuken::Later.options[:logfile], 'ab') do |f|
+            io.reopen(f)
+          end
+          io.sync = true
+        end
+        $stdin.reopen('/dev/null')
+
+        initialize_logger
+      end
+
+      def write_pid
+        if path = Shoryuken::Later.options[:pidfile]
+          File.open(path, 'w') do |f|
+            f.puts Process.pid
+          end
+        end
+      end
+
+      def parse_options(argv)
+        opts = {}
+
+        @parser = OptionParser.new do |o|
+          o.on '-d', '--daemon', 'Daemonize process' do |arg|
+            opts[:daemon] = arg
+          end
+
+          o.on '-t', '--table TABLE...', 'Table to process' do |arg|
+            Shoryuken::Later.tables << args
+          end
+
+          o.on '-r', '--require [PATH|DIR]', 'Location of the worker' do |arg|
+            opts[:require] = arg
+          end
+
+          o.on '-C', '--config PATH', 'Path to YAML config file' do |arg|
+            opts[:config_file] = arg
+          end
+
+          o.on '-R', '--rails', 'Load Rails' do |arg|
+            opts[:rails] = arg
+          end
+
+          o.on '-L', '--logfile PATH', 'Path to writable logfile' do |arg|
+            opts[:logfile] = arg
+          end
+
+          o.on '-P', '--pidfile PATH', 'Path to pidfile' do |arg|
+            opts[:pidfile] = arg
+          end
+
+          o.on '-v', '--verbose', 'Print more verbose output' do |arg|
+            opts[:verbose] = arg
+          end
+
+          o.on '-V', '--version', 'Print version and exit' do |arg|
+            puts "Shoryuken::Later #{Shoryuken::Later::VERSION}"
+            exit 0
+          end
+        end
+
+        @parser.banner = 'shoryuken-later [options]'
+        @parser.on_tail '-h', '--help', 'Show help' do
+          logger.info @parser
+          exit 1
+        end
+        @parser.parse!(argv)
+        opts
+      end
+
+      def handle_signal(sig)
+        logger.info "Got #{sig} signal"
+
+        case sig
+        when 'USR1'
+          logger.info "Received USR1, will soft shutdown down"
+
+          launcher.stop
+
+          exit 0
+        when 'TTIN'
+          Thread.list.each do |thread|
+            logger.info "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
+            if thread.backtrace
+              logger.info thread.backtrace.join("\n")
+            else
+              logger.info "<no backtrace available>"
+            end
+          end
+          
+          idle   = launcher.manager.instance_variable_get(:@idle).size
+          busy   = launcher.manager.instance_variable_get(:@busy).size
+          tables = launcher.manager.instance_variable_get(:@tables)
+
+          logger.info "Idle: #{idle}, Busy: #{busy}, Polled Tables: #{tables.join(', ')}"
+        else
+          logger.info "Received #{sig}, will shutdown down"
+
+          raise Interrupt
+        end
+      end
+
+      def setup_options(args)
+        options = parse_options(args)
+
+        # yield parsed options in case we need to do more setup before configuration is parsed
+        yield(options) if block_given?
+
+        config = options[:config_file] ? parse_config(options[:config_file]).deep_symbolize_keys : {}
+
+        Shoryuken::Later.options.merge!(config)
+
+        Shoryuken::Later.options.merge!(options)
+        
+        # Tables from command line options take precedence...
+        unless Shoryuken::Later.tables.any?
+          tables = Shoryuken::Later.options[:later][:tables]
+
+          # Use the default table if none were specified in the config file.
+          tables << Shoryuken::Later.default_table if tables.empty?
+          
+          Shoryuken::Later.tables.replace(tables)
+        end
+      end
+
+      def parse_config(config_file)
+        if File.exist?(config_file)
+          YAML.load(ERB.new(IO.read(config_file)).result)
+        else
+          raise ArgumentError, "Config file #{config_file} does not exist"
+        end
+      end
+
+      def initialize_logger
+        Shoryuken::Logging.initialize_logger(Shoryuken::Later.options[:logfile]) if Shoryuken::Later.options[:logfile]
+
+        Shoryuken::Later.logger.level = Logger::DEBUG if Shoryuken::Later.options[:verbose]
+      end
+
+      def validate!
+        raise ArgumentError, 'No tables given to poll' if Shoryuken::Later.tables.empty?
+
+        if Shoryuken::Later.options[:aws][:access_key_id].nil? && Shoryuken::Later.options[:aws][:secret_access_key].nil?
+          if ENV['AWS_ACCESS_KEY_ID'].nil? && ENV['AWS_SECRET_ACCESS_KEY'].nil?
+            raise ArgumentError, 'No AWS credentials supplied'
+          end
+        end
+
+        initialize_aws
+
+        Shoryuken::Later.tables.uniq.each do |table|
+          # validate all tables and AWS credentials consequently
+          begin
+            unless Shoryuken::Later::Client.tables(table).exists?
+              raise ArgumentError, "Table '#{table}' does not exist"
+            end
+          rescue => e
+            raise
+          end
+        end
+      end
+
+      def initialize_aws
+        # aws-sdk tries to load the credentials from the ENV variables: AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+        # when not explicit supplied
+        AWS.config Shoryuken::Later.options[:aws] if Shoryuken::Later.options[:aws]
+      end
+
+      def require_workers
+        require Shoryuken::Later.options[:require] if Shoryuken::Later.options[:require]
+      end
+    end
+  end
+end

data/lib/shoryuken/later/client.rb

@@ -0,0 +1,26 @@
+require 'securerandom'
+
+module Shoryuken
+  module Later
+    class Client
+      @@tables = {}
+        
+      class << self
+        def tables(table)
+          @@tables[table.to_s] ||= ddb.tables[table].tap do |tbl|
+            tbl.hash_key = [:id, :string]
+          end
+        end
+        
+        def put_item(table, attributes, options={unless_exists: 'id'})
+          attributes['id'] ||= SecureRandom.uuid
+          tables(table).items.create(attributes, options)
+        end
+        
+        def ddb
+          @ddb ||= AWS::DynamoDB.new
+        end
+      end
+    end
+  end
+end