trinidad_sidekiq_extension 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in trinidad_sidekiq_extension.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Adrian Madrid
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,51 @@
+# Trinidad Sidekiq Extension
+
+Run Sidekiq workers within the Trinidad process.
+
+Inspiration: https://github.com/carlhoerberg/trinidad_threaded_resque_extension
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'trinidad_sidekiq_extension', require: false
+
+And then execute:
+
+    $ bundle
+
+Then configure it via config/trinidad.rb:
+
+```ruby
+require 'bundler/setup'
+Trinidad.configure do |config|
+  config.jruby_min_runtimes = 1
+  config.jruby_max_runtimes = 1
+  ...
+  config.web_apps = {
+    default = {
+      rackup: 'config.ru'
+      extensions = {
+        sidekiq: {
+          arguments: '-c 10 -r ./config/sidekiq'
+        }
+      }
+    }
+  }
+end
+```
+
+## Usage
+
+Start trinidad and enjoy the low memory usage while having high concurrency.
+
+Note that becasue JRuby can't fork you want to have well behaved workers, watch out for memory leakage. Naturally your workers have to be thread safe.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/lib/trinidad_sidekiq_extension.rb

@@ -0,0 +1,15 @@
+require 'trinidad'
+require_relative "trinidad_sidekiq_extension/version"
+require_relative "trinidad_sidekiq_extension/background_manager"
+require_relative "trinidad_sidekiq_extension/lifecycle_listener"
+
+module Trinidad
+  module Extensions
+    class SidekiqWebAppExtension < WebAppExtension
+      def configure(app_context)
+        app_context.add_lifecycle_listener Sidekiq::LifecycleListener.new(@options)
+      end
+    end
+  end
+end
+

data/lib/trinidad_sidekiq_extension/background_manager.rb

@@ -0,0 +1,152 @@
+$stdout.sync = true
+
+require 'yaml'
+require 'singleton'
+require 'optparse'
+require 'celluloid'
+require 'erb'
+
+require 'sidekiq'
+require 'sidekiq/util'
+require 'sidekiq/manager'
+require 'sidekiq/scheduled'
+
+module Sidekiq
+  class BackgroundManager
+    include Util
+    include Singleton
+
+    # Used for BM testing
+    attr_accessor :code
+    attr_accessor :manager
+    attr_reader :interrupted
+
+    def initialize
+      @code            = nil
+      @interrupt_mutex = Mutex.new
+      @interrupted     = false
+    end
+
+    def configure(given_options = { })
+      @code = nil
+      Sidekiq.logger
+
+      logger.debug "BM configure : options : #{options.inspect}" if options[:verbose]
+      logger.debug "BM configure : given_options : #{given_options.inspect}" if options[:verbose]
+      config_options = parse_config given_options
+      logger.debug "BM configure : config_options : #{config_options.inspect}" if options[:verbose]
+      options.merge! config_options
+      logger.debug "BM configure : options : #{options.inspect}" if options[:verbose]
+
+      Sidekiq.logger.level = Logger::DEBUG if options[:verbose]
+      Celluloid.logger = nil unless options[:verbose]
+
+      validate!
+      boot_system
+    end
+
+    def run
+      logger.debug "Booting Sidekiq BM #{Sidekiq::VERSION} with Redis at #{redis { |x| x.client.id }}"
+      logger.debug "Running in #{RUBY_DESCRIPTION}"
+      logger.debug "Dir.pwd : #{Dir.pwd}"
+      logger.debug Sidekiq::LICENSE
+
+      @manager = Sidekiq::Manager.new(options)
+      poller   = Sidekiq::Scheduled::Poller.new
+      begin
+        logger.debug 'BM Starting processing ...'
+        @manager.start!
+        poller.poll!(true)
+        sleep
+      rescue Interrupt
+        logger.debug 'BM Shutting down ...'
+        poller.terminate! if poller.alive?
+        @manager.stop!(:shutdown => true, :timeout => options[:timeout])
+        @manager.wait(:shutdown)
+        # WILL NOT explicitly exit because Trinidad will take care of exiting
+        # in its own sweet time.
+        # exit(0)
+      end
+    end
+
+    def interrupt
+      @interrupt_mutex.synchronize do
+        unless @interrupted
+          logger.debug "BM interrupting ..."
+          @interrupted = true
+          Thread.main.raise Interrupt
+        end
+      end
+    end
+
+    private
+
+    def die(code)
+      exit(code)
+    end
+
+    def options
+      Sidekiq.options
+    end
+
+    def detected_environment
+      options[:environment] ||= ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    def boot_system
+      ENV['RACK_ENV'] = ENV['RAILS_ENV'] = detected_environment
+
+      raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
+
+      logger.debug "BM Booting system with require [#{options[:require]}] ..."
+      if File.directory?(options[:require])
+        logger.debug "BM require [#{options[:require]}] is a directory, loading Rails ..."
+        begin
+          require 'rails'
+          require 'sidekiq/rails'
+          logger.debug "BM requiring #{options[:require]} ..."
+          require File.expand_path("#{options[:require]}/config/environment.rb")
+          ::Rails.application.eager_load!
+        rescue LoadError => e
+          logger.debug "EXCEPTION: #{e.class.name} : #{e.message}"
+          logger.debug "We could not load rails, you will run into trouble!"
+          raise "BM Could not load rails for require [#{options[:require]}], exiting ..."
+        end
+      else
+        logger.debug "BM require [#{options[:require]}] is NOT a directory, requiring file ..."
+        require options[:require]
+      end
+    end
+
+    def validate!
+      options[:queues] << 'default' if options[:queues].empty?
+
+      if !File.exist?(options[:require]) ||
+        (File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
+        logger.info "=================================================================="
+        logger.info "  Please point sidekiq to a Rails 3 application or a Ruby file    "
+        logger.info "  to load your worker classes with options[:require]= [DIR|FILE]. "
+        logger.info "=================================================================="
+        logger.info "options : #{options.inspect}"
+        die(1)
+      end
+    end
+
+    def parse_config(given_options)
+      if given_options[:config_file] && File.exist?(given_options[:config_file])
+        opts   = YAML.load(ERB.new(IO.read(given_options[:config_file])).result)
+        queues = opts.delete(:queues) || []
+        queues.each { |name, weight| parse_queues(opts, name, weight) }
+        given_options.update opts
+      end
+      given_options
+    end
+
+    def parse_queues(opts, q, weight)
+      [weight.to_i, 1].max.times do
+        (opts[:queues] ||= []) << q
+      end
+    end
+
+  end
+end

data/lib/trinidad_sidekiq_extension/lifecycle_listener.rb

@@ -0,0 +1,49 @@
+module Trinidad
+  module Extensions
+    module Sidekiq
+      class LifecycleListener
+        include Trinidad::Tomcat::LifecycleListener
+        attr_accessor :options, :workers, :threads
+
+        def initialize(options = { })
+          @options = options || {}
+        end
+
+        def lifecycleEvent(event)
+          case event.type
+            when Trinidad::Tomcat::Lifecycle::AFTER_START_EVENT
+              STDOUT << "[Trinidad:Sidekiq] event [#{event.type}] << start >>\n" if options[:verbose]
+              start_cli
+            when Trinidad::Tomcat::Lifecycle::BEFORE_STOP_EVENT
+              STDOUT << "[Trinidad:Sidekiq] event [#{event.type}] << stop >>\n" if options[:verbose]
+              stop_cli
+            else
+              STDOUT << "[Trinidad:Sidekiq] event [#{event.type}] skipped\n" if options[:verbose]
+          end
+        end
+
+        def start_cli
+          STDOUT << "[Trinidad:Sidekiq] starting sidekiq bm with options [#{@options}]\n" if options[:verbose]
+          unless options[:require]
+            raise "You probably want to send a require option to the sidekiq background manager ..." if options[:verbose]
+          end
+
+          STDOUT << "[Trinidad:Sidekiq] getting it going ...\n"
+          bm = ::Sidekiq::BackgroundManager.instance
+          STDOUT << "[Trinidad:Sidekiq] got original bm (#{bm.class.name}) #{bm.inspect} ...\n" if options[:verbose]
+          bm.configure @options
+          STDOUT << "[Trinidad:Sidekiq] got modified bm (#{bm.class.name}) #{bm.inspect} ...\n" if options[:verbose]
+          res = bm.run
+          STDOUT << "[Trinidad:Sidekiq] got bm running (#{res.class.name}) #{res.inspect} ...\n" if options[:verbose]
+        end
+
+        def stop_cli
+          STDOUT << "[Trinidad:Sidekiq] Stopping sidekiq cli ...\n" if options[:verbose]
+          res = ::Sidekiq::BackgroundManager.instance.interrupt
+          STDOUT << "[Trinidad:Sidekiq] got bm stopped (#{res.class.name})#{res.inspect} ...\n" if options[:verbose]
+        end
+
+      end
+    end
+  end
+end

data/lib/trinidad_sidekiq_extension/version.rb

@@ -0,0 +1,3 @@
+module TrinidadThreadedSidekiqExtension
+  VERSION = "0.0.1"
+end

data/trinidad_sidekiq_extension.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/trinidad_sidekiq_extension/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Adrian Madrid"]
+  gem.email         = ["aemadrid@gmail.com"]
+  gem.description   = %q{Runs Sidekiq workers within the Trinidad application server}
+  gem.summary       = %q{Runs Sidekiq workers within the Trinidad application server}
+  gem.homepage      = "https://github.com/aemadrid/trinidad_sidekiq_extension"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "trinidad_sidekiq_extension"
+  gem.require_paths = ["lib"]
+  gem.version       = TrinidadThreadedSidekiqExtension::VERSION
+
+  gem.add_development_dependency "rake"
+  gem.add_runtime_dependency "trinidad"
+  gem.add_runtime_dependency "sidekiq"
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: trinidad_sidekiq_extension
+version: !ruby/object:Gem::Version
+  prerelease:
+  version: 0.0.1
+platform: ruby
+authors:
+- Adrian Madrid
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2012-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: trinidad
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: !binary |-
+          MA==
+    none: false
+  prerelease: false
+  type: :runtime
+description: Runs Sidekiq workers within the Trinidad application server
+email:
+- aemadrid@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/trinidad_sidekiq_extension.rb
+- lib/trinidad_sidekiq_extension/background_manager.rb
+- lib/trinidad_sidekiq_extension/lifecycle_listener.rb
+- lib/trinidad_sidekiq_extension/version.rb
+- trinidad_sidekiq_extension.gemspec
+homepage: https://github.com/aemadrid/trinidad_sidekiq_extension
+licenses: []
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: !binary |-
+        MA==
+  none: false
+requirements: []
+rubyforge_project:
+rubygems_version: 1.8.24
+signing_key:
+specification_version: 3
+summary: Runs Sidekiq workers within the Trinidad application server
+test_files: []