delayed-threaded 0.16.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 32e63adc06acd90d954baed1e872c113239941d9
+  data.tar.gz: bb4d43d6376e85f84efe0945c14149143ca59140
+SHA512:
+  metadata.gz: 46418908010e3463ca50d1c6a17d069670869b40cb220f2c3729b6924b89b99006ddf07a1e0259f7b2429b5d01e690be9bf8bd9692494c6c39cb3848f9a92403
+  data.tar.gz: f50279911f6e6024570079a3f6715a72136d9853e53949ebad1f62591422656881b0eee64fd1db29ab505d2bdfb0d68d46c55f77e9393b3ddeac4bfc9a769db2

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/tmp/
+/test/db/*.db
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,21 @@
+sudo: false
+language: ruby
+jdk:
+  - oraclejdk8
+rvm:
+  - jruby-1.7.26
+  - jruby-9.1.16.0
+before_install: gem install bundler -v 1.16.1
+env:
+  - delayed_job="~>4.1.5" activerecord="~>4.2"
+  - delayed_job="~>4.0.6" activerecord="~>4.2"
+  - delayed_job="~>3.0" # activerecord="~>3.2"
+  - delayed_job="~>4.1" activerecord="~>4.2.6"
+matrix:
+  allow_failures:
+    #- rvm: jruby-head
+  include:
+    - rvm: jruby-9.1.17.0
+      env: delayed_job="~>4.1.5" activerecord="~>5.0.5"
+    - rvm: jruby-9.1.17.0
+      env: delayed_job="~>4.1.5" activerecord="~>5.1.6"

data/Gemfile

@@ -0,0 +1,36 @@
+source "https://rubygems.org"
+
+gemspec
+
+if ENV['delayed_job']
+  if ENV['delayed_job'] == 'master'
+    gem 'delayed_job', :git => 'git://github.com/collectiveidea/delayed_job.git'
+    gem 'delayed_job_active_record', :require => nil
+  else
+    gem 'delayed_job', version = ENV['delayed_job']
+    unless version =~ /~?\s?2\.\d/ # delayed_job_active_record only for DJ >= 3.0
+      gem 'delayed_job_active_record', :require => nil
+    end
+    if version =~ /~?\s?4\.[1]/ # add_dependency "delayed_job", ">= 4.1"
+      gem 'delayed_cron_job', :require => nil
+    end
+  end
+else
+  gem 'delayed_job'
+  gem 'delayed_job_active_record', :require => nil
+  gem 'delayed_cron_job', :require => nil
+end
+
+if ENV['activerecord']
+  gem 'activerecord', version = ENV['activerecord'], :require => nil
+  if version =~ /~?\s?4\.[12]/
+    gem 'activerecord-jdbc-adapter', '~> 1.3.20', :require => nil, :platform => :jruby
+  else
+    gem 'activerecord-jdbc-adapter', :require => nil, :platform => :jruby
+  end
+else
+  gem 'activerecord', :require => nil # for tests
+  gem 'activerecord-jdbc-adapter', :require => nil, :platform => :jruby
+end
+
+gem 'jdbc-sqlite3', '~> 3.20.1', :platform => :jruby

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2018 Karol Bucek
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOa AND
+NONINFRINGEMENT. IN NO EVENT SaALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+# Delayed::Threaded
+
+Allows to start [DJ][0] in the same process using `Thread.new { ... }`
+
+Extracted from [JRuby-Rack-Worker][1]: an effort to run Ruby workers in threads
+with Java (JRuby-Rack) deployments (previously known as `Delayed::JRubyWorker`).
+
+**NOTE: JRuby only, for now (PRs welcome)!**
+
+## Installation
+
+```ruby
+gem 'delayed-threaded'
+```
+
+and `bundle` or install it yourself as `gem install delayed-threaded`.
+
+## Usage
+
+```ruby
+def start_worker
+  options = { :quiet => true }
+  options[:queues] = (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
+  options[:min_priority] = ENV['MIN_PRIORITY']
+  options[:max_priority] = ENV['MAX_PRIORITY']
+  # beyond `rake delayed:work` compatibility :
+  if read_ahead = ENV['READ_AHEAD'] # DEFAULT_READ_AHEAD = 5
+    options[:read_ahead] = read_ahead.to_i
+  end
+  if sleep_delay = ENV['SLEEP_DELAY'] # DEFAULT_SLEEP_DELAY = 5
+    options[:sleep_delay] = sleep_delay.to_f
+  end
+
+  worker = Delayed::Threaded::Worker.new(options)
+  worker.start
+rescue Exception => e
+  msg = "FATAL #{e.inspect}"
+  if backtrace = e.backtrace
+    msg << ":\n  #{backtrace.join("\n  ")}"
+  end
+  STDERR.puts(msg)
+end
+
+Thread.new { start_worker }
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies.
+Then, run `rake test` to run the tests. You can also run `bin/console` for an
+interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+## Copyright
+
+Copyright (c) 2018 [Karol Bucek](http://kares.org).
+See LICENSE (http://en.wikipedia.org/wiki/MIT_License) for details.
+
+[0]: https://github.com/collectiveidea/delayed_job
+[1]: https://github.com/kares/jruby-rack-worker

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+task :default => :test

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "delayed/threaded"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/delayed-threaded.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+
+Gem::Specification.new do |spec|
+  spec.name          = "delayed-threaded"
+  spec.authors       = ["Karol Bucek"]
+  spec.email         = ["self@kares.org"]
+  spec.licenses      = ['MIT']
+
+  path = File.expand_path('lib/delayed/threaded/version.rb', File.dirname(__FILE__))
+  spec.version = File.read(path).match( /.*VERSION\s*=\s*['"](.*)['"]/m )[1]
+
+  spec.summary       = %q{Making Delayed::Job a polite (thread-safe) in process citizen.}
+  spec.description   = "Allows to start DJ in the same process using Thread.new { ... } "
+  spec.homepage      = "https://github.com/kares/delayed-threaded"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|temp)/})
+  end
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency "delayed_job", ">= 3.0", "< 4.2"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit", "~> 2.5.3"
+  spec.add_development_dependency "test-unit-context"
+  spec.add_development_dependency "mocha"
+end

data/lib/delayed/threaded/compat.rb

@@ -0,0 +1,4 @@
+module Delayed
+  # @deprecated
+  JRubyWorker = Threaded::Worker
+end

data/lib/delayed/threaded/sleep_calculator.rb

@@ -0,0 +1,33 @@
+module Delayed
+  module Threaded
+    module SleepCalculator
+
+      def sleep(time)
+        Kernel.sleep calc_sleep_time(time)
+      end
+
+      private
+
+      @@last = java.util.concurrent.atomic.AtomicLong.new
+
+      def calc_sleep_time(time)
+        count = thread_count rescue nil
+        return time if ! count || count <= 1 || time <= 0
+
+        last = @@last.get_and_set now = java.lang.System.current_time_millis
+        return time if ( now - last ) > time * 1000
+
+        # time / count.to_f - optimal pause time between threads
+        diff = ( time / count.to_f ) - ( now - last ) * 0.001
+        # converge to ~ pauses between worker threads (might add up to 10% to sleep time)
+        diff > 0 ? ( time + diff / 5.0 ) : time
+        # (now - last) * 0.001 < opt_pause ? ( time + time / 11.1 ) : time
+      end
+
+      def thread_count
+        $worker_manager ? $worker_manager.thread_count : false
+      end
+
+    end
+  end
+end

data/lib/delayed/threaded/sync_lifecycle.rb

@@ -0,0 +1,34 @@
+require 'thread'
+
+module Delayed
+  module Threaded
+    module SyncLifecycle
+
+    def self.extended(base)
+      klass = base.singleton_class
+      klass.send :alias_method, :setup_lifecycle_impl, :setup_lifecycle
+      klass.send :alias_method, :setup_lifecycle, :setup_lifecycle_sync
+      klass.send :private, :setup_lifecycle_impl
+    end
+
+    def setup_lifecycle_sync
+      # not really necessary, but a plugin might expect setup to run exactly once
+      (@lifecycle ||= nil) || synchronize do
+        unless @lifecycle
+          setup_lifecycle_impl
+          #@lifecycle = Delayed::Lifecycle.new
+          #lifecycle = @lifecycle
+        end
+        @lifecycle # make sure returns lifecycle (for DJ < 4.1 compat layer)
+      end
+    end
+
+    MUTEX = Mutex.new
+    private_constant :MUTEX if respond_to?(:private_constant)
+
+    def synchronize(&block); MUTEX.synchronize(&block) end
+    private :synchronize
+
+    end
+  end
+end

data/lib/delayed/threaded/version.rb

@@ -0,0 +1,5 @@
+module Delayed
+  module Threaded
+    VERSION = '0.16.0'
+  end
+end

data/lib/delayed/threaded/worker.rb

@@ -0,0 +1,172 @@
+require 'java'
+require 'delayed_job' unless defined?(Delayed::Worker)
+require 'socket' # due Socket.gethostname in Delayed::Worker#name
+
+require 'delayed/threaded/sleep_calculator'
+require 'delayed/threaded/sync_lifecycle'
+
+module Delayed::Threaded
+  # Threaded DJ worker implementation.
+  # - inspired by Delayed::Command
+  # - no daemons dependency + thread-safe
+  class Worker < Delayed::Worker
+
+    include SleepCalculator
+
+    # @override to return the same as Delayed::Worker.lifecycle (uses class instance state)
+    def self.lifecycle; Delayed::Worker.lifecycle end
+
+    # @override since `initialize` (DJ 4.1) does: `self.class.setup_lifecycle`
+    def self.setup_lifecycle; Delayed::Worker.setup_lifecycle end
+
+    unless defined? Delayed::Worker.setup_lifecycle
+      # adapt DJ 3.0/4.0 :
+      # def self.lifecycle
+      #   @lifecycle ||= Delayed::Lifecycle.new
+      # end
+      def superclass.lifecycle
+        @lifecycle ||= setup_lifecycle
+      end
+      def superclass.setup_lifecycle
+        @lifecycle = Delayed::Lifecycle.new
+      end
+    end
+
+    # @patch make sure concurrent worker threads do not cause multiple initializations
+    Delayed::Worker.extend SyncLifecycle if Delayed.const_defined? :Lifecycle
+
+    THREAD_LOCAL_ACCESSORS = [
+      :min_priority, :max_priority, :sleep_delay, :read_ahead, :queues, :exit_on_complete
+    ]
+    private_constant :THREAD_LOCAL_ACCESSORS if respond_to?(:private_constant)
+    # due Delayed::Worker#initialize(options = {}) :
+    #
+    # [:min_priority, :max_priority, :sleep_delay, :read_ahead, :queues, :exit_on_complete].each do |option|
+    #   self.class.send("#{option}=", options[option]) if options.key?(option)
+    # end
+
+    class Config
+      attr_accessor *THREAD_LOCAL_ACCESSORS
+      def key?(name); ! instance_variable_get(:"@#{name}").nil? end
+    end
+
+    THREAD_LOCAL_ACCESSORS.each do |name|
+      class_eval(<<-EOS, __FILE__, __LINE__ + 1)
+        def self.#{name}=(val)
+          (Thread.current[:delayed_threaded_worker_config] ||= Config.new).#{name} = val
+        end
+        def self.#{name}
+          if (config = Thread.current[:delayed_threaded_worker_config]) && config.key?(:#{name})
+            config.#{name}
+          else
+            superclass.#{name}
+          end
+        end
+      EOS
+    end
+    # e.g. :
+    #
+    #  def self.min_priority=(value)
+    #    (Thread.current[:delayed_jruby_worker_config] ||= Config.new).min_priority = value
+    #  end
+    #
+    #  def self.min_priority
+    #    if (config = Thread.current[:delayed_jruby_worker_config]) && config.key?(:min_priority)
+    #      config.min_priority
+    #    else
+    #      Worker.min_priority
+    #    end
+    #  end
+
+    def name
+      if (@name ||= nil).nil?
+        # super - [prefix]host:hostname pid:process_pid
+        begin
+          @name = "#{super} thread:#{thread_id}".freeze
+        rescue
+          @name = "#{@name_prefix}thread:#{thread_id}".freeze
+        end
+      end
+      @name
+    end
+
+    def to_s; name; end
+
+    def thread_id
+      # NOTE: JRuby might set a bit long name for Thread.new { ... } code e.g.
+      # RubyThread-1: /home/[...]/src/test/ruby/delayed/jruby_worker_test.rb:163
+      if name = java.lang.Thread.currentThread.getName
+        if name.size > 100 && match = name.match(/(.*?)\:\s.*?[\/\\]+/)
+          match[1]
+        else
+          name
+        end
+      end
+    end
+
+    def exit!
+      return if @exit # #stop?
+      say "Stoping job worker"
+      @exit = true # #stop
+      if Delayed.const_defined?(:Job) && Delayed::Job.respond_to?(:clear_locks!)
+        Delayed::Job.clear_locks!(name)
+      end
+    end
+
+    unless defined? Delayed::Lifecycle # DJ 2.x (< 3.0)
+      require 'benchmark'
+      # in case DJ 2.1 loads AS 3.x we're need `[1,2].sum` :
+      require 'active_support/core_ext/enumerable' rescue nil
+
+      def start
+        say "Starting job worker"
+        trap
+
+        loop do
+          result = nil
+
+          realtime = Benchmark.realtime do
+            result = work_off
+          end
+
+          count = result.sum
+
+          break if @exit
+
+          if count.zero?
+            sleep(self.class.sleep_delay)
+          else
+            say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / realtime, result.last]
+          end
+
+          break if @exit
+        end
+      end
+
+      def stop?; !!@exit; end
+      def stop; @exit = true; end
+
+    end
+
+    protected
+
+    def trap(name = nil)
+      # catch invocations from #start traps TERM and INT
+      at_exit { exit! } if ! name || name.to_s == 'TERM'
+    end
+
+  end
+
+end
+
+Dir.chdir( Rails.root ) if defined?(Rails.root) && Dir.getwd.to_s != Rails.root.to_s
+
+if ! Delayed::Worker.backend && ! Delayed.const_defined?(:Lifecycle)
+  Delayed::Worker.guess_backend # deprecated on DJ since 3.0
+end
+
+# NOTE: no explicit logger configuration - DJ logger defaults to Rails.logger
+# if this is not desired - e.g. one wants script/delayed_job's logger behavior
+# it's more correct to configure in an initializer rather then forcing the use
+# of delayed_job.log (like Delayed::Command does) ...
+# Delayed::Worker.logger = Logger.new(File.join(Rails.root, 'log', 'delayed_job.log'))

data/lib/delayed/threaded.rb

@@ -0,0 +1,5 @@
+require 'delayed/threaded/version'
+
+require 'delayed/threaded/worker'
+
+require 'delayed/threaded/compat'

metadata

@@ -0,0 +1,149 @@
+--- !ruby/object:Gem::Specification
+name: delayed-threaded
+version: !ruby/object:Gem::Version
+  version: 0.16.0
+platform: ruby
+authors:
+- Karol Bucek
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2018-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+  name: delayed_job
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.2'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.5.3
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-context
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: mocha
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Allows to start DJ in the same process using Thread.new { ... } '
+email:
+- self@kares.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- delayed-threaded.gemspec
+- lib/delayed/threaded.rb
+- lib/delayed/threaded/compat.rb
+- lib/delayed/threaded/sleep_calculator.rb
+- lib/delayed/threaded/sync_lifecycle.rb
+- lib/delayed/threaded/version.rb
+- lib/delayed/threaded/worker.rb
+homepage: https://github.com/kares/delayed-threaded
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.14.1
+signing_key:
+specification_version: 4
+summary: Making Delayed::Job a polite (thread-safe) in process citizen.
+test_files: []