fleiss 0.1.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45bf4b9f2a7a9d88cbdd4feb8fb2992d6545554695c791a1f327a5a26da3e53d
-  data.tar.gz: 4434eac634c36952365be272f5f8327462577db2cc338c97b1f242f2f59d14bf
+  metadata.gz: 2d7f27ee73ed5c0434f79ec98f0c249f4f41ece528807cabc3efd29f8ebf79c0
+  data.tar.gz: e597168cfb8931240110ad45b4ee49937128bdb4dce826d142954253ce29dc9d
 SHA512:
-  metadata.gz: d008669212582e7e5190ee0901024555c5031d0295b934d08dd91508d0b83251ea7790540c0e4d8339cae15b6aa9c6c043d5fc39eaa18bc5a696a30e23bc4a0f
-  data.tar.gz: fc3f5555a6290abebd712e545172fea30eb4be0074da2eb86bc1cb136216af563054111d45599f2e9ddb3560a94c00256e7a8d00b3f7b69e849d39264911131d
+  metadata.gz: 737815c833f1ff98e4c18c54709b7671ecd26cabb8500e5f93375a4ac528b4a8344381fe906db2e27726e2217de036a406aa83263514e2475798e5a15831f037
+  data.tar.gz: dae7da18743673e38465150031a48734e7911d7894c0131a8d9469d472650f30ea53a73d6217bcc6c4f237c93fa8dfd147ad9df4ac3542e1f015e75dbbda4c4c

data/.gitignore

@@ -1,2 +1,3 @@
 .rubocop-*
 pkg/
+spec/tmp

data/.rubocop.yml

@@ -1,5 +1,8 @@
 inherit_from:
-  - https://bitbucket.org/bsm/misc/raw/master/rubocop/default.yml
+  - https://gitlab.com/bsm/misc/raw/master/rubocop/default.yml
+
+AllCops:
+  TargetRubyVersion: "2.6"
 
 Security/Open:
   Exclude:

data/.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 rvm:
-  - 2.5
-  - 2.4
+  - 2.7
+  - 2.6
 before_install:
+  - gem install bundler
   - mysql -e 'CREATE DATABASE fleiss_test;'
   - psql -c 'CREATE DATABASE fleiss_test;' -U postgres
 services:

data/Gemfile.lock

@@ -1,73 +1,74 @@
 PATH
   remote: .
   specs:
-    fleiss (0.1.3)
-      activejob (>= 5.0)
-      activerecord (>= 5.0)
+    fleiss (0.4.0)
+      activejob (>= 6.0)
+      activerecord (>= 6.0)
       concurrent-ruby
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (5.2.2)
-      activesupport (= 5.2.2)
+    activejob (6.0.3.1)
+      activesupport (= 6.0.3.1)
       globalid (>= 0.3.6)
-    activemodel (5.2.2)
-      activesupport (= 5.2.2)
-    activerecord (5.2.2)
-      activemodel (= 5.2.2)
-      activesupport (= 5.2.2)
-      arel (>= 9.0)
-    activesupport (5.2.2)
+    activemodel (6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activerecord (6.0.3.1)
+      activemodel (= 6.0.3.1)
+      activesupport (= 6.0.3.1)
+    activesupport (6.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    arel (9.0.0)
+      zeitwerk (~> 2.2, >= 2.2.2)
     ast (2.4.0)
-    concurrent-ruby (1.1.3)
+    concurrent-ruby (1.1.6)
     diff-lcs (1.3)
-    globalid (0.4.1)
+    globalid (0.4.2)
       activesupport (>= 4.2.0)
-    i18n (1.2.0)
+    i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.1)
-    minitest (5.11.3)
-    mysql2 (0.5.2)
-    parallel (1.12.1)
-    parser (2.5.3.0)
+    minitest (5.14.1)
+    mysql2 (0.5.3)
+    parallel (1.19.1)
+    parser (2.7.1.3)
       ast (~> 2.4.0)
-    pg (1.1.3)
-    powerpack (0.1.2)
+    pg (1.2.3)
     rainbow (3.0.0)
-    rake (12.3.2)
-    rspec (3.8.0)
-      rspec-core (~> 3.8.0)
-      rspec-expectations (~> 3.8.0)
-      rspec-mocks (~> 3.8.0)
-    rspec-core (3.8.0)
-      rspec-support (~> 3.8.0)
-    rspec-expectations (3.8.2)
+    rake (13.0.1)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-mocks (3.8.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.8.0)
-    rspec-support (3.8.0)
-    rubocop (0.61.1)
-      jaro_winkler (~> 1.5.1)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.84.0)
       parallel (~> 1.10)
-      parser (>= 2.5, != 2.5.1.1)
-      powerpack (~> 0.1)
+      parser (>= 2.7.0.1)
       rainbow (>= 2.2.2, < 4.0)
+      rexml
+      rubocop-ast (>= 0.0.3)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.4.0)
-    ruby-progressbar (1.10.0)
-    sqlite3 (1.3.13)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (0.0.3)
+      parser (>= 2.7.0.1)
+    ruby-progressbar (1.10.1)
+    sqlite3 (1.4.2)
     thread_safe (0.3.6)
-    tzinfo (1.2.5)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    unicode-display_width (1.4.0)
+    unicode-display_width (1.7.0)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
@@ -83,4 +84,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   1.16.4
+   2.1.4

data/bin/fleiss

@@ -5,17 +5,20 @@ $LOAD_PATH.push(lib_dir) unless $LOAD_PATH.include?(lib_dir)
 
 require 'fleiss/cli'
 
+STDOUT.sync = true
+STDERR.sync = true
+
 cli = Fleiss::CLI.instance
 begin
   cli.parse!
   cli.run!
 rescue ArgumentError => e
-  STDERR.puts " ! #{e.message}\n"
+  warn " ! #{e.message}\n"
   STDERR.puts
-  STDERR.puts cli.parser
+  warn cli.parser
   exit 1
 rescue StandardError => e
-  STDERR.puts e.message
-  STDERR.puts e.backtrace.join("\n")
+  warn e.message
+  warn e.backtrace.join("\n")
   exit 1
 end

data/fleiss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'fleiss'
-  s.version       = '0.1.3'
+  s.version       = '0.4.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Minimialist background jobs backed by ActiveJob and ActiveRecord.)
@@ -12,10 +12,10 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
   s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.2'
+  s.required_ruby_version = '>= 2.6'
 
-  s.add_dependency 'activejob', '>= 5.0'
-  s.add_dependency 'activerecord', '>= 5.0'
+  s.add_dependency 'activejob', '>= 6.0'
+  s.add_dependency 'activerecord', '>= 6.0'
   s.add_dependency 'concurrent-ruby'
 
   s.add_development_dependency 'bundler'

data/lib/fleiss.rb

@@ -1,9 +1,15 @@
+require 'active_job'
+require 'active_job/queue_adapters/fleiss_adapter'
+require 'fleiss/backend'
+
 module Fleiss
   DEFAULT_QUEUE = 'default'.freeze
 
   def self.backend
-    self::Backend::ActiveRecord
+    @backend ||= self::Backend::ActiveRecord
   end
-end
 
-require 'fleiss/backend'
+  def self.backend=(value)
+    @backend = value
+  end
+end

data/lib/fleiss/backend/active_record/concern.rb

@@ -8,15 +8,19 @@ module Fleiss
           scope :in_queue, ->(qs) { where(queue_name: Array.wrap(qs)) }
           scope :finished, -> { where.not(finished_at: nil) }
           scope :not_finished, -> { where(finished_at: nil) }
-          scope :not_expired,  ->(now=Time.zone.now) { where(arel_table[:expires_at].eq(nil).or(arel_table[:expires_at].gt(now))) }
+          scope :not_expired,  ->(now = Time.zone.now) { where(arel_table[:expires_at].eq(nil).or(arel_table[:expires_at].gt(now))) }
           scope :started,      -> { where(arel_table[:started_at].not_eq(nil)) }
           scope :not_started,  -> { where(arel_table[:started_at].eq(nil)) }
-          scope :scheduled,    ->(now=Time.zone.now) { where(arel_table[:scheduled_at].gt(now)) }
+          scope :scheduled,    ->(now = Time.zone.now) { where(arel_table[:scheduled_at].gt(now)) }
         end
 
         module ClassMethods
+          def wrap_perform(&block)
+            connection_pool.with_connection(&block)
+          end
+
           # @return [ActiveRecord::Relation] pending scope
-          def pending(now=Time.zone.now)
+          def pending(now = Time.zone.now)
             not_finished
               .not_expired(now)
               .not_started

data/lib/fleiss/cli.rb

@@ -2,7 +2,6 @@ require 'singleton'
 require 'optparse'
 require 'yaml'
 require 'erb'
-require 'logger'
 
 module Fleiss
   class CLI
@@ -15,7 +14,6 @@ module Fleiss
       require: [],
       concurrency: 10,
       wait_time: 1.0,
-      logfile: STDOUT,
     }.freeze
 
     attr_reader :opts
@@ -24,7 +22,7 @@ module Fleiss
       @opts = DEFAULT_OPTIONS.dup
     end
 
-    def parse!(argv=ARGV)
+    def parse!(argv = ARGV)
       parser.parse!(argv)
 
       # Check config file
@@ -46,14 +44,11 @@ module Fleiss
       opts[:require].each {|n| require n }
       require 'fleiss/worker'
 
-      logdev = opts[:logfile]
-      logdev.sync = true if logdev.respond_to?(:sync=)
-
+      ActiveJob::Base.logger = ActiveSupport::TaggedLogging.new(ActiveSupport::Logger.new(opts[:logfile])) if opts[:logfile]
       Fleiss::Worker.run \
         queues: opts[:queues],
         concurrency: opts[:concurrency],
-        wait_time: opts[:wait_time],
-        logger: Logger.new(logdev)
+        wait_time: opts[:wait_time]
     end
 
     def parser # rubocop:disable Metrics/MethodLength

data/lib/fleiss/executor.rb

@@ -0,0 +1,22 @@
+require 'fleiss'
+require 'concurrent/executor/simple_executor_service'
+
+class Fleiss::Executor < Concurrent::SimpleExecutorService
+  attr_reader :max_size
+
+  def post(&block)
+    super unless capacity.zero?
+  end
+
+  def capacity
+    val = @max_size - @count.value
+    val.positive? ? val : 0
+  end
+
+  private
+
+  def ns_initialize(opts = {})
+    super()
+    @max_size = opts.fetch(:max_size, 1).to_i
+  end
+end

data/lib/fleiss/worker.rb

@@ -1,11 +1,9 @@
 require 'fleiss'
-require 'concurrent/executor/fixed_thread_pool'
-require 'concurrent/atomic/atomic_fixnum'
-require 'logger'
+require 'fleiss/executor'
 require 'securerandom'
 
 class Fleiss::Worker
-  attr_reader :queues, :uuid, :wait_time, :logger
+  attr_reader :queues, :uuid, :wait_time
 
   # Shortcut for new(*args).run
   def self.run(*args)
@@ -18,24 +16,22 @@ class Fleiss::Worker
   # @option [Array<String>] :queues queues to watch. Default: ["default"]
   # @option [Integer] :concurrency the number of concurrent pool. Default: 10
   # @option [Numeric] :wait_time maximum time (in seconds) to wait for jobs when retrieving next batch. Default: 1s.
-  # @option [Logger] :logger optional logger.
-  def initialize(queues: [Fleiss::DEFAULT_QUEUE], concurrency: 10, wait_time: 1, logger: nil)
+  def initialize(queues: [Fleiss::DEFAULT_QUEUE], concurrency: 10, wait_time: 1)
     @uuid      = SecureRandom.uuid
     @queues    = Array(queues)
-    @pool      = Concurrent::FixedThreadPool.new(concurrency, fallback_policy: :discard)
+    @pool      = Fleiss::Executor.new(max_size: concurrency)
     @wait_time = wait_time
-    @logger    = logger || Logger.new(nil)
   end
 
   # Run starts the worker
   def run
-    logger.info "Worker #{uuid} starting - queues: #{queues.inspect}, concurrency: #{@pool.max_length}"
+    log(:info) { "Worker #{uuid} starting - queues: #{queues.inspect}, concurrency: #{@pool.max_size}" }
     loop do
       run_cycle
       sleep @wait_time
     end
   rescue SignalException => e
-    logger.info "Worker #{uuid} received #{e.message}. Shutting down..."
+    log(:info) { "Worker #{uuid} received #{e.message}. Shutting down..." }
   ensure
     @pool.shutdown
     @pool.wait_for_termination
@@ -43,20 +39,31 @@ class Fleiss::Worker
 
   private
 
+  def log(severity, &block)
+    logger = ActiveJob::Base.logger
+    if logger.respond_to?(:tagged)
+      logger.tagged('Fleiss') { logger.send(severity, &block) }
+    else
+      logger.send(severity, &block)
+    end
+  end
+
   def run_cycle
-    return if @pool.shuttingdown?
+    return unless @pool.running?
 
-    capacity = @pool.max_length - @pool.scheduled_task_count + @pool.completed_task_count
-    return unless capacity.positive?
+    limit = @pool.capacity
+    return unless limit.positive?
 
     batch = Fleiss.backend
                   .in_queue(queues)
                   .pending
-                  .limit(capacity)
+                  .limit(limit)
                   .to_a
 
     batch.each do |job|
-      @pool.post { perform(job) }
+      @pool.post do
+        Fleiss.backend.wrap_perform { perform(job) }
+      end
     end
   rescue StandardError => e
     handle_exception e, 'running cycle'
@@ -67,7 +74,7 @@ class Fleiss::Worker
     owner     = "#{uuid}/#{thread_id}"
     return unless job.start(owner)
 
-    logger.info { "Worker #{uuid} execute job ##{job.id} (by thread #{thread_id})" }
+    log(:info) { "Worker #{uuid} execute job ##{job.id} (by thread #{thread_id})" }
     finished = false
     begin
       ActiveJob::Base.execute job.job_data
@@ -83,12 +90,12 @@ class Fleiss::Worker
   end
 
   def handle_exception(err, intro)
-    lines = [
-      "Worker #{uuid} error on #{intro}:",
-      "#{err.class.name}: #{err.message}",
-      err.backtrace,
-    ].compact.flatten
-
-    logger.error lines.join("\n")
+    log(:error) do
+      [
+        "Worker #{uuid} error on #{intro}:",
+        "#{err.class.name}: #{err.message}",
+        err.backtrace,
+      ].compact.flatten.join("\n")
+    end
   end
 end

data/spec/fleiss/executor_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+require 'fleiss/executor'
+
+RSpec.describe Fleiss::Executor do
+  subject { described_class.new max_size: 2 }
+  after   { subject.kill }
+
+  it 'should check capacity' do
+    expect(described_class.new.capacity).to eq(1)
+
+    expect(subject.capacity).to eq(2)
+    subject.post { sleep(1) }
+    expect(subject.capacity).to eq(1)
+    subject.post { sleep(1) }
+    expect(subject.capacity).to eq(0)
+  end
+
+  it 'should discard execution when capacity is reached' do
+    n = Concurrent::AtomicFixnum.new(0)
+    10.times do
+      10.times { subject.post { n.increment } }
+      sleep(0.001)
+    end
+    subject.shutdown
+    subject.wait_for_termination(1)
+    expect(n.value).to be_within(10).of(20)
+  end
+end

data/spec/spec_helper.rb

@@ -5,15 +5,19 @@ require 'fleiss'
 require 'fleiss/backend/active_record/migration'
 require 'active_job'
 require 'active_job/queue_adapters/fleiss_adapter'
-require 'tempfile'
+require 'fileutils'
 
 ActiveJob::Base.queue_adapter = :fleiss
 ActiveJob::Base.logger = Logger.new(nil)
 
 Time.zone_default = Time.find_zone!('UTC')
 
-database_url = ENV['DATABASE_URL'] || "sqlite3://#{Tempfile.new(['fleiss-test', '.sqlite3']).path}"
-ActiveRecord::Base.configurations['test'] = { 'url' => database_url, 'pool' => 20 }
+tmpdir = File.expand_path('./tmp', __dir__)
+FileUtils.rm_rf tmpdir
+FileUtils.mkdir_p tmpdir
+
+database_url = ENV['DATABASE_URL'] || "sqlite3://#{tmpdir}/fleiss-test.sqlite3"
+ActiveRecord::Base.configurations = { 'test' => { 'url' => database_url, 'pool' => 20 } }
 
 ActiveRecord::Base.establish_connection :test
 ActiveRecord::Base.connection.drop_table('fleiss_jobs', if_exists: true)
@@ -32,7 +36,7 @@ class TestJob < ActiveJob::Base
     72.hours
   end
 
-  def perform(msg=nil)
+  def perform(msg = nil)
     raise 'Failing' if msg == 'raise'
 
     self.class.performed.push(msg)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fleiss
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-12-19 00:00:00.000000000 Z
+date: 2020-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -176,8 +176,10 @@ files:
 - lib/fleiss/backend/active_record/concern.rb
 - lib/fleiss/backend/active_record/migration.rb
 - lib/fleiss/cli.rb
+- lib/fleiss/executor.rb
 - lib/fleiss/worker.rb
 - spec/fleiss/backend/active_record_spec.rb
+- spec/fleiss/executor_spec.rb
 - spec/fleiss/worker_spec.rb
 - spec/fleiss_spec.rb
 - spec/spec_helper.rb
@@ -193,20 +195,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.2'
+      version: '2.6'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Minimialist background jobs backed by ActiveJob and ActiveRecord.
 test_files:
 - spec/fleiss/backend/active_record_spec.rb
+- spec/fleiss/executor_spec.rb
 - spec/fleiss/worker_spec.rb
 - spec/fleiss_spec.rb
 - spec/spec_helper.rb