fleiss 0.4.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cb419b5311dcda76bb9c1743b08ea56cfee270fbe6f90721347e492b2dfec292
-  data.tar.gz: 995152a6f0482b04f272ed70ab1426be7fd225eed7c30c80de638e2b56f7e119
+  metadata.gz: 595c3a8342470029088f398a77d8ca94cbaf78ff78a26bdda6ba31e204dc776f
+  data.tar.gz: '084f59a371721136d41294463fdd5cb41f0bdf99f958ca30cbf991863c1d35c1'
 SHA512:
-  metadata.gz: 9d12678587c252a310e7fac2cc80a420076b229a3cf93458016b572b749d002b652d53ce08cf74b28e75b92f50be6c8aac948d146261a1b113db1fa844827a99
-  data.tar.gz: '0107013391774abea71a113ac3dc3849c3a08e340ff534d68468629b2e4832798763320e11251ee57ee927c5f4210ed64bf945eedbc57f08086f7415c2fd0a2c'
+  metadata.gz: 336dcf65cd04c2e3dd005aef1e5c59d11dc6c25ab01b7cfe2b627f101c992d43e554f4a97a21a7d8886e1c0c7e19e400008d746c0d1fec302fa942dcdcd04171
+  data.tar.gz: 500004b1f9825d10dce9ae1b1178e27f71898e45ab7df6376b19f542afd24444dd8913286f9b46d4a71d2eecf1ba6c843f9a286216f9a8221b9a00891f2da5c9

data/.github/workflows/test.yml

@@ -11,9 +11,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby-version: ["2.6", "2.7", "3.0"]
+        ruby-version: ["2.7", "3.0", "3.1"]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby-version }}

data/.rubocop.yml

@@ -5,7 +5,7 @@ inherit_mode:
   merge:
     - Exclude
 AllCops:
-  TargetRubyVersion: "2.6"
+  TargetRubyVersion: "2.7"
 
 Security/Open:
   Exclude:

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fleiss (0.4.3)
+    fleiss (0.5.0)
       activejob (>= 6.0)
       activerecord (>= 6.0)
       concurrent-ruby
@@ -9,80 +9,79 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    activejob (6.1.4)
-      activesupport (= 6.1.4)
+    activejob (7.0.3.1)
+      activesupport (= 7.0.3.1)
       globalid (>= 0.3.6)
-    activemodel (6.1.4)
-      activesupport (= 6.1.4)
-    activerecord (6.1.4)
-      activemodel (= 6.1.4)
-      activesupport (= 6.1.4)
-    activesupport (6.1.4)
+    activemodel (7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activerecord (7.0.3.1)
+      activemodel (= 7.0.3.1)
+      activesupport (= 7.0.3.1)
+    activesupport (7.0.3.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-      zeitwerk (~> 2.3)
     ast (2.4.2)
-    concurrent-ruby (1.1.9)
-    diff-lcs (1.4.4)
-    globalid (0.5.1)
+    concurrent-ruby (1.1.10)
+    diff-lcs (1.5.0)
+    globalid (1.0.0)
       activesupport (>= 5.0)
-    i18n (1.8.10)
+    i18n (1.11.0)
       concurrent-ruby (~> 1.0)
-    minitest (5.14.4)
-    mysql2 (0.5.3)
-    parallel (1.20.1)
-    parser (3.0.2.0)
+    json (2.6.2)
+    minitest (5.16.2)
+    mysql2 (0.5.4)
+    parallel (1.22.1)
+    parser (3.1.2.0)
       ast (~> 2.4.1)
-    pg (1.2.3)
-    rainbow (3.0.0)
+    pg (1.4.1)
+    rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.1.1)
+    regexp_parser (2.5.0)
     rexml (3.2.5)
-    rspec (3.10.0)
-      rspec-core (~> 3.10.0)
-      rspec-expectations (~> 3.10.0)
-      rspec-mocks (~> 3.10.0)
-    rspec-core (3.10.1)
-      rspec-support (~> 3.10.0)
-    rspec-expectations (3.10.1)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-mocks (3.10.2)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.10.0)
-    rspec-support (3.10.2)
-    rubocop (1.18.3)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.0)
+    rubocop (1.31.2)
+      json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.0.0.0)
+      parser (>= 3.1.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
-      rexml
-      rubocop-ast (>= 1.7.0, < 2.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.18.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.8.0)
-      parser (>= 3.0.1.1)
+    rubocop-ast (1.19.1)
+      parser (>= 3.1.1.0)
     rubocop-bsm (0.6.0)
       rubocop (~> 1.0)
       rubocop-performance
       rubocop-rake
       rubocop-rspec
-    rubocop-performance (1.11.4)
+    rubocop-performance (1.14.2)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
     rubocop-rake (0.6.0)
       rubocop (~> 1.0)
-    rubocop-rspec (2.4.0)
-      rubocop (~> 1.0)
-      rubocop-ast (>= 1.1.0)
+    rubocop-rspec (2.12.1)
+      rubocop (~> 1.31)
     ruby-progressbar (1.11.0)
-    sqlite3 (1.4.2)
+    sqlite3 (1.4.4)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.0.0)
-    zeitwerk (2.4.2)
+    unicode-display_width (2.2.0)
 
 PLATFORMS
   ruby
@@ -98,4 +97,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   2.2.16
+   2.3.16

data/README.md

@@ -38,6 +38,16 @@ class ExpringJob < ActiveJob::Base
 end
 ```
 
+Allow to subscribe on worker perform method and detect errors
+
+```ruby
+ActiveSupport::Notifications.subscribe('worker_perform.fleiss') do |event|
+  break unless event.payload.key?(:exception_object)
+
+  Raven.capture_exception(event.payload[:exception_object])
+end
+```
+
 Include the data migration:
 
 ```ruby

data/fleiss.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'fleiss'
-  s.version       = '0.4.3'
+  s.version       = '0.5.0'
   s.authors       = ['Black Square Media Ltd']
   s.email         = ['info@blacksquaremedia.com']
   s.summary       = %(Minimialist background jobs backed by ActiveJob and ActiveRecord.)
@@ -10,9 +10,8 @@ Gem::Specification.new do |s|
 
   s.executables   = ['fleiss']
   s.files         = `git ls-files -z`.split("\x0").reject {|f| f.match(%r{^spec/}) }
-  s.test_files    = `git ls-files -z -- spec/*`.split("\x0")
   s.require_paths = ['lib']
-  s.required_ruby_version = '>= 2.6'
+  s.required_ruby_version = '>= 2.7'
 
   s.add_dependency 'activejob', '>= 6.0'
   s.add_dependency 'activerecord', '>= 6.0'
@@ -25,4 +24,5 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rubocop-bsm'
   s.add_development_dependency 'sqlite3'
+  s.metadata['rubygems_mfa_required'] = 'true'
 end

data/lib/active_job/queue_adapters/fleiss_adapter.rb

@@ -3,11 +3,11 @@ require 'fleiss'
 module ActiveJob
   module QueueAdapters
     class FleissAdapter
-      def enqueue(job) #:nodoc:
+      def enqueue(job) # :nodoc:
         enqueue_at(job, nil)
       end
 
-      def enqueue_at(job, scheduled_at) #:nodoc:
+      def enqueue_at(job, scheduled_at) # :nodoc:
         job_id = Fleiss.backend.enqueue(job, scheduled_at: scheduled_at)
         job.provider_job_id = job_id
         job_id

data/lib/fleiss/backend/active_record/concern.rb

@@ -20,7 +20,7 @@ module Fleiss
           def wrap_perform(&block)
             connection_pool.with_connection(&block)
           rescue ::ActiveRecord::StatementInvalid
-            ::ActiveRecord::Base.clear_active_connections!
+            ::ActiveRecord::Base.clear_all_connections!
             raise
           end
 

data/lib/fleiss/cli.rb

@@ -31,7 +31,7 @@ module Fleiss
       return unless opts[:config]
 
       # Load config file
-      conf = YAML.safe_load(ERB.new(IO.read(opts[:config])).result)
+      conf = YAML.safe_load(ERB.new(File.read(opts[:config])).result)
       raise ArgumentError, "File in #{opts[:config]} does not contain a valid configuration" unless conf.is_a?(Hash)
 
       conf.each do |key, value|

data/lib/fleiss/worker.rb

@@ -11,8 +11,6 @@ class Fleiss::Worker
   end
 
   # Init a new worker instance
-  # @param [ConnectionPool] disque client connection pool
-  # @param [Hash] options
   # @option [Array<String>] :queues queues to watch. Default: ["default"]
   # @option [Integer] :concurrency the number of concurrent pool. Default: 10
   # @option [Numeric] :wait_time maximum time (in seconds) to wait for jobs when retrieving next batch. Default: 1s.
@@ -62,21 +60,24 @@ class Fleiss::Worker
 
     batch.each do |job|
       @pool.post do
-        Fleiss.backend.wrap_perform { perform(job) }
+        thread_id = Thread.current.object_id.to_s(16).reverse
+        Fleiss.backend.wrap_perform { perform(job, thread_id) }
+      rescue StandardError => e
+        log_exception e, "processing job ##{job.id} (by thread #{thread_id})"
       end
     end
   rescue StandardError => e
-    handle_exception e, 'running cycle'
+    log_exception e, 'running cycle'
   end
 
-  def perform(job)
-    thread_id = Thread.current.object_id.to_s(16).reverse
-    owner     = "#{uuid}/#{thread_id}"
+  def perform(job, thread_id)
+    owner = "#{uuid}/#{thread_id}"
     return unless job.start(owner)
 
     log(:info) { "Worker #{uuid} execute job ##{job.id} (by thread #{thread_id})" }
     finished = false
-    begin
+
+    ActiveSupport::Notifications.instrument('worker_perform.fleiss', id: job.id, uuid: uuid, thread_id: thread_id) do |payload|
       ActiveJob::Base.execute job.job_data
       finished = true
     rescue StandardError
@@ -84,12 +85,11 @@ class Fleiss::Worker
       raise
     ensure
       finished ? job.finish(owner) : job.reschedule(owner)
+      payload[:finished] = finished
     end
-  rescue StandardError => e
-    handle_exception e, "processing job ##{job.id} (by thread #{thread_id})"
   end
 
-  def handle_exception(err, intro)
+  def log_exception(err, intro)
     log(:error) do
       [
         "Worker #{uuid} error on #{intro}:",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fleiss
 version: !ruby/object:Gem::Version
-  version: 0.4.3
+  version: 0.5.0
 platform: ruby
 authors:
 - Black Square Media Ltd
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-07-29 00:00:00.000000000 Z
+date: 2022-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activejob
@@ -178,15 +178,11 @@ files:
 - lib/fleiss/cli.rb
 - lib/fleiss/executor.rb
 - lib/fleiss/worker.rb
-- spec/fleiss/backend/active_record_spec.rb
-- spec/fleiss/executor_spec.rb
-- spec/fleiss/worker_spec.rb
-- spec/fleiss_spec.rb
-- spec/spec_helper.rb
 homepage: https://github.com/bsm/fleiss
 licenses:
 - Apache-2.0
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -195,20 +191,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.15
+rubygems_version: 3.3.7
 signing_key: 
 specification_version: 4
 summary: Minimialist background jobs backed by ActiveJob and ActiveRecord.
-test_files:
-- spec/fleiss/backend/active_record_spec.rb
-- spec/fleiss/executor_spec.rb
-- spec/fleiss/worker_spec.rb
-- spec/fleiss_spec.rb
-- spec/spec_helper.rb
+test_files: []

data/spec/fleiss/backend/active_record_spec.rb

@@ -1,135 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Fleiss::Backend::ActiveRecord do
-  def retrieve(job)
-    described_class.find(job.provider_job_id)
-  end
-
-  it 'persists jobs' do
-    job = TestJob.perform_later
-    rec = retrieve(job)
-
-    expect(rec.attributes).to include(
-      'queue_name'  => 'test-queue',
-      'owner'       => nil,
-      'started_at'  => nil,
-      'finished_at' => nil,
-    )
-    expect(rec.scheduled_at).to be_within(2.seconds).of(Time.zone.now)
-    expect(rec.expires_at).to be_within(2.seconds).of(3.days.from_now)
-    expect(rec.job_data).to include(
-      'job_class'       => 'TestJob',
-      'arguments'       => [],
-      'executions'      => 0,
-      'locale'          => 'en',
-      'priority'        => nil,
-      'provider_job_id' => nil,
-      'queue_name'      => 'test-queue',
-    )
-  end
-
-  it 'enqueues with delay' do
-    job = TestJob.set(wait: 1.day).perform_later
-    rec = retrieve(job)
-    expect(rec.scheduled_at).to be_within(2.seconds).of(1.day.from_now)
-  end
-
-  it 'enqueues with priority' do
-    job = TestJob.set(priority: 8).perform_later
-    rec = retrieve(job)
-    expect(rec.priority).to eq(8)
-  end
-
-  it 'exposes active job ID' do
-    job = TestJob.perform_later
-    rec = retrieve(job)
-    expect(rec.job_id.size).to eq(36)
-  end
-
-  it 'scopes pending' do
-    j1 = TestJob.perform_later
-    expect(retrieve(j1).start('owner')).to be_truthy
-    expect(retrieve(j1).finish('owner')).to be_truthy
-
-    j2 = TestJob.perform_later
-    _j3 = TestJob.set(wait: 1.hour).perform_later
-    j4 = TestJob.set(priority: 2).perform_later
-
-    expect(described_class.pending.ids).to eq [j4.provider_job_id, j2.provider_job_id]
-  end
-
-  it 'scopes in_progress' do
-    _j1 = TestJob.perform_later
-    j2 = TestJob.perform_later
-    expect(retrieve(j2).start('owner')).to be_truthy
-
-    j3 = TestJob.perform_later
-    expect(retrieve(j3).start('owner')).to be_truthy
-    expect(described_class.in_progress('owner').ids).to match_array [j2.provider_job_id, j3.provider_job_id]
-    expect(described_class.in_progress('other').ids).to be_empty
-
-    expect(retrieve(j3).finish('owner')).to be_truthy
-    expect(described_class.in_progress('owner').ids).to eq [j2.provider_job_id]
-  end
-
-  it 'scopes by queue' do
-    j1 = TestJob.perform_later
-    j2 = TestJob.set(queue: 'other').perform_later
-    expect(described_class.in_queue('test-queue').ids).to eq [j1.provider_job_id]
-    expect(described_class.in_queue('other').ids).to eq [j2.provider_job_id]
-  end
-
-  it 'starts' do
-    job = TestJob.perform_later
-    rec = retrieve(job)
-    expect(rec.start('owner')).to be_truthy
-    expect(rec.start('other')).to be_falsey
-    expect(rec.reload.owner).to eq('owner')
-    expect(rec.started_at).to be_within(2.seconds).of(Time.zone.now)
-  end
-
-  it 'locks atomically' do
-    24.times do
-      TestJob.perform_later
-    end
-    counts = (1..4).map do |n|
-      Thread.new do
-        described_class.pending.to_a.count {|j| j.start "owner-#{n}" }
-      end
-    end.map(&:value)
-    expect(counts.sum).to eq(24)
-  end
-
-  it 'finishes' do
-    job = TestJob.perform_later
-    rec = retrieve(job)
-    expect(rec.finish('owner')).to be_falsey
-    expect(rec.start('owner')).to be_truthy
-    expect(rec.finish('other')).to be_falsey
-    expect(rec.finish('owner')).to be_truthy
-    expect(rec.reload.owner).to eq('owner')
-    expect(rec.started_at).to be_within(2.seconds).of(Time.zone.now)
-    expect(rec.finished_at).to be_within(2.seconds).of(Time.zone.now)
-  end
-
-  it 'reschedules' do
-    job = TestJob.perform_later
-    rec = retrieve(job)
-    expect(rec.reschedule('owner')).to be_falsey
-    expect(rec.start('owner')).to be_truthy
-    expect(rec.reschedule('other')).to be_falsey
-    expect(rec.reschedule('owner')).to be_truthy
-    expect(rec.reload.owner).to be_nil
-    expect(rec.started_at).to be_nil
-    expect(rec.scheduled_at).to be_within(2.seconds).of(Time.zone.now)
-  end
-
-  it 'reconnects' do
-    expect(::ActiveRecord::Base).to receive(:clear_active_connections!).once.and_return(nil)
-
-    expect do
-      described_class.wrap_perform { raise ::ActiveRecord::StatementInvalid }
-    end
-      .to raise_error(::ActiveRecord::StatementInvalid) # re-raised anyway
-  end
-end

data/spec/fleiss/executor_spec.rb

@@ -1,29 +0,0 @@
-require 'spec_helper'
-require 'fleiss/executor'
-
-RSpec.describe Fleiss::Executor do
-  subject { described_class.new max_size: 2 }
-
-  after   { subject.kill }
-
-  it 'checks capacity' do
-    expect(described_class.new.capacity).to eq(1)
-
-    expect(subject.capacity).to eq(2)
-    subject.post { sleep(1) }
-    expect(subject.capacity).to eq(1)
-    subject.post { sleep(1) }
-    expect(subject.capacity).to eq(0)
-  end
-
-  it 'discards execution when capacity is reached' do
-    n = Concurrent::AtomicFixnum.new(0)
-    10.times do
-      10.times { subject.post { n.increment } }
-      sleep(0.001)
-    end
-    subject.shutdown
-    subject.wait_for_termination(1)
-    expect(n.value).to be_within(10).of(20)
-  end
-end

data/spec/fleiss/worker_spec.rb

@@ -1,47 +0,0 @@
-require 'spec_helper'
-require 'fleiss/worker'
-
-RSpec.describe Fleiss::Worker do
-  subject do
-    described_class.new queues: TestJob.queue_name, wait_time: 0.01
-  end
-
-  let! :runner do
-    t = Thread.new { subject.run }
-    t.abort_on_exception = true
-    t
-  end
-
-  after do
-    runner.kill
-  end
-
-  def wait_for
-    100.times do
-      break if yield
-
-      sleep(0.1)
-    end
-    expect(yield).to be_truthy
-  end
-
-  it 'runs' do
-    # seed 24 jobs
-    24.times {|n| TestJob.perform_later(n) }
-    wait_for { Fleiss.backend.not_finished.count.positive? }
-
-    # ensure runner processes them all
-    wait_for { Fleiss.backend.not_finished.count.zero? }
-
-    # check what's been performed
-    expect(TestJob.performed.size).to eq(24)
-    expect(Fleiss.backend.finished.count).to eq(24)
-    expect(TestJob.performed).to match_array(0..23)
-  end
-
-  it 'handles failing jobs' do
-    TestJob.perform_later('raise')
-    wait_for { Fleiss.backend.not_finished.count.zero? }
-    expect(Fleiss.backend.finished.count).to eq(1)
-  end
-end

data/spec/fleiss_spec.rb

@@ -1,13 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Fleiss do
-  it 'has a backend' do
-    expect(described_class.backend).to eq(Fleiss::Backend::ActiveRecord)
-  end
-
-  it 'enqueues' do
-    expect do
-      TestJob.set(wait: 1.week).perform_later
-    end.to change { described_class.backend.count }.by(1)
-  end
-end

data/spec/spec_helper.rb

@@ -1,51 +0,0 @@
-ENV['RACK_ENV'] ||= 'test'
-
-require 'rspec'
-require 'fleiss'
-require 'fleiss/backend/active_record/migration'
-require 'active_job'
-require 'active_job/queue_adapters/fleiss_adapter'
-require 'fileutils'
-
-ActiveJob::Base.queue_adapter = :fleiss
-ActiveJob::Base.logger = Logger.new(nil)
-
-Time.zone_default = Time.find_zone!('UTC')
-
-tmpdir = File.expand_path('./tmp', __dir__)
-FileUtils.rm_rf tmpdir
-FileUtils.mkdir_p tmpdir
-
-database_url = ENV['DATABASE_URL'] || "sqlite3://#{tmpdir}/fleiss-test.sqlite3"
-ActiveRecord::Base.configurations = { 'test' => { 'url' => database_url, 'pool' => 20 } }
-
-ActiveRecord::Base.establish_connection :test
-ActiveRecord::Base.connection.drop_table('fleiss_jobs', if_exists: true)
-ActiveRecord::Migration.suppress_messages do
-  Fleiss::Backend::ActiveRecord::Migration.migrate(:up)
-end
-
-class TestJob < ActiveJob::Base
-  queue_as 'test-queue'
-
-  def self.performed
-    @performed ||= []
-  end
-
-  def ttl
-    72.hours
-  end
-
-  def perform(msg = nil)
-    raise 'Failing' if msg == 'raise'
-
-    self.class.performed.push(msg)
-  end
-end
-
-RSpec.configure do |c|
-  c.after do
-    TestJob.performed.clear
-    Fleiss.backend.delete_all
-  end
-end