joblin 0.1.0

data/spec/batching/pool_spec.rb

@@ -0,0 +1,161 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::Batching::Pool do
+  include ActiveJob::TestHelper
+
+  subject { described_class.new(order: pool_order, concurrency: pool_concurrency) }
+
+  let(:pool) { subject }
+
+  let(:pool_order) { :fifo }
+  let(:pool_concurrency) { 2 }
+
+  describe '#initialize' do
+    subject { described_class }
+
+    it 'creates pid when called without it' do
+      expect(subject.new.pid).not_to be_nil
+    end
+
+    it 'reuses pid when called with it' do
+      batch = subject.new('dayPO5KxuRXXxw')
+      expect(batch.pid).to eq('dayPO5KxuRXXxw')
+    end
+  end
+
+  def load_pool(count = 3)
+    jobs = count.times.map do |i|
+      { job: "BatchTestJobBase", args: [1] }
+    end
+    subject.add_jobs(jobs, skip_refill: true)
+  end
+
+  describe "#job_checked_in" do
+    it "gets called" do
+      expect(Joblin::Batching::Pool).to receive(:job_checked_in).twice
+      load_pool
+      perform_enqueued_jobs do
+        pool.send :refill_allotment
+        Sidekiq::Worker.drain_all
+      end
+    end
+
+    it "refills the pool" do
+      expect(Joblin::Batching::Pool).to receive(:job_checked_in).and_call_original.exactly(3).times
+
+      load_pool
+      perform_enqueued_jobs do
+        pool.send :refill_allotment
+        Sidekiq::Worker.drain_all
+      end
+    end
+  end
+
+  describe "#cleanup_if_empty" do
+    it "cleans if pool is empty and allowed to close" do
+      expect(pool).to receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has pending" do
+      load_pool
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has active" do
+      subject.redis.hset("#{subject.send(:redis_key)}-active", "blocked", "{}")
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool has activating" do
+      subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool is empty but is kept open" do
+      pool.keep_open!
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+
+    it "doesn't clean if pool is empty but clean_when_empty is false" do
+      subject.redis.hset(subject.send(:redis_key), "clean_when_empty", "false")
+      expect(pool).to_not receive(:cleanup_redis)
+      pool.cleanup_if_empty
+    end
+  end
+
+  shared_examples "basic pool tests" do
+    describe "#push_job_to_pool" do
+      it "adds a job to the pool" do
+        subject.send(:push_job_to_pool, { job: 'job1' })
+        expect(subject.pending_count).to eq(1)
+      end
+    end
+
+    describe "#refill_allotment" do
+      it "refills the pool with jobs" do
+        load_pool
+        expect(Joblin::Batching::ChainBuilder).to receive(:enqueue_job).twice
+        subject.send(:refill_allotment)
+      end
+
+      it "limits to the concurrency count" do
+        load_pool
+        expect(Joblin::Batching::ChainBuilder).to receive(:enqueue_job).twice
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 1
+      end
+
+      it "considers already active jobs against concurrency" do
+        load_pool
+        subject.redis.hset("#{subject.send(:redis_key)}-active", "blocked", "{}")
+        expect(Joblin::Batching::ChainBuilder).to receive(:enqueue_job).once
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 2
+      end
+
+      it "considers activating jobs against concurrency" do
+        load_pool
+        subject.redis.hincrby(subject.send(:redis_key), "_active_count", 1)
+        expect(Joblin::Batching::ChainBuilder).to receive(:enqueue_job).once
+        expect(subject.send(:refill_allotment)).to eql 2
+        expect(subject.pending_count).to eql 2
+      end
+
+      it "doesn't fail if the pool is gone" do
+        load_pool
+        subject.cleanup_redis
+        expect(Joblin::Batching::ChainBuilder).not_to receive(:enqueue_job)
+        expect(subject.send(:refill_allotment)).to eql -1
+      end
+
+      it "doesn't fail if the pool is empty" do
+        expect(subject.send(:refill_allotment)).to eql 0
+      end
+    end
+  end
+
+  context "FIFO Pool" do
+    let(:pool_order) { :fifo }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "LIFO Pool" do
+    let(:pool_order) { :lifo }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "Random Pool" do
+    let(:pool_order) { :random }
+    it_behaves_like "basic pool tests"
+  end
+
+  context "Priority Pool" do
+    let(:pool_order) { :priority }
+    it_behaves_like "basic pool tests"
+  end
+end

data/spec/batching/status_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+
+RSpec.describe Joblin::Batching::Batch::Status do
+  let(:batch) { Joblin::Batching::Batch.new() }
+  let(:bid) { batch.bid }
+  subject { described_class.new(bid) }
+
+  describe '#join' do
+    it 'raises info' do
+      expect { subject.join }.to raise_error('Not supported')
+    end
+  end
+
+  describe '#pending' do
+    context 'when not initalized' do
+      it 'returns 0 pending jobs' do
+        expect(subject.pending).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.jobs do BatchTestWorker.perform_async end }
+      it 'returns pending jobs' do
+        expect(subject.pending).to eq(1)
+      end
+    end
+  end
+
+  describe '#failures' do
+    context 'when not initalized' do
+      it 'returns 0 failed jobs' do
+        expect(subject.failures).to eq(0)
+      end
+    end
+
+    context 'when more than 0' do
+      before { batch.append_jobs(bid) }
+      before { Joblin::Batching::Batch.process_failed_job(bid, 'FAILEDID') }
+
+      it 'returns failed jobs' do
+        expect(subject.failures).to eq(1)
+      end
+    end
+  end
+
+  describe '#failure_info' do
+    context 'when not initalized' do
+      it 'returns empty array' do
+        expect(subject.failure_info).to eq([])
+      end
+    end
+
+    context 'when with error' do
+      before { batch.jobs{}; Joblin::Batching::Batch.process_failed_job(bid, 'jid123') }
+
+      it 'returns array with failed jids' do
+        expect(subject.failure_info).to eq(['jid123'])
+      end
+    end
+  end
+
+  describe '#data' do
+    it 'returns batch description' do
+      expect(subject.data).to include(failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil)
+    end
+  end
+
+  describe '#created_at' do
+    it 'returns time' do
+      batch = Joblin::Batching::Batch.new
+      batch.jobs do BatchTestWorker.perform_async end
+      status = described_class.new(batch.bid)
+      expect(status.created_at).not_to be_nil
+    end
+  end
+end

data/spec/batching/support/base_job.rb

@@ -0,0 +1,19 @@
+class BatchTestJobBase < ActiveJob::Base
+  def perform(*args, **kwargs)
+  end
+
+  def self.perform_async(*args)
+    perform_later(*args)
+  end
+end
+
+class FailingBatchTestJobBase < BatchTestJobBase
+  def perform
+    raise "Foo"
+  end
+end
+
+class BatchTestWorker < BatchTestJobBase
+  def perform
+  end
+end

data/spec/batching/support/sample_callback.rb

@@ -0,0 +1,2 @@
+  class SampleCallback; end
+  class SampleCallback2; end

data/spec/internal/config/database.yml

@@ -0,0 +1,5 @@
+test:
+  database: joblin-test
+  adapter: postgresql
+  encoding: unicode
+  pool: 15

data/spec/internal/config/routes.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.routes.draw do
+  # Add your own routes here, or remove this file if you don't have need for it.
+end

data/spec/internal/config/storage.yml

@@ -0,0 +1,3 @@
+test:
+  service: Disk
+  root: tmp/storage

data/spec/internal/db/schema.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+ActiveRecord::Schema.define do
+  # Set up any tables you need to exist for your test suite that don't belong
+  # in migrations.
+end