workerholic 0.0.15 → 0.0.16

data/spec/integration/dequeuing_and_job_processing_spec.rb

@@ -1,7 +1,7 @@
 require_relative '../spec_helper'
 
 describe 'dequeuing and processesing of jobs' do
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
 
   xit 'successfully dequeues and process a simple job' do
     serialized_job = Workerholic::JobSerializer.serialize(

data/spec/integration/enqueuing_jobs_spec.rb

@@ -1,15 +1,20 @@
 require_relative '../spec_helper'
 
 describe 'enqueuing jobs to Redis' do
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
 
   context 'successfully creates a job and enqueues it in Redis' do
     it 'enqueues a simple job in redis' do
       SimpleJobTest.new.perform_async('test job')
-      serialized_job = redis.lpop(TEST_QUEUE)
+      serialized_job = redis.lpop(WORKERHOLIC_QUEUE_NAMESPACE + TEST_QUEUE)
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
-      expected_job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'], wrapper: SimpleJobTest)
+      expected_job = Workerholic::JobWrapper.new(
+        klass: SimpleJobTest,
+        arguments: ['test job'],
+        wrapper: SimpleJobTest,
+        queue: WORKERHOLIC_QUEUE_NAMESPACE + TEST_QUEUE
+      )
       expected_job.statistics.enqueued_at = job_from_redis.statistics.enqueued_at
 
       expect(job_from_redis.to_hash).to eq(expected_job.to_hash)
@@ -17,13 +22,14 @@ describe 'enqueuing jobs to Redis' do
 
     it 'enqueues a complex job in redis' do
       ComplexJobTest.new.perform_async('test job', { a: 1, b: 2 }, [1, 2, 3])
-      serialized_job = redis.lpop(TEST_QUEUE)
+      serialized_job = redis.lpop(WORKERHOLIC_QUEUE_NAMESPACE + TEST_QUEUE)
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
       expected_job = Workerholic::JobWrapper.new(
         klass: ComplexJobTest,
         arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]],
-        wrapper: ComplexJobTest
+        wrapper: ComplexJobTest,
+        queue: WORKERHOLIC_QUEUE_NAMESPACE + TEST_QUEUE
       )
 
       expected_job.statistics.enqueued_at = job_from_redis.statistics.enqueued_at
@@ -31,20 +37,27 @@ describe 'enqueuing jobs to Redis' do
       expect(job_from_redis.to_hash).to eq(expected_job.to_hash)
     end
 
-    xit 'enqueues a delayed job in redis' do
+    it 'enqueues a delayed job in redis' do
       DelayedJobTest.new.perform_delayed(100, 'test job')
-      serialized_job = redis.lpop(TEST_QUEUE)
+
+      serialized_job, execution_time = redis.zrange('workerholic:scheduled_jobs', 0, 0, with_scores: true).first
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
-      expected_job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'], wrapper: SimpleJobTest)
+      expected_job = Workerholic::JobWrapper.new(
+        klass: DelayedJobTest,
+        arguments: ['test job'],
+        wrapper: DelayedJobTest,
+        queue: 'workerholic:queue:main'
+      )
       expected_job.statistics.enqueued_at = job_from_redis.statistics.enqueued_at
 
       expect(job_from_redis.to_hash).to eq(expected_job.to_hash)
+      expect(Time.now.to_i + 100).to eq(execution_time.to_i)
     end
 
     it 'enqueues a job with the right statistics' do
       SimpleJobTest.new.perform_async('test_job')
-      serialized_job = redis.lpop(TEST_QUEUE)
+      serialized_job = redis.lpop(WORKERHOLIC_QUEUE_NAMESPACE + TEST_QUEUE)
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
       expect(job_from_redis.statistics.enqueued_at).to be < Time.now.to_f

data/spec/job_retry_spec.rb

@@ -7,7 +7,7 @@ class JobWithError
 end
 
 describe Workerholic::JobRetry do
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
 
   it 'increments retry count' do
     job = Workerholic::JobWrapper.new(class: JobWithError, arguments: [])

data/spec/job_scheduler_spec.rb

@@ -1,47 +1,59 @@
 require_relative 'spec_helper'
 
 describe Workerholic::JobScheduler do
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
   let(:scheduler) do
     Workerholic::JobScheduler.new(
-      sorted_set: Workerholic::SortedSet.new(TEST_SCHEDULED_SORTED_SET),
-      queue_name: TEST_QUEUE
+      sorted_set: Workerholic::SortedSet.new(TEST_SCHEDULED_SORTED_SET)
     )
   end
+  let(:serialized_job) do
+    job = Workerholic::JobWrapper.new(
+      class: ComplexJobTest,
+      arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]],
+      queue: Workerholic::Queue.new.name
+    )
 
-  let(:redis) { Redis.new }
+    Workerholic::JobSerializer.serialize(job)
+  end
 
-  context 'with non-empty set' do
-    let(:serialized_job) do
-      job = Workerholic::JobWrapper.new(
-        class: ComplexJobTest,
-        arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]]
-      )
+  it 'checks the time for scheduled job inside sorted set' do
+    score = Time.now.to_f
+    scheduler.schedule(serialized_job, score)
 
-      Workerholic::JobSerializer.serialize(job)
-    end
+    expect(scheduler.job_due?).to eq(true)
+  end
 
-    it 'checks the time for scheduled job inside sorted set' do
-      score = Time.now.to_f
-      scheduler.schedule(serialized_job, score)
+  it 'fetches a job from a sorted set' do
+    score = Time.now.to_f
+    scheduler.schedule(serialized_job, score)
+    scheduler.enqueue_due_jobs
 
-      expect(scheduler.job_due?).to eq(true)
-    end
+    expect(scheduler.sorted_set.empty?).to eq(true)
+  end
 
-    it 'fetches a job from a sorted set' do
-      score = Time.now.to_f
-      scheduler.schedule(serialized_job, score)
-      scheduler.enqueue_due_jobs
+  it 'enqueues due job to the main queue by default' do
+    score = Time.now.to_f
+    scheduler.schedule(serialized_job, score)
+    scheduler.enqueue_due_jobs
 
-      expect(scheduler.sorted_set.empty?).to eq(true)
-    end
+    expect(Workerholic::Queue.new.empty?).to eq(false)
+    expect(Workerholic::Queue.new.dequeue).to eq(serialized_job)
+  end
+
+  it 'enqueues due job in specific queue' do
+    job = Workerholic::JobWrapper.new(
+      class: ComplexJobTest,
+      arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]],
+      queue: TEST_QUEUE
+    )
+    serialized_job = Workerholic::JobSerializer.serialize(job)
+    score = Time.now.to_f
 
-    it 'enqueues due job to the main queue' do
-      score = Time.now.to_f
-      scheduler.schedule(serialized_job, score)
-      scheduler.enqueue_due_jobs
+    scheduler.schedule(serialized_job, score)
+    scheduler.enqueue_due_jobs
 
-      expect(scheduler.queue.empty?).to eq(false)
-      expect(scheduler.queue.dequeue).to eq(serialized_job)
-    end
+    expect(Workerholic::Queue.new(TEST_QUEUE).empty?).to eq(false)
+    expect(Workerholic::Queue.new(TEST_QUEUE).dequeue).to eq(serialized_job)
   end
 end

data/spec/job_wrapper_spec.rb

@@ -2,12 +2,13 @@ require_relative 'spec_helper'
 
 describe Workerholic::JobWrapper do
   it 'returns a hash with job meta info and job stats info' do
-    job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'])
+    job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'], queue: TEST_QUEUE)
 
     expected_result = {
       klass: SimpleJobTest,
       wrapper: nil,
       arguments: ['test job'],
+      queue: TEST_QUEUE,
       retry_count: 0,
       execute_at: nil,
       statistics: {

data/spec/queue_spec.rb

@@ -1,7 +1,7 @@
 require_relative 'spec_helper'
 
 describe Workerholic::Queue do
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
   let(:queue) { Workerholic::Queue.new(TEST_QUEUE) }
   let(:job) { 'test job' }
 

data/spec/sorted_set_spec.rb

@@ -2,7 +2,7 @@ require_relative 'spec_helper'
 
 describe Workerholic::SortedSet do
   let(:job) {{ class: SimpleJobTest, arguments: [] }}
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
   let(:sorted_set) { Workerholic::SortedSet.new(TEST_SCHEDULED_SORTED_SET) }
 
   it 'adds a serialized job to the sorted set' do

data/spec/spec_helper.rb

@@ -20,10 +20,6 @@ RSpec.configure do |config|
   config.shared_context_metadata_behavior = :apply_to_host_groups
 
   config.before do
-    Redis.new.del(TEST_QUEUE, ANOTHER_TEST_QUEUE, BALANCER_TEST_QUEUE, ANOTHER_BALANCER_TEST_QUEUE, TEST_SCHEDULED_SORTED_SET, HASH_TEST)
-  end
-
-  config.after do
-    Redis.new.del(TEST_QUEUE, ANOTHER_TEST_QUEUE, BALANCER_TEST_QUEUE, ANOTHER_BALANCER_TEST_QUEUE, TEST_SCHEDULED_SORTED_SET, HASH_TEST)
+    Redis.new(url: Workerholic::REDIS_URL).flushdb
   end
 end

data/spec/statistics_api_spec.rb

@@ -0,0 +1,73 @@
+require_relative 'spec_helper'
+
+describe Workerholic::StatsAPI do
+  let(:storage) { Workerholic::Storage::RedisWrapper.new }
+  let(:job) { Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job']) }
+
+  it 'returns full statistics for category' do
+    namespace = 'workerholic:stats:completed_jobs:*'
+
+    Workerholic::StatsStorage.save_job('completed_jobs', job)
+
+    jobs_classes = storage.get_keys_for_namespace(namespace)
+
+    serialized_job = storage.sorted_set_all_members(jobs_classes.first)
+    deserialized_job = Workerholic::JobSerializer.deserialize_stats(serialized_job.first)
+
+    stats_api_result = Workerholic::StatsAPI.job_statistics(category: 'completed_jobs')
+    first_job_stat = stats_api_result.first[0]
+
+    expect(stats_api_result.size).to eq 1
+    expect(first_job_stat).to eq deserialized_job
+  end
+
+  it 'returns jobs count statistics for category' do
+    Workerholic::StatsStorage.save_job('completed_jobs', job)
+
+    stats_api_result = Workerholic::StatsAPI.job_statistics(category: 'completed_jobs', count_only: true)
+    stat_counter = stats_api_result.first
+
+    expect(stat_counter).to eq [job.klass.to_s, 1]
+  end
+
+  context 'with scheduled job' do
+    let(:namespace) { 'workerholic:scheduled_jobs' }
+
+    before do
+      sorted_set = Workerholic::SortedSet.new(namespace)
+
+      job.execute_at = Time.now.to_f + 10
+      job_hash = job.to_hash
+      job_hash[:klass] = job.klass.to_s
+      job_hash[:wrapper] = nil
+
+      sorted_set.add(Workerholic::JobSerializer.serialize(job_hash), job_hash[:execute_at])
+    end
+
+    it 'returns scheduled jobs statistics' do
+      serialized_job = storage.sorted_set_all_members(namespace).first
+      deserialized_job = Workerholic::JobSerializer.deserialize_stats(serialized_job)
+
+      stats_api_result = Workerholic::StatsAPI.scheduled_jobs
+
+      first_scheduled_job = stats_api_result.first
+
+      expect(first_scheduled_job).to eq deserialized_job
+    end
+
+    it 'returns scheduled jobs count' do
+      stats_api_result = Workerholic::StatsAPI.scheduled_jobs(count_only: true)
+
+      expect(stats_api_result).to eq 1
+    end
+  end
+
+  it 'returns historical statistics for namespace' do
+    category = 'completed_jobs'
+    Workerholic::StatsStorage.update_historical_stats(category, job.klass.to_s)
+
+    history_hash = Workerholic::StatsAPI.history_for_period(category: category, period: 1)
+
+    expect(history_hash[:job_counts]).to match [1, 0]
+  end
+end

data/spec/statistics_storage_spec.rb

@@ -0,0 +1,42 @@
+require_relative 'spec_helper'
+
+describe Workerholic::StatsStorage do
+  let(:storage) { Workerholic::Storage::RedisWrapper.new }
+  let(:stats_namespace) { 'workerholic:stats' }
+
+  it 'saves job stats' do
+    job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'])
+
+    job_hash = job.to_hash
+    job_hash[:klass] = job_hash[:klass].to_s
+    job_hash[:wrapper] = nil
+
+    local_namespace = ":completed_jobs:#{job.klass.to_s}"
+    namespace = stats_namespace + local_namespace
+
+    Workerholic::StatsStorage.save_job('completed_jobs', job)
+
+    serialized_stats = storage.sorted_set_all_members(namespace).first
+    deserialized_stats = Workerholic::JobSerializer.deserialize_stats(serialized_stats)
+
+    expect(storage.sorted_set_size(namespace)).to eq 1
+    expect(deserialized_stats).to eq job_hash
+  end
+
+  it 'saves process memory usage' do
+    Workerholic::StatsStorage.save_processes_memory_usage
+
+    namespace = stats_namespace + ':memory:processes'
+    process_data = storage.hash_get_all(namespace)
+
+    expect(storage.hash_get(namespace, process_data.keys.first)).to eq process_data.values.first
+  end
+
+  it 'cleans previous metrics records' do
+    Workerholic::StatsStorage.delete_memory_stats
+
+    namespace = stats_namespace + ':memory:processes'
+
+    expect(storage.hash_get_all(namespace).empty?).to be true
+  end
+end

data/spec/storage_spec.rb

@@ -2,7 +2,7 @@ require_relative 'spec_helper'
 
 describe Workerholic::Storage do
   let(:storage) { Workerholic::Storage::RedisWrapper.new }
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
   let(:queue_name) { TEST_QUEUE }
   let(:job) { 'test job' }
 
@@ -63,10 +63,10 @@ describe Workerholic::Storage do
     end
 
     it 'returns the workerholic queue names that are in redis' do
-      storage.push(queue_name, job)
-      storage.push(ANOTHER_TEST_QUEUE, job)
+      storage.push(WORKERHOLIC_QUEUE_NAMESPACE + queue_name, job)
+      storage.push(WORKERHOLIC_QUEUE_NAMESPACE + ANOTHER_TEST_QUEUE, job)
 
-      expect(storage.fetch_queue_names).to match_array([queue_name, ANOTHER_TEST_QUEUE])
+      expect(storage.fetch_queue_names).to match_array([WORKERHOLIC_QUEUE_NAMESPACE + queue_name, WORKERHOLIC_QUEUE_NAMESPACE + ANOTHER_TEST_QUEUE])
     end
 
     it 'sets k and a value to a hash in redis' do

data/spec/worker_balancer_spec.rb

@@ -1,10 +1,13 @@
 require_relative 'spec_helper'
 
-TESTED_QUEUES = [BALANCER_TEST_QUEUE, ANOTHER_BALANCER_TEST_QUEUE]
+TESTED_QUEUES = [
+  WORKERHOLIC_QUEUE_NAMESPACE + BALANCER_TEST_QUEUE,
+  WORKERHOLIC_QUEUE_NAMESPACE + ANOTHER_BALANCER_TEST_QUEUE
+]
 
 describe Workerholic::WorkerBalancer do
   let(:storage) { Workerholic::Storage::RedisWrapper.new }
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
 
   before do
     100.times do |n|

data/spec/worker_spec.rb

@@ -17,7 +17,7 @@ class WorkerJobTest
 end
 
 describe Workerholic::Worker do
-  let(:redis) { Redis.new }
+  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
   let(:job) do
     {
       klass: WorkerJobTest,

data/web/application.rb

@@ -1,6 +1,7 @@
+$LOAD_PATH << __dir__ + '/../lib'
 require 'sinatra/base'
 
-# require 'sinatra/reloader'
+require 'sinatra/reloader'
 require 'json'
 require 'workerholic'
 
@@ -11,6 +12,8 @@ class WorkerholicWeb < Sinatra::Base
   end
 
   get '/overview' do
+    @processes = Workerholic::StatsAPI.process_stats
+
     erb :overview
   end
 
@@ -49,13 +52,22 @@ class WorkerholicWeb < Sinatra::Base
     erb :queues
   end
 
+  get '/history' do
+    @days = params[:days]
+    @classes = Workerholic::StatsAPI.jobs_classes(true)
+    @class = params[:class] || 'completed'
+
+    erb :history
+  end
+
   get '/overview-data' do
     JSON.generate({
       completed_jobs: Workerholic::StatsAPI.job_statistics( {category: 'completed_jobs', count_only: true} ),
       failed_jobs: Workerholic::StatsAPI.job_statistics( {category: 'failed_jobs', count_only: true} ),
       queued_jobs: Workerholic::StatsAPI.queued_jobs,
+      scheduled_jobs: Workerholic::StatsAPI.scheduled_jobs( { count_only: true }),
       workers_count: Workerholic.workers_count,
-      # memory_usage: Workerholic::StatsAPI.memory_usage
+      memory_usage: Workerholic::StatsAPI.process_stats,
     })
   end
 
@@ -72,4 +84,13 @@ class WorkerholicWeb < Sinatra::Base
     })
   end
 
+  get '/historic-data' do
+    puts params[:className]
+    params[:className] = nil if params[:className] == 'completed'
+
+    JSON.generate({
+      completed_jobs: Workerholic::StatsAPI.history_for_period({ category: 'completed_jobs', klass: params[:className], period: params[:days].to_i }),
+      failed_jobs: Workerholic::StatsAPI.history_for_period({ category: 'failed_jobs', klass: params[:className], period: params[:days].to_i })
+    })
+  end
 end