dj_split 1.0.1 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc2e050c9e58032f167dc04f7d4685be6479e28bc5400fa53d7ff988fdafce7a
-  data.tar.gz: 91cccdc1b6ed01676da03b2652ee91ac4b09cc8f3d052ada1f798a147325e92f
+  metadata.gz: 4e4fe039dd8c959184e603089140121b8b271f6173427817fe54c78989afba57
+  data.tar.gz: d18389f1d0c76235d6b937b7e65de15e7fdb883530f86b3557c30f32fe79c9d9
 SHA512:
-  metadata.gz: dc3ef609a3fb6a3951800bb97fd711127d223735a3c2bef66b156b381d047ba7270f922c5bc776d0895725211c82feacca6a969c9f6192db5399fc040e79e148
-  data.tar.gz: a533c0b7839dd8c491971780f63328cc2966b4fbf2f3f9176ce5f54f55d83a054348514368e59c250e1488247a2b505fd46f539b10706ea2662d519ef55ca862
+  metadata.gz: 6393267924205a39bd92412ab52cc843694fb54bdb6dac8d08bf83fc80098c8eac91d658b708209085d84ac80d889e68bbecb9cd0a2e0e3fef4b859df7e52b0a
+  data.tar.gz: af50d3434ac9ea3790ab6d4cf927fc7bafc01bdc9fb16f8d9e30e90c982229af72872a34b5484554afd24487a40acd7b65b798137ca21cf99bc54bfa000bd4c0

data/README.md

@@ -1,4 +1,13 @@
-# Delayed Job Split Feature
+**If you're viewing this at https://github.com/nehalamin93/dj_split,
+you're reading the documentation for the master branch.
+[View documentation for the latest release
+(1.0.1).](https://github.com/nehalamin93/dj_split/tree/v1.0.1)**
+
+Delayed Job Split Feature
+=========================
+[![Gem Version](https://badge.fury.io/rb/dj_split.png)][gem]
+
+[gem]: https://rubygems.org/gems/dj_split
 
 Class [**"DjSplit::Split"**](https://github.com/nehalamin93/dj_split/blob/master/lib/dj_split/split.rb) is designed to **Split Time Taking Delayed Jobs, Crons, Bulk Operations, etc** into **smaller size multiple Delayed Jobs**.
 These **Sub-Jobs** should be **mutually exclusive** of each other and should be able to run **concurrently**.

data/dj_split.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["nehalamin93@gmail.com"]
 
   spec.summary       = %q{Parallelise Time Taking Job across Multiple Processes in Single or Multiple Servers by Splitting into Delayed Jobs.}
-  spec.description   = %q{Gem is designed to "Split or Break" Time Taking Jobs(Delayed Jobs, Crons, Bulk Operations, etc.) into smaller size Mutually Exclusive Delayed Jobs. These Sub-Jobs can be picked by multiple workers in Single or Multiple Servers. After splitting and enqueuing, the process will wait for the sub-jobs to complete and also processes sub-jobs instead of blocking. Parallelism can be achieved across multiple servers through Delayed Jobs which can directly impact performance. Performance can improve up to "n+1" times, where n = number of workers picking the jobs.}
+  spec.description   = %q{Gem is designed to "Split or Break" Time Taking Jobs(Delayed Jobs, Crons, Bulk Operations, etc.) into smaller size Mutually Exclusive Delayed Jobs. These Sub-Jobs can be picked by "Multiple Workers" in "Single" or "Multiple Servers". After splitting and enqueuing, the process will wait for the sub-jobs to complete and also processes sub-jobs instead of blocking. "Parallelism" can be achieved across multiple servers through Delayed Jobs which can directly impact performance. "Performance" can improve up to "n+1" times, where n = number of workers picking the jobs.}
   spec.homepage      = ""
   spec.license       = "MIT"
 
@@ -25,12 +25,12 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", ">= 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "5.11.3"
+  spec.add_development_dependency "minitest", "~> 5.11"
   spec.add_development_dependency 'rspec', '~> 3.6'
   spec.add_development_dependency "mocha", "~> 1.2"
 
   spec.add_dependency 'delayed_job', '>= 3.0'
   spec.add_dependency 'sqlite3'
-  spec.add_dependency 'delayed_job_active_record', '>= 0.4'
+  spec.add_dependency 'delayed_job_active_record', '>= 4.0'
   spec.add_development_dependency 'rails'
 end

data/lib/dj_split/split.rb

@@ -3,6 +3,8 @@ module DjSplit
 
     OPTIMAL_SPLIT_SIZE = 200
     DEFAULT_SPLIT_INDEX = 2
+    STALE_JOBS_TIMEOUT = 900 # Seconds. Can be passed in options.
+    SLEEP_TIME = 0.2 # Seconds.
 
     def initialize(options)
       @queue_options  = options[:queue_options] || {}
@@ -49,32 +51,21 @@ module DjSplit
       count = 0
       while(jobs_processed_by_other_workers_currently?)
         count += 1
-        sleep(1.0/5.0)
-        handle_stale_jobs if check_stale_jobs?(count)
+        sleep(SLEEP_TIME)
+        handle_stale_jobs if check_for_timeout?(count)
       end
     end
 
-    # handles a scenario when Job is locked by a worker and worker got killed
+    # Raise an error in scenario such as: Job is locked by a worker and worker got killed
     def handle_stale_jobs
-      stale_jobs = get_processing_jobs_by_other_workers.select{ |dj| !get_delayed_jobs_pids.include?(dj.locked_by.split("pid:")[1].strip) }
-      raise "Stale Delayed Jobs of Group Id(#{@split_group_id}): #{stale_jobs.pluck(:id)}" if stale_jobs.present?
+      stale_jobs = get_processing_jobs_by_other_workers
+      raise "Stale Delayed Jobs of Group Id(#{@split_group_id}): #{stale_jobs.pluck(:id)}"
     end
 
     def get_processing_jobs_by_other_workers
       Delayed::Job.where(split_group_id: @split_group_id, failed_at: nil).where.not(locked_by: nil)
     end
 
-    def get_delayed_jobs_pids
-      bash_op = `ps aux | grep delayed_job  | grep -v 'grep' | grep -v 'bin/sh' | grep -v 'tail' | grep -v 'ruby' | grep -v 'trace'` 
-      processes = bash_op.split("\n")
-      pid_array = []
-      processes.each do |process|
-        split_process = process.split(" ")
-        pid_array << split_process[1]
-      end
-      pid_array
-    end
-
     def handle_failed_jobs
       failed_jobs = Delayed::Job.where(split_group_id: @split_group_id).where.not(failed_at: nil)
       raise "Failed Delayed Jobs of Group Id(#{@split_group_id}): #{failed_jobs.pluck(:id)}" if failed_jobs.exists?
@@ -88,7 +79,11 @@ module DjSplit
       @split_options[:size] || OPTIMAL_SPLIT_SIZE
     end
 
-    def get_sliced_ids(ids) 
+    def get_stale_job_timeout_value
+      @split_options[:timeout] || STALE_JOBS_TIMEOUT
+    end
+
+    def get_sliced_ids(ids)
       ids.each_slice(get_split_size)
     end
 
@@ -101,9 +96,9 @@ module DjSplit
       Time.now.to_i.to_s[5..-1] + rand(1000000000).to_s
     end
 
-    # Check for stale jobs 1 out of 2000 times.
-    def check_stale_jobs?(count)
-      (count%2000 == 1999) 
+    # Check whether the timeout is reached?
+    def check_for_timeout?(count)
+      (SLEEP_TIME * count) > get_stale_job_timeout_value
     end
   end 
 end

data/lib/dj_split/version.rb

@@ -1,3 +1,3 @@
 module DjSplit
-  VERSION = "1.0.1"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dj_split
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.1.1
 platform: ruby
 authors:
 - Md Nehal Amin
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-04 00:00:00.000000000 Z
+date: 2018-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,16 +42,16 @@ dependencies:
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.11.3
+        version: '5.11'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 5.11.3
+        version: '5.11'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.4'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -138,10 +138,10 @@ dependencies:
         version: '0'
 description: Gem is designed to "Split or Break" Time Taking Jobs(Delayed Jobs, Crons,
   Bulk Operations, etc.) into smaller size Mutually Exclusive Delayed Jobs. These
-  Sub-Jobs can be picked by multiple workers in Single or Multiple Servers. After
-  splitting and enqueuing, the process will wait for the sub-jobs to complete and
-  also processes sub-jobs instead of blocking. Parallelism can be achieved across
-  multiple servers through Delayed Jobs which can directly impact performance. Performance
+  Sub-Jobs can be picked by "Multiple Workers" in "Single" or "Multiple Servers".
+  After splitting and enqueuing, the process will wait for the sub-jobs to complete
+  and also processes sub-jobs instead of blocking. "Parallelism" can be achieved across
+  multiple servers through Delayed Jobs which can directly impact performance. "Performance"
   can improve up to "n+1" times, where n = number of workers picking the jobs.
 email:
 - nehalamin93@gmail.com