rukawa 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e90c29a3fb28cf675e9c34e2be8a10e642819f19
-  data.tar.gz: f967d48f8faced73d118c7c752e3f1c28cb062fd
+  metadata.gz: b165a0824c474f449b62bd61beed9f96f3e11c39
+  data.tar.gz: bfaf7d5212c500d6084b07e4ca36baf745ab3efb
 SHA512:
-  metadata.gz: c98fbb5ebf627f6133337c4d88e88556ce199eec049a65ca610f1faa0c74f932461086cbd0687d18a28fd52378c6cb9d2d4a15522423e113d7c1a90318c5f0a5
-  data.tar.gz: 1c763aa161305e1d6db416be8f8237d40c4f432e9e3b8edefaad3002e179eece6da0bbc118913f1261ba4425e25cbb5ec5d77484ce16a588a6e71291f6489f27
+  metadata.gz: 0109918d72f5527124754e5ffdffa7c4ae97661a9c4a871e2c2620bc96e1514ee8affd3015a9e25d74134ee547c248eb41488f172333b8aa017ec1654e66cb4c
+  data.tar.gz: 2c662f7a992ae4172e4cb93d2ba5937bb25ef6aee7ddc9e5ef3cd929cc274fb5fec974035225c80a606cfd74d50a1d1138501594420f5e11c862902d6fb3bc1f

data/README.md

@@ -262,6 +262,57 @@ var2: value2
 }
 ```
 
+### Semaphore
+
+You can control concurrency consumption.
+
+```ruby
+class Job < Rukawa::Job
+  set_resource_count 2
+
+  def run
+    # process
+  end
+end
+```
+
+This job use 2 concurrency. (this does not means that job use 2 threads)
+If concurrency is less than jobs's resource count, resource count is set concurrency size.
+
+### Callback
+
+- before\_run
+- after\_run
+- around\_run
+- after\_fail
+
+```ruby
+class Job < Rukawa::Job
+  before_run :wait_other_resource
+  after_run :notify_finish
+  around_run do |job, blk|
+    begin
+      setup_resource
+      blk.call
+    ensure
+      release_resource
+    end
+  end
+
+  def run
+    # process
+  end
+
+  def wait_other_resource
+    sleep(3)
+  end
+
+  def notify_finish
+    send_notification_to_slack
+  end
+end
+```
+
 ### Config Example
 
 ```

data/lib/rukawa.rb

@@ -2,12 +2,18 @@ require "concurrent"
 
 module Rukawa
   class << self
-    def logger
-      config.logger
+    def init
+      unless @initialized
+        @store = Concurrent::Hash.new
+        @executor = Concurrent::FixedThreadPool.new(config.concurrency)
+        @semaphore = Concurrent::Semaphore.new(config.concurrency)
+        @initialized = true
+      end
     end
+    attr_reader :store, :executor, :semaphore
 
-    def store
-      @store ||= Concurrent::Hash.new
+    def logger
+      config.logger
     end
 
     def configure
@@ -17,10 +23,6 @@ module Rukawa
     def config
       Configuration.instance
     end
-
-    def executor
-      @executor ||= Concurrent::FixedThreadPool.new(config.concurrency)
-    end
   end
 end
 

data/lib/rukawa/job.rb

@@ -3,15 +3,31 @@ require 'rukawa/abstract_job'
 require 'rukawa/dependency'
 require 'rukawa/state'
 require 'active_support/core_ext/class'
+require 'active_support/callbacks'
 
 module Rukawa
   class Job < AbstractJob
+    include ActiveSupport::Callbacks
+    define_callbacks :run,
+      terminator: ->(_,result) { result == false },
+      skip_after_callbacks_if_terminated: true,
+      scope: [:kind, :name],
+      only: [:before, :around, :after]
+
+    define_callbacks :fail,
+      terminator: ->(_,result) { result == false },
+      skip_after_callbacks_if_terminated: true,
+      scope: [:kind, :name],
+      only: [:after]
+
     attr_accessor :in_comings, :out_goings
     attr_reader :state, :started_at, :finished_at, :variables
 
     class_attribute :retryable, :retry_limit, :retry_exception_type, :retry_wait, instance_writer: false
     class_attribute :dependency_type, instance_writer: false
+    class_attribute :resource_count, instance_reader: false, instance_writer: false
     self.dependency_type = Dependency::AllSuccess
+    self.resource_count = 1
 
     class << self
       def set_retryable(limit: 8, type: nil, wait: nil)
@@ -24,6 +40,50 @@ module Rukawa
       def set_dependency_type(name)
         self.dependency_type = Rukawa::Dependency.get(name)
       end
+
+      def set_resource_count(count)
+        self.resource_count = count
+      end
+
+      def before_run(*args, **options, &block)
+        set_callback :run, :before, *args, **options, &block
+      end
+
+      def after_run(*args, **options, &block)
+        options[:prepend] = true
+        conditional = ActiveSupport::Callbacks::Conditionals::Value.new { |v|
+          v != false
+        }
+        options[:if] = Array(options[:if]) << conditional
+        set_callback :run, :after, *args, **options, &block
+      end
+
+      def after_fail(*args, **options, &block)
+        options[:prepend] = true
+        conditional = ActiveSupport::Callbacks::Conditionals::Value.new { |v|
+          v != false
+        }
+        options[:if] = Array(options[:if]) << conditional
+        set_callback :fail, :after, *args, **options, &block
+      end
+
+      def around_run(*args, **options, &block)
+        set_callback :run, :around, *args, **options, &block
+      end
+    end
+
+    around_run :acquire_resource
+
+    around_run do |_, blk|
+      Rukawa.logger.info("Start #{self.class}")
+      blk.call
+      Rukawa.logger.info("Finish #{self.class}")
+    end
+
+    around_run do |_, blk|
+      set_state(:running)
+      blk.call
+      set_state(:finished)
     end
 
     def initialize(parent_job_net, variables)
@@ -69,13 +129,12 @@ module Rukawa
         set_state(:skipped)
       else
         check_dependencies(results)
-        Rukawa.logger.info("Start #{self.class}")
-        set_state(:running)
-        run
-        Rukawa.logger.info("Finish #{self.class}")
-        set_state(:finished)
+        run_callbacks :run do
+          run
+        end
       end
     rescue => e
+      run_callbacks :fail
       handle_error(e)
       Rukawa.logger.error("Retry #{self.class}")
       retry
@@ -153,5 +212,16 @@ module Rukawa
       Rukawa.store[self.class] ||= Concurrent::Hash.new
       Rukawa.store[self.class][key] = value
     end
+
+    def resource_count
+      [self.class.resource_count, Rukawa.config.concurrency].min
+    end
+
+    def acquire_resource
+      Rukawa.semaphore.acquire(resource_count)
+      yield
+    ensure
+      Rukawa.semaphore.release(resource_count)
+    end
   end
 end

data/lib/rukawa/runner.rb

@@ -16,6 +16,7 @@ module Rukawa
     end
 
     def run(batch_mode = false, refresh_interval = DEFAULT_REFRESH_INTERVAL)
+      Rukawa.init
       Rukawa.logger.info("=== Start Rukawa ===")
       futures = @root_job_net.dataflows.each(&:execute)
       until futures.all?(&:complete?)

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/sample/jobs/sample_job.rb

@@ -4,6 +4,12 @@ module ExecuteLog
   end
 end
 
+class Notifier
+  def after_run(job)
+    puts "#{job.name}:#{job.state.name}:#{job.elapsed_time_from}"
+  end
+end
+
 class SampleJob < Rukawa::Job
   def run
     sleep rand(5)
@@ -13,6 +19,7 @@ end
 
 class Job1 < SampleJob
   set_description "Job1 description body"
+  after_run Notifier.new
 
   def run
     p variables unless variables.empty?
@@ -20,6 +27,10 @@ class Job1 < SampleJob
   end
 end
 class Job2 < SampleJob
+  after_fail do |job|
+    p "Job2 Failed!!! #{job.state.name}"
+  end
+
   def run
     raise "job2 error"
   end
@@ -29,6 +40,7 @@ end
 class Job4 < SampleJob
   # inherited by subclass
   set_dependency_type :one_success
+  set_resource_count 3
 end
 class Job5 < SampleJob
   # inherited by subclass

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-30 00:00:00.000000000 Z
+date: 2016-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -208,7 +208,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Hyper simple job workflow engine