rukawa 0.5.1 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e9f4f1e22085cbcfcfafc854b03bc5538ee70044
-  data.tar.gz: 1644fab37cea4a2b001306a84641dbfb3a8da9e6
+  metadata.gz: 34dae79e6a5490afb50c25cd0f1a6784bba6e0da
+  data.tar.gz: 3e0df7f16eceb68d984fb0219fa5c7349fb529be
 SHA512:
-  metadata.gz: 08a3537bb614bec8ccff3e16443c44f72edae4cfa163821fbc7d2e0367081d4c689024be12c67895c59230024cc3ff6fbb44aae1056b3b7831599a6deeb912e2
-  data.tar.gz: bca2d4b61780f68ad08598f5e51d5f472438db4fb6dceb7f0d109242d461bd9bc69d19fdce337d9c6eea662e922d04a1a6b8a770167b590132023aff0ffa3cf8
+  metadata.gz: aa945187329bc472dc19f63e9086eba30232711100763e461f025cb44f6941755a5036a1fcac7872b414ff427f73ae3887daef221225fb617dca1fafc25650fc
+  data.tar.gz: 5a03f462462c210c33f03ec2bc61acf45c386c4c25314539f4d42af05b7e48aaf870782142df15281a6eb59fc8f4bd3d887e31211927239323e4c1c3bef11f6e

data/README.md

@@ -385,9 +385,25 @@ Options:
 Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
 ```
 
+### Usage from Ruby program
+
+```ruby
+currency = 4
+job_net = YourJobNetClass.new(nil, {"var1" => "value1"}, Context.new(currency))
+promise = job_net.run do
+  puts "Job Running"
+end
+
+promise.then do |futures|
+  errors = futures.map(&:reason).compact
+  unless errors.empty?
+    puts "JobNet has errors"
+  end
+end
+```
+
 ## ToDo
 - Write more tests
-- Enable use variables
 
 ## Development
 

data/lib/rukawa.rb

@@ -2,16 +2,6 @@ require "concurrent"
 
 module Rukawa
   class << self
-    def init
-      unless @initialized
-        @store = Concurrent::Hash.new
-        @executor = Concurrent::FixedThreadPool.new(config.concurrency)
-        @semaphore = Concurrent::Semaphore.new(config.concurrency)
-        @initialized = true
-      end
-    end
-    attr_reader :store, :executor, :semaphore
-
     def logger
       config.logger
     end
@@ -35,6 +25,7 @@ end
 
 require 'active_support'
 require "rukawa/version"
+require 'rukawa/context'
 require 'rukawa/errors'
 require 'rukawa/state'
 require 'rukawa/dependency'

data/lib/rukawa/cli.rb

@@ -35,14 +35,19 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, variables, *job_classes)
+      job_net = job_net_class.new(nil, variables, Context.new, *job_classes)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]
         job_net.output_dot(options[:dot], format: options[:format])
       end
 
-      exit 1 unless result
+      unless result
+        puts "\nIf you want to retry, run following command."
+        failed_jobs = job_net.dag.jobs.each_with_object([]) { |j, arr| arr << j.class.to_s if j.state == Rukawa::State::Error }
+        puts "  rukawa run #{job_net_name} #{failed_jobs.join(" ")}"
+        exit 1
+      end
     end
 
     desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
@@ -55,7 +60,7 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, {}, *job_classes)
+      job_net = job_net_class.new(nil, {}, Context.new, *job_classes)
       job_net.output_dot(options[:output], format: options[:format])
     end
 
@@ -70,7 +75,7 @@ module Rukawa
 
       job_classes = job_name.map { |name| get_class(name) }
       job_net_class = anonymous_job_net_class(*job_classes)
-      job_net = job_net_class.new(nil, variables)
+      job_net = job_net_class.new(nil, variables, Context.new)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]

data/lib/rukawa/context.rb

@@ -0,0 +1,16 @@
+module Rukawa
+  class Context
+    attr_reader :store, :executor, :semaphore
+
+    def initialize(currency = nil)
+      @store = Concurrent::Hash.new
+      @executor = Concurrent::FixedThreadPool.new(currency || Rukawa.config.concurrency)
+      @executor.auto_terminate = true
+      @semaphore = Concurrent::Semaphore.new(currency || Rukawa.config.concurrency)
+    end
+
+    def shutdown
+      @executor.shutdown if @executor.running?
+    end
+  end
+end

data/lib/rukawa/dag.rb

@@ -14,11 +14,11 @@ module Rukawa
       @edges = Set.new
     end
 
-    def build(job_net, variables, dependencies)
+    def build(job_net, variables, context, dependencies)
       deps = tsortable_hash(dependencies).tsort
 
       deps.each do |job_class|
-        job = job_class.new(job_net, variables)
+        job = job_class.new(job_net, variables, context)
         @nodes << job
         @jobs << job if job.is_a?(Job)
 

data/lib/rukawa/job.rb

@@ -86,9 +86,10 @@ module Rukawa
       set_state(:finished)
     end
 
-    def initialize(parent_job_net, variables)
+    def initialize(parent_job_net, variables, context)
       @parent_job_net = parent_job_net
       @variables = variables
+      @context = context
       @in_comings = Set.new
       @out_goings = Set.new
       @retry_count = 0
@@ -112,7 +113,7 @@ module Rukawa
       return @dataflow if @dataflow
       return @dataflow = bypass_dataflow if @state.bypassed?
 
-      @dataflow = Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
+      @dataflow = Concurrent.dataflow_with(@context.executor, *depend_dataflows) do |*results|
         do_run(*results)
         @state
       end
@@ -162,7 +163,7 @@ module Rukawa
     end
 
     def bypass_dataflow
-      Concurrent.dataflow_with(Rukawa.executor, *depend_dataflows) do |*results|
+      Concurrent.dataflow_with(@context.executor, *depend_dataflows) do
         Rukawa.logger.info("Skip #{self.class}")
         @state
       end
@@ -209,8 +210,8 @@ module Rukawa
     end
 
     def store(key, value)
-      Rukawa.store[self.class] ||= Concurrent::Hash.new
-      Rukawa.store[self.class][key] = value
+      @context.store[self.class] ||= Concurrent::Hash.new
+      @context.store[self.class][key] = value
     end
 
     def resource_count
@@ -218,10 +219,10 @@ module Rukawa
     end
 
     def acquire_resource
-      Rukawa.semaphore.acquire(resource_count)
+      @context.semaphore.acquire(resource_count)
       yield
     ensure
-      Rukawa.semaphore.release(resource_count)
+      @context.semaphore.release(resource_count)
     end
   end
 end

data/lib/rukawa/job_net.rb

@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
 module Rukawa
   class JobNet < AbstractJob
     include Enumerable
-    attr_reader :dag
+    attr_reader :dag, :context
 
     class << self
       def dependencies
@@ -11,10 +11,11 @@ module Rukawa
       end
     end
 
-    def initialize(parent_job_net, variables, *resume_job_classes)
+    def initialize(parent_job_net, variables, context, *resume_job_classes)
       @parent_job_net = parent_job_net
+      @context = context
       @dag = Dag.new
-      @dag.build(self, variables, self.class.dependencies)
+      @dag.build(self, variables, context, self.class.dependencies)
       @resume_job_classes = resume_job_classes
 
       unless resume_job_classes.empty?
@@ -32,6 +33,31 @@ module Rukawa
       end
     end
 
+    def execute
+      dataflows.each(&:execute)
+    end
+
+    def run(wait_interval = 1)
+      promise = Concurrent::Promise.new do
+        futures = execute
+        until futures.all?(&:complete?)
+          yield self if block_given?
+          sleep wait_interval
+        end
+        errors = futures.map(&:reason).compact
+
+        unless errors.empty?
+          errors.each do |err|
+            next if err.is_a?(DependencyUnsatisfied)
+            Rukawa.logger.error(err)
+          end
+        end
+
+        futures
+      end
+      promise.execute
+    end
+
     def started_at
       @dag.nodes.min_by { |j| j.started_at ? j.started_at.to_i : Float::INFINITY }.started_at
     end

data/lib/rukawa/overview.rb

@@ -6,7 +6,7 @@ module Rukawa
         header << "Dependencies" if with_jobs
         table = Terminal::Table.new headings: header do |t|
           JobNet.descendants.each do |job_net|
-            list_table_row(t, job_net, job_net.dependencies, with_jobs: with_jobs)
+            list_table_row(t, job_net, with_jobs: with_jobs)
           end
         end
         puts table
@@ -23,7 +23,7 @@ module Rukawa
         puts table
       end
 
-      def list_table_row(table, job_net, level = 0, with_jobs: false)
+      def list_table_row(table, job_net, with_jobs: false)
         row = [Paint[job_net.name, :bold, :underline], job_net.desc]
         row << "" if with_jobs
         table << row

data/lib/rukawa/runner.rb

@@ -12,33 +12,28 @@ module Rukawa
 
     def initialize(root_job_net)
       @root_job_net = root_job_net
-      @errors = []
     end
 
     def run(batch_mode = false, refresh_interval = DEFAULT_REFRESH_INTERVAL)
-      Rukawa.init
-      Rukawa.logger.info("=== Start Rukawa ===")
-      futures = @root_job_net.dataflows.each(&:execute)
-      until futures.all?(&:complete?)
-        Overview.display_running_status(@root_job_net) unless batch_mode
-        sleep refresh_interval
+      displayed_at = Time.at(0)
+      promise = @root_job_net.run do
+        unless batch_mode
+          if Time.now - displayed_at >= refresh_interval
+            displayed_at = Time.now
+            Overview.display_running_status(@root_job_net)
+          end
+        end
       end
-      Rukawa.logger.info("=== Finish Rukawa ===")
+      futures = promise.value
 
       Overview.display_running_status(@root_job_net) unless batch_mode
       puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
 
-      errors = futures.map(&:reason).compact
-
-      unless errors.empty?
-        errors.each do |err|
-          next if err.is_a?(DependencyUnsatisfied)
-          Rukawa.logger.error(err)
-        end
-        return false
+      if futures.all?(&:fulfilled?)
+        true
+      else
+        false
       end
-
-      true
     end
   end
 end

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.5.1"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-06-18 00:00:00.000000000 Z
+date: 2016-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -171,6 +171,7 @@ files:
 - lib/rukawa/abstract_job.rb
 - lib/rukawa/cli.rb
 - lib/rukawa/configuration.rb
+- lib/rukawa/context.rb
 - lib/rukawa/dag.rb
 - lib/rukawa/dependency.rb
 - lib/rukawa/errors.rb
@@ -208,7 +209,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Hyper simple job workflow engine