rukawa 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a662fb54093353cccddf1ea735d9599c9ab55868
-  data.tar.gz: 1f860deaedc962f72de32f02b0760994aa5c41ba
+  metadata.gz: 13b76e6a6b3bab23573723fee0d2c9049d1b5052
+  data.tar.gz: f39e1766871bd26e710edd2868edf39c625f1a0f
 SHA512:
-  metadata.gz: c9945a19a8990cc809d96e3c4f5ada00a8b56924234627250f7bfdcfe6562eb4d997240eb7aabe34434ce80abb47af0486ceb4c7e383d2a68652d2899218b8a3
-  data.tar.gz: b1e02a5e7d17be36a7a658674834e89c9bf626948cb1a181e73d19a04c129cf58e3a19d347c56014a490abc4b665e187e19fc45988e8e80ffe1a327e792a6f58
+  metadata.gz: 9037f65f1a2c502a52cd53b3abc2758f2dcbb9deb43595f67687325c09924a298da2a7b79505a34bed2638128a278a8c5997efe07c4c4ee1884fb8f70620e19c
+  data.tar.gz: 42bb250a6a17de86a9d23f2343352d26426185a176313fdef7befcd4117aa4bac846be2d93393fac3e7970d1d18b115c2dadd87936a0512cb59ace0675b91d28

data/README.md

@@ -227,6 +227,21 @@ Main usage is manual reentering.
 % dot -Tpng -o SampleJobNet.png SampleJobNet.dot
 ```
 
+### Config Example
+
+```
+# rukawa.rb
+
+Rukawa.configure do |c|
+  c.logger = OtherLogger.new
+  c.concurrency = 4
+  c.graph.concentrate = true
+  c.graph.nodesep = 0.8
+end
+```
+
+see. [Rukawa::Configuration](https://github.com/joker1007/rukawa/blob/master/lib/rukawa/configuration.rb)
+
 ### help
 ```
 % bundle exec rukawa help run

data/lib/rukawa/abstract_job.rb

@@ -1,18 +1,28 @@
 require 'set'
 require 'rukawa/state'
+require 'active_support/core_ext/class'
 
 module Rukawa
   class AbstractJob
     attr_reader :parent_job_net
+    extend ActiveSupport::DescendantsTracker
 
+    class_attribute :skip_rules, instance_writer: false
+    self.skip_rules = []
     class << self
-      def skip_rules
-        @skip_rules ||= []
+      def add_skip_rule(callable_or_symbol)
+        self.skip_rules = skip_rules + [callable_or_symbol]
       end
 
-      def add_skip_rule(callable_or_symbol)
-        skip_rules.push(callable_or_symbol)
+      def description
+        @description
+      end
+      alias :desc :description
+
+      def set_description(body)
+        @description = body
       end
+      alias :set_desc :set_description
     end
 
     def name
@@ -30,10 +40,6 @@ module Rukawa
       end
     end
 
-    def skip_rules
-      self.class.skip_rules
-    end
-
     def elapsed_time_from(time = Time.now)
       return finished_at - started_at if started_at && finished_at
       return time - started_at if started_at
@@ -46,11 +52,11 @@ module Rukawa
       return "N/A" unless elapsed
 
       hour = elapsed.to_i / 3600
-      min = elapsed.to_i / 60
+      min = (elapsed - hour * 3600).to_i / 60
       sec = (elapsed - hour * 3600 - min * 60).to_i
 
-      hour_format = min > 0 ? "%dh " % hour : ""
-      min_format = min > 0 ? "%dm " % min : ""
+      hour_format = hour > 0 ? "%dh " % hour : ""
+      min_format = hour > 0 || min > 0 ? "%dm " % min : ""
       sec_format = "#{sec}s"
       "#{hour_format}#{min_format}#{sec_format}"
     end

data/lib/rukawa/cli.rb

@@ -1,5 +1,6 @@
 require 'thor'
 require 'rukawa/runner'
+require 'rukawa/overview'
 
 module Rukawa
   class Cli < Thor
@@ -10,20 +11,19 @@ module Rukawa
     method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
     method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
     method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
-    method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
+    method_option :log, aliases: "-l", type: :string, desc: "Default: ./rukawa.log"
     method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
+    method_option :syslog, type: :boolean, default: false, desc: "Output log to syslog"
     method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
     method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
     def _run(job_net_name, *job_name)
       load_config
-      Rukawa.configure do |c|
-        c.log_file = options[:stdout] ? $stdout : options[:log]
-        c.concurrency = options[:concurrency] if options[:concurrency]
-      end
+      set_logger
+      set_concurrency
       load_job_definitions
 
-      job_net_class = Object.const_get(job_net_name)
-      job_classes = job_name.map { |name| Object.const_get(name) }
+      job_net_class = get_class(job_net_name)
+      job_classes = job_name.map { |name| get_class(name) }
       job_net = job_net_class.new(nil, *job_classes)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
@@ -42,8 +42,8 @@ module Rukawa
       load_config
       load_job_definitions
 
-      job_net_class = Object.const_get(job_net_name)
-      job_classes = job_name.map { |name| Object.const_get(name) }
+      job_net_class = get_class(job_net_name)
+      job_classes = job_name.map { |name| get_class(name) }
       job_net = job_net_class.new(nil, *job_classes)
       job_net.output_dot(options[:output])
     end
@@ -60,13 +60,11 @@ module Rukawa
     method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
     def run_job(*job_name)
       load_config
-      Rukawa.configure do |c|
-        c.log_file = options[:stdout] ? $stdout : options[:log]
-        c.concurrency = options[:concurrency] if options[:concurrency]
-      end
+      set_logger
+      set_concurrency
       load_job_definitions
 
-      job_classes = job_name.map { |name| Object.const_get(name) }
+      job_classes = job_name.map { |name| get_class(name) }
       job_net_class = anonymous_job_net_class(*job_classes)
       job_net = job_net_class.new(nil)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
@@ -78,6 +76,25 @@ module Rukawa
       exit 1 unless result
     end
 
+    desc "list", "List JobNet"
+    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
+    method_option :jobs, aliases: "-j", type: :boolean, desc: "Show jobs", default: false
+    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
+    def list
+      load_config
+      load_job_definitions
+      Rukawa::Overview.list_job_net(with_jobs: options[:jobs])
+    end
+
+    desc "list_job", "List Job"
+    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
+    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
+    def list_job
+      load_config
+      load_job_definitions
+      Rukawa::Overview.list_job
+    end
+
     private
 
     def load_config
@@ -88,6 +105,27 @@ module Rukawa
       end
     end
 
+    def set_logger
+      Rukawa.configure do |c|
+        if options[:stdout]
+          c.logger = Logger.new($stdout)
+        elsif options[:syslog]
+          require 'syslog/logger'
+          c.logger = Syslog::Logger.new('rukawa')
+        elsif options[:log]
+          c.logger = Logger.new(options[:log])
+        else
+          c.logger ||= Logger.new('./rukawa.log');
+        end
+      end
+    end
+
+    def set_concurrency
+      Rukawa.configure do |c|
+        c.concurrency = options[:concurrency] if options[:concurrency]
+      end
+    end
+
     def default_config_file
       "./rukawa.rb"
     end
@@ -103,6 +141,13 @@ module Rukawa
       end
     end
 
+    def get_class(name)
+      Object.const_get(name)
+    rescue NameError
+      $stderr.puts("`#{name}` class is not found")
+      exit 1
+    end
+
     def anonymous_job_net_class(*job_classes)
       Class.new(JobNet) do
         self.singleton_class.send(:define_method, :dependencies) do

data/lib/rukawa/configuration.rb

@@ -6,9 +6,12 @@ require 'concurrent'
 module Rukawa
   class Configuration < Delegator
     include Singleton
+    attr_accessor :logger
 
     def initialize
-      @config = OpenStruct.new(log_file: "./rukawa.log", concurrency: Concurrent.processor_count)
+      @config = OpenStruct.new(
+        concurrency: Concurrent.processor_count
+      )
       @config.graph = GraphConfig.new.tap { |c| c.rankdir = "LR" }
     end
 

data/lib/rukawa/dependency.rb

@@ -0,0 +1,59 @@
+module Rukawa
+  module Dependency
+    def self.get(name)
+      const_get(name.to_s.split("_").map(&:capitalize).join)
+    end
+
+    class Base
+      def initialize(*results)
+        @results = results
+      end
+
+      def resolve
+        raise NotImplementedError
+      end
+    end
+
+    class AllSuccess < Base
+      def resolve
+        @results.all? { |r| r && r.success? }
+      end
+    end
+
+    class AllDone < Base
+      def resolve
+        true
+      end
+    end
+
+    class OneSuccess < Base
+      def resolve
+        @results.empty? || @results.any? { |r| r && r.success? }
+      end
+    end
+
+    class AllSuccessOrSkipped < Base
+      def resolve
+        @results.all? { |r| r && (r.success? || r.skipped?) }
+      end
+    end
+
+    class OneSuccessOrSkipped < Base
+      def resolve
+        @results.empty? || @results.any? { |r| r && (r.success? || r.skipped?) }
+      end
+    end
+
+    class AllFailed < Base
+      def resolve
+        @results.none? { |r| r }
+      end
+    end
+
+    class OneFailed < Base
+      def resolve
+        @results.empty? || @results.any? { |r| r.nil? }
+      end
+    end
+  end
+end

data/lib/rukawa/errors.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  class DependentJobFailure < StandardError; end
+  class DependencyUnsatisfied < StandardError; end
 end

data/lib/rukawa/job.rb

@@ -1,15 +1,37 @@
 require 'concurrent'
 require 'rukawa/abstract_job'
+require 'rukawa/dependency'
+require 'rukawa/state'
+require 'active_support/core_ext/class'
 
 module Rukawa
   class Job < AbstractJob
     attr_accessor :in_comings, :out_goings
     attr_reader :state, :started_at, :finished_at
 
+    class_attribute :retryable, :retry_limit, :retry_exception_type, :retry_wait, instance_writer: false
+    class_attribute :dependency_type, instance_writer: false
+    self.dependency_type = Dependency::AllSuccess
+
+    class << self
+      def set_retryable(limit: 8, type: nil, wait: nil)
+        self.retryable = true
+        self.retry_limit = limit
+        self.retry_exception_type = type
+        self.retry_wait = wait
+      end
+
+      def set_dependency_type(name)
+        self.dependency_type = Rukawa::Dependency.get(name)
+      end
+    end
+
     def initialize(parent_job_net)
       @parent_job_net = parent_job_net
       @in_comings = Set.new
       @out_goings = Set.new
+      @retry_count = 0
+      @retry_wait = 1
       set_state(:waiting)
     end
 
@@ -42,7 +64,7 @@ module Rukawa
       @started_at = Time.now
       check_dependencies(results)
 
-      if skip? || results.any?(&:skipped?)
+      if skip?
         Rukawa.logger.info("Skip #{self.class}")
         set_state(:skipped)
       else
@@ -54,7 +76,8 @@ module Rukawa
       end
     rescue => e
       handle_error(e)
-      raise
+      Rukawa.logger.error("Retry #{self.class}")
+      retry
     ensure
       @finished_at = Time.now
     end
@@ -85,16 +108,44 @@ module Rukawa
       end
     end
 
+    def dependency_type
+      self.class.dependency_type
+    end
+
     def check_dependencies(results)
-      unless results.all? { |r| !r.nil? }
+      dependency = dependency_type.new(*results)
+      unless dependency.resolve
         set_state(:aborted)
-        raise DependentJobFailure
+        raise DependencyUnsatisfied
       end
     end
 
     def handle_error(e)
       Rukawa.logger.error("Error #{self.class} by #{e}")
-      set_state(:error) unless e.is_a?(DependentJobFailure)
+      if retry?(e)
+        @retry_count += 1
+        set_state(:waiting)
+        sleep @retry_wait
+        @retry_wait = self.class.retry_wait ? self.class.retry_wait : @retry_wait * 2
+      else
+        set_state(:error) unless e.is_a?(DependencyUnsatisfied)
+        raise e
+      end
+    end
+
+    def retry?(e)
+      return false unless self.class.retryable
+
+      type_condition = case self.class.retry_exception_type
+      when Array
+        self.class.retry_exception_type.include?(e.class)
+      when Class
+        e.is_a?(self.class.retry_exception_type)
+      when nil
+        !e.is_a?(DependencyUnsatisfied)
+      end
+
+      type_condition && (self.class.retry_limit.nil? || self.class.retry_limit == 0 || @retry_count < self.class.retry_limit)
     end
 
     def store(key, value)

data/lib/rukawa/overview.rb

@@ -0,0 +1,58 @@
+module Rukawa
+  module Overview
+    class << self
+      def list_job_net(with_jobs: false)
+        header = ["Job", "Desc"]
+        header << "Dependencies" if with_jobs
+        table = Terminal::Table.new headings: header do |t|
+          JobNet.descendants.each do |job_net|
+            list_table_row(t, job_net, job_net.dependencies, with_jobs: with_jobs)
+          end
+        end
+        puts table
+      end
+
+      def list_job
+        header = ["Job", "Desc"]
+        table = Terminal::Table.new headings: header do |t|
+          Job.descendants.each do |job|
+            row = [Paint[job.name, :bold, :underline], job.desc]
+            t << row
+          end
+        end
+        puts table
+      end
+
+      def list_table_row(table, job_net, level = 0, with_jobs: false)
+        row = [Paint[job_net.name, :bold, :underline], job_net.desc]
+        row << "" if with_jobs
+        table << row
+        if with_jobs
+          job_net.dependencies.each do |inner_j, deps|
+            table << [Paint["#{"  "}#{inner_j.name}"], inner_j.desc, deps.join(", ")]
+          end
+        end
+      end
+
+      def display_running_status(root_job_net)
+        table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
+          root_job_net.each_with_index do |j|
+            running_table_row(t, j)
+          end
+        end
+        puts table
+      end
+
+      def running_table_row(table, job, level = 0)
+        if job.is_a?(JobNet)
+          table << [Paint["#{"  " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
+          job.each do |inner_j|
+            running_table_row(table, inner_j, level + 1)
+          end
+        else
+          table << [Paint["#{"  " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
+        end
+      end
+    end
+  end
+end

data/lib/rukawa/runner.rb

@@ -1,5 +1,6 @@
 require 'terminal-table'
 require 'paint'
+require 'rukawa/overview'
 
 module Rukawa
   class Runner
@@ -18,19 +19,19 @@ module Rukawa
       Rukawa.logger.info("=== Start Rukawa ===")
       futures = @root_job_net.dataflows.each(&:execute)
       until futures.all?(&:complete?)
-        display_table unless batch_mode
+        Overview.display_running_status(@root_job_net) unless batch_mode
         sleep refresh_interval
       end
       Rukawa.logger.info("=== Finish Rukawa ===")
 
-      display_table unless batch_mode
+      Overview.display_running_status(@root_job_net) unless batch_mode
       puts "Finished #{@root_job_net.name} in #{@root_job_net.formatted_elapsed_time_from}"
 
       errors = futures.map(&:reason).compact
 
       unless errors.empty?
         errors.each do |err|
-          next if err.is_a?(DependentJobFailure)
+          next if err.is_a?(DependencyUnsatisfied)
           Rukawa.logger.error(err)
         end
         return false
@@ -38,27 +39,5 @@ module Rukawa
 
       true
     end
-
-    private
-
-    def display_table
-      table = Terminal::Table.new headings: ["Job", "Status", "Elapsed Time"] do |t|
-        @root_job_net.each_with_index do |j|
-          table_row(t, j)
-        end
-      end
-      puts table
-    end
-
-    def table_row(table, job, level = 0)
-      if job.is_a?(JobNet)
-        table << [Paint["#{"  " * level}#{job.class}", :bold, :underline], Paint[job.state.colored, :bold, :underline], Paint[job.formatted_elapsed_time_from, :bold, :underline]]
-        job.each do |inner_j|
-          table_row(table, inner_j, level + 1)
-        end
-      else
-        table << [Paint["#{"  " * level}#{job.class}", :bold], job.state.colored, job.formatted_elapsed_time_from]
-      end
-    end
   end
 end

data/lib/rukawa/state.rb

@@ -16,6 +16,10 @@ module Rukawa::State
       other
     end
 
+    def success?
+      false
+    end
+
     %i(running? skipped? bypassed? error? aborted? waiting? finished?).each do |sym|
       define_method(sym) do
         false
@@ -66,6 +70,10 @@ module Rukawa::State
       :yellow
     end
 
+    def self.success?
+      true
+    end
+
     def self.bypassed?
       true
     end
@@ -138,6 +146,10 @@ module Rukawa::State
       :green
     end
 
+    def self.success?
+      true
+    end
+
     def self.finished?
       true
     end

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.3.2"
+  VERSION = "0.3.3"
 end

data/lib/rukawa.rb

@@ -3,7 +3,7 @@ require "concurrent"
 module Rukawa
   class << self
     def logger
-      @logger ||= Logger.new(config.log_file)
+      config.logger
     end
 
     def store
@@ -24,9 +24,11 @@ module Rukawa
   end
 end
 
+require 'active_support'
 require "rukawa/version"
 require 'rukawa/errors'
 require 'rukawa/state'
+require 'rukawa/dependency'
 require 'rukawa/configuration'
 require 'rukawa/job_net'
 require 'rukawa/job'

data/rukawa.gemspec

@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "activesupport", ">= 4"
   spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "thor"
   spec.add_runtime_dependency "terminal-table"
@@ -27,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-power_assert"
+  spec.add_development_dependency "rspec-parameterized"
 end

data/sample/jobs/sample_job.rb

@@ -12,19 +12,29 @@ class SampleJob < Rukawa::Job
 end
 
 class Job1 < SampleJob
+  set_description "Job1 description body"
 end
 class Job2 < SampleJob
+  def run
+    raise "job2 error"
+  end
 end
 class Job3 < SampleJob
 end
 class Job4 < SampleJob
+  # inherited by subclass
+  set_dependency_type :one_success
 end
 class Job5 < SampleJob
+  # inherited by subclass
+  set_retryable limit: 3, wait: 2, type: RuntimeError
+
   def run
     raise "job5 error"
   end
 end
 class Job6 < SampleJob
+  set_dependency_type :one_failed
 end
 class Job7 < SampleJob
 end
@@ -47,6 +57,7 @@ class InnerJob4 < SampleJob
 end
 
 class InnerJob5 < SampleJob
+  # inherited by subclass
   add_skip_rule ->(job) { job.is_a?(SampleJob) }
 end
 

data/sample/result.dot

@@ -2,7 +2,7 @@ digraph "SampleJobNet" {
 label = "SampleJobNet";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 Job1 [style = filled,fillcolor = green];
-Job2 [style = filled,fillcolor = green];
+Job2 [style = filled,fillcolor = red];
 Job3 [style = filled,fillcolor = green];
 Job4 [style = filled,fillcolor = green];
 subgraph "cluster_InnerJobNet" {
@@ -17,15 +17,15 @@ InnerJob2 [style = filled,fillcolor = red];
 }
 Job8 [style = filled,fillcolor = magenta];
 Job5 [style = filled,fillcolor = red];
-Job6 [style = filled,fillcolor = magenta];
-Job7 [style = filled,fillcolor = magenta];
+Job6 [style = filled,fillcolor = green];
+Job7 [style = filled,fillcolor = green];
 subgraph "cluster_InnerJobNet2" {
 label = "InnerJobNet2";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
 InnerJob4 [style = filled,fillcolor = green];
 InnerJob5 [style = filled,fillcolor = yellow];
-InnerJob6 [style = filled,fillcolor = yellow];
+InnerJob6 [style = filled,fillcolor = magenta];
 "InnerJob4" -> "InnerJob5";
 "InnerJob4" -> "InnerJob6";
 "InnerJob5" -> "InnerJob6";

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-25 00:00:00.000000000 Z
+date: 2016-04-10 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -122,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec-parameterized
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Hyper simple job workflow engine
 email:
 - kakyoin.hierophant@gmail.com
@@ -144,9 +172,11 @@ files:
 - lib/rukawa/cli.rb
 - lib/rukawa/configuration.rb
 - lib/rukawa/dag.rb
+- lib/rukawa/dependency.rb
 - lib/rukawa/errors.rb
 - lib/rukawa/job.rb
 - lib/rukawa/job_net.rb
+- lib/rukawa/overview.rb
 - lib/rukawa/runner.rb
 - lib/rukawa/state.rb
 - lib/rukawa/version.rb