rukawa 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef694d2105bd3b75de436eede093ba3319bec085
-  data.tar.gz: ba02c37e5fa934b2eff3ae26079f3a5f6819bae5
+  metadata.gz: e90c29a3fb28cf675e9c34e2be8a10e642819f19
+  data.tar.gz: f967d48f8faced73d118c7c752e3f1c28cb062fd
 SHA512:
-  metadata.gz: c028cc8c8d6dd8caf3341f5d637034a17a56e2e6a61775d32003b385bc3484586081cb18629cec53f7ca1ed653b056cc1ceca830c52c5f4adb17f5f8fccb628f
-  data.tar.gz: 6414396c4ceee37a8d2f8e0eea69867f30197ab826ce1f1ea1329ab666e44a31ed573c3e5df86a5b232d27157500ccb77cfa22b32ca15c38e5e45cec2eec4f9c
+  metadata.gz: c98fbb5ebf627f6133337c4d88e88556ce199eec049a65ca610f1faa0c74f932461086cbd0687d18a28fd52378c6cb9d2d4a15522423e113d7c1a90318c5f0a5
+  data.tar.gz: 1c763aa161305e1d6db416be8f8237d40c4f432e9e3b8edefaad3002e179eece6da0bbc118913f1261ba4425e25cbb5ec5d77484ce16a588a6e71291f6489f27

data/README.md

@@ -227,6 +227,41 @@ Main usage is manual reentering.
 % dot -Tpng -o SampleJobNet.png SampleJobNet.dot
 ```
 
+### Use variables
+
+```ruby
+class Job < Rukawa::Job
+  def run
+    # access via `variables` method
+    # return freezed Hash object
+    puts variables["var1"]
+    puts variables["var2"]
+  end
+end
+```
+
+```sh
+% bundle exec rukawa run SampleJobNet --var var1:value1 var2:value2
+# or
+% bundle exec rukawa run SampleJobNet --varfile variables.yml
+# or
+% bundle exec rukawa run SampleJobNet --varfile variables.json
+```
+
+```yml
+# variables.yml
+var1: value1
+var2: value2
+```
+
+```javascript
+// variables.json
+{
+  "var1": "value1",
+  "var2": "value2"
+}
+```
+
 ### Config Example
 
 ```
@@ -249,17 +284,19 @@ Usage:
   rukawa run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...
 
 Options:
-  -c, [--concurrency=N]           # Default: cpu count
-      [--variables=key:value]
-      [--config=CONFIG]           # If this options is not set, try to load ./rukawa.rb
-      [--job-dirs=one two three]  # Load job directories
-  -b, [--batch], [--no-batch]     # If batch mode, not display running status
-  -l, [--log=LOG]
-                                  # Default: ./rukawa.log
-      [--stdout], [--no-stdout]   # Output log to stdout
-  -d, [--dot=DOT]                 # Output job status by dot format
-  -r, [--refresh-interval=N]      # Refresh interval for running status information
-                                  # Default: 3
+      [--config=CONFIG]                     # If this options is not set, try to load ./rukawa.rb
+      [--job-dirs=JOB_DIR1 JOB_DIR2]        # Load job directories
+  -c, [--concurrency=N]                     # Default: cpu count
+  --var, [--variables=KEY:VALUE KEY:VALUE]
+      [--varfile=VARFILE]                   # variable definition file. ex (variables.yml, variables.json)
+  -b, [--batch], [--no-batch]               # If batch mode, not display running status
+  -l, [--log=LOG]                           # Default: ./rukawa.log
+      [--stdout], [--no-stdout]             # Output log to stdout
+      [--syslog], [--no-syslog]             # Output log to syslog
+  -d, [--dot=DOT]                           # Output job status by dot format
+  -f, [--format=FORMAT]                     # Output job status format: png, svg, pdf, ... etc
+  -r, [--refresh-interval=N]                # Refresh interval for running status information
+                                            # Default: 3
 
 Run jobnet. If JOB_NET is set, resume from JOB_NAME
 
@@ -269,17 +306,19 @@ Usage:
   rukawa run_job JOB_NAME [JOB_NAME] ...
 
 Options:
-  -c, [--concurrency=N]           # Default: cpu count
-      [--variables=key:value]
-      [--config=CONFIG]           # If this options is not set, try to load ./rukawa.rb
-      [--job-dirs=one two three]  # Load job directories
-  -b, [--batch], [--no-batch]     # If batch mode, not display running status
-  -l, [--log=LOG]
-                                  # Default: ./rukawa.log
-      [--stdout], [--no-stdout]   # Output log to stdout
-  -d, [--dot=DOT]                 # Output job status by dot format
-  -r, [--refresh-interval=N]      # Refresh interval for running status information
-                                  # Default: 3
+      [--config=CONFIG]                     # If this options is not set, try to load ./rukawa.rb
+      [--job-dirs=JOB_DIR1 JOB_DIR2]        # Load job directories
+  -c, [--concurrency=N]                     # Default: cpu count
+  --var, [--variables=KEY:VALUE KEY:VALUE]
+      [--varfile=VARFILE]                   # variable definition file. ex (variables.yml, variables.json)
+  -b, [--batch], [--no-batch]               # If batch mode, not display running status
+  -l, [--log=LOG]                           # Default: ./rukawa.log
+      [--stdout], [--no-stdout]             # Output log to stdout
+      [--syslog], [--no-syslog]             # Output log to syslog
+  -d, [--dot=DOT]                           # Output job status by dot format
+  -f, [--format=FORMAT]                     # Output job status format: png, svg, pdf, ... etc
+  -r, [--refresh-interval=N]                # Refresh interval for running status information
+                                            # Default: 3
 
 Run specific jobs.
 

data/lib/rukawa/cli.rb

@@ -4,20 +4,29 @@ require 'rukawa/overview'
 
 module Rukawa
   class Cli < Thor
+
+    def self.base_options
+      method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
+      method_option :job_dirs, type: :array, default: [], desc: "Load job directories", banner: "JOB_DIR1 JOB_DIR2"
+    end
+
+    def self.run_options
+      method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
+      method_option :variables, aliases: "--var", type: :hash, default: {}, banner: "KEY:VALUE KEY:VALUE"
+      method_option :varfile, type: :string, default: nil, desc: "variable definition file. ex (variables.yml, variables.json)"
+      method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
+      method_option :log, aliases: "-l", type: :string, desc: "Default: ./rukawa.log"
+      method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
+      method_option :syslog, type: :boolean, default: false, desc: "Output log to syslog"
+      method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
+      method_option :format, aliases: "-f", type: :string, desc: "Output job status format: png, svg, pdf, ... etc"
+      method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
+    end
+
     desc "run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Run jobnet. If JOB_NET is set, resume from JOB_NAME"
     map "run" => "_run"
-    map %w[version -v] => "__print_version"
-    method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
-    method_option :variables, type: :hash, default: {}
-    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
-    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
-    method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
-    method_option :log, aliases: "-l", type: :string, desc: "Default: ./rukawa.log"
-    method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
-    method_option :syslog, type: :boolean, default: false, desc: "Output log to syslog"
-    method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
-    method_option :format, aliases: "-f", type: :string, desc: "Output job status format: png, svg, pdf, ... etc"
-    method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
+    base_options
+    run_options
     def _run(job_net_name, *job_name)
       load_config
       set_logger
@@ -26,7 +35,7 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, *job_classes)
+      job_net = job_net_class.new(nil, variables, *job_classes)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]
@@ -37,8 +46,7 @@ module Rukawa
     end
 
     desc "graph JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Output jobnet graph. If JOB_NET is set, simulate resumed job sequence"
-    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
-    method_option :job_dirs, type: :array, default: []
+    base_options
     method_option :output, aliases: "-o", type: :string, required: true
     method_option :format, aliases: "-f", type: :string
     def graph(job_net_name, *job_name)
@@ -47,20 +55,13 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, *job_classes)
+      job_net = job_net_class.new(nil, {}, *job_classes)
       job_net.output_dot(options[:output], format: options[:format])
     end
 
     desc "run_job JOB_NAME [JOB_NAME] ...", "Run specific jobs."
-    method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
-    method_option :variables, type: :hash, default: {}
-    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
-    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
-    method_option :batch, aliases: "-b", type: :boolean, default: false, desc: "If batch mode, not display running status"
-    method_option :log, aliases: "-l", type: :string, default: "./rukawa.log"
-    method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
-    method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
-    method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
+    base_options
+    run_options
     def run_job(*job_name)
       load_config
       set_logger
@@ -69,7 +70,7 @@ module Rukawa
 
       job_classes = job_name.map { |name| get_class(name) }
       job_net_class = anonymous_job_net_class(*job_classes)
-      job_net = job_net_class.new(nil)
+      job_net = job_net_class.new(nil, variables)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]
@@ -80,9 +81,8 @@ module Rukawa
     end
 
     desc "list", "List JobNet"
-    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
+    base_options
     method_option :jobs, aliases: "-j", type: :boolean, desc: "Show jobs", default: false
-    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
     def list
       load_config
       load_job_definitions
@@ -90,14 +90,14 @@ module Rukawa
     end
 
     desc "list_job", "List Job"
-    method_option :config, type: :string, default: nil, desc: "If this options is not set, try to load ./rukawa.rb"
-    method_option :job_dirs, type: :array, default: [], desc: "Load job directories"
+    base_options
     def list_job
       load_config
       load_job_definitions
       Rukawa::Overview.list_job
     end
 
+    map %w[version -v] => "__print_version"
     desc "version(-v)", "Print the version"
     def __print_version
       puts "rukawa #{Rukawa::VERSION}"
@@ -165,5 +165,31 @@ module Rukawa
         define_method(:name) { "AnonymousJobNet" }
       end
     end
+
+    def variables
+      if options[:varfile]
+        read_varfile.freeze
+      else
+        options[:variables].freeze
+      end
+    end
+
+    def read_varfile
+      unless File.exist?(options[:varfile])
+        $stderr.puts("`#{options[:varfile]}` is not found")
+        exit 1
+      end
+
+      extname = File.extname(options[:varfile])
+      if %w(.yml .yaml).include?(extname)
+        require 'yaml'
+        deserializer = ->(data) { YAML.load(data) }
+      elsif %w(.js .json).include?(extname)
+        require 'json'
+        deserializer = ->(data) { JSON.load(data) }
+      end
+
+      deserializer.call(File.read(options[:varfile]))
+    end
   end
 end

data/lib/rukawa/dag.rb

@@ -14,11 +14,11 @@ module Rukawa
       @edges = Set.new
     end
 
-    def build(job_net, dependencies)
+    def build(job_net, variables, dependencies)
       deps = tsortable_hash(dependencies).tsort
 
       deps.each do |job_class|
-        job = job_class.new(job_net)
+        job = job_class.new(job_net, variables)
         @nodes << job
         @jobs << job if job.is_a?(Job)
 

data/lib/rukawa/job.rb

@@ -7,7 +7,7 @@ require 'active_support/core_ext/class'
 module Rukawa
   class Job < AbstractJob
     attr_accessor :in_comings, :out_goings
-    attr_reader :state, :started_at, :finished_at
+    attr_reader :state, :started_at, :finished_at, :variables
 
     class_attribute :retryable, :retry_limit, :retry_exception_type, :retry_wait, instance_writer: false
     class_attribute :dependency_type, instance_writer: false
@@ -26,8 +26,9 @@ module Rukawa
       end
     end
 
-    def initialize(parent_job_net)
+    def initialize(parent_job_net, variables)
       @parent_job_net = parent_job_net
+      @variables = variables
       @in_comings = Set.new
       @out_goings = Set.new
       @retry_count = 0

data/lib/rukawa/job_net.rb

@@ -11,10 +11,10 @@ module Rukawa
       end
     end
 
-    def initialize(parent_job_net, *resume_job_classes)
+    def initialize(parent_job_net, variables, *resume_job_classes)
       @parent_job_net = parent_job_net
       @dag = Dag.new
-      @dag.build(self, self.class.dependencies)
+      @dag.build(self, variables, self.class.dependencies)
       @resume_job_classes = resume_job_classes
 
       unless resume_job_classes.empty?

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.3.4"
+  VERSION = "0.4.0"
 end

data/sample/jobs/sample_job.rb

@@ -13,6 +13,11 @@ end
 
 class Job1 < SampleJob
   set_description "Job1 description body"
+
+  def run
+    p variables unless variables.empty?
+    super
+  end
 end
 class Job2 < SampleJob
   def run

data/sample/variables.json

@@ -0,0 +1 @@
+{"var1": "var1", "var2": 2, "var3": {"var3-1": 1}}

data/sample/variables.yml

@@ -0,0 +1,6 @@
+var1: "var1"
+var2: 2
+var3:
+  - 3
+  - 4
+  - "five"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - joker1007
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-04-12 00:00:00.000000000 Z
+date: 2016-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -187,6 +187,8 @@ files:
 - sample/result.dot
 - sample/result.png
 - sample/rukawa.rb
+- sample/variables.json
+- sample/variables.yml
 homepage: https://github.com/joker1007/rukawa
 licenses: []
 metadata: {}