rukawa 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0345bf03c443cf5a22b3cde1e5397ccb86aa16ee
-  data.tar.gz: 317a36ce8f9664ef86fee7b8928e05f3cc101664
+  metadata.gz: 999a3ff79a503ecea000dd2bed167c22a449f8cf
+  data.tar.gz: 4fc79536e34bbe56be38e7761634ffaff46fa388
 SHA512:
-  metadata.gz: 8a0bda03e6575c683811947f1427390408125c0e397bd33c13325c0678a317219686e5d45110f24df666da96db3efca48737c66e6835f217b55c9e9af2857444
-  data.tar.gz: f68ec7053149239557a6f3dea6e4c3d0f72f75620a79a3f7d77dbe69c1ea9d83bba9c4490dd477db0ebc4cfdcc4325f20dd00b5ab03be73d905f961b5484288e
+  metadata.gz: d5cf437492c3461a218b817834fbd0d50cdcb613b06a587b5770458acd18fc0945f1640cd19447eacae2357003a1e371a605f77f7646c6032596b4ed7fdcc10e
+  data.tar.gz: b0c0649300511b87a1afe1d9ba04fbb2084169c718fdef5ef0f322333a7881204ff082717b871d495fd34f1f0da123c165d85213bd2f9dc29dc018dc169638ce

data/lib/rukawa/cli.rb

@@ -3,7 +3,7 @@ require 'rukawa/runner'
 
 module Rukawa
   class Cli < Thor
-    desc "run JOB_NET_NAME", "Run jobnet"
+    desc "run JOB_NET_NAME [JOB_NAME] [JOB_NAME] ...", "Run jobnet. If JOB_NET is set, resume from JOB_NAME"
     map "run" => "_run"
     method_option :concurrency, aliases: "-c", type: :numeric, default: nil, desc: "Default: cpu count"
     method_option :variables, type: :hash, default: {}
@@ -14,7 +14,7 @@ module Rukawa
     method_option :stdout, type: :boolean, default: false, desc: "Output log to stdout"
     method_option :dot, aliases: "-d", type: :string, default: nil, desc: "Output job status by dot format"
     method_option :refresh_interval, aliases: "-r", type: :numeric, default: 3, desc: "Refresh interval for running status information"
-    def _run(job_net_name)
+    def _run(job_net_name, *job_name)
       load_config
       Rukawa.configure do |c|
         c.log_file = options[:stdout] ? $stdout : options[:log]
@@ -23,7 +23,8 @@ module Rukawa
       load_job_definitions
 
       job_net_class = Object.const_get(job_net_name)
-      job_net = job_net_class.new(nil, options[:variables])
+      job_classes = job_name.map { |name| Object.const_get(name) }
+      job_net = job_net_class.new(nil, *job_classes)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]

data/lib/rukawa/dag.rb

@@ -4,21 +4,28 @@ module Rukawa
   class Dag
     include Enumerable
 
-    attr_reader :jobs, :edges
+    attr_reader :nodes, :jobs, :edges
 
-    def initialize(job_net, dependencies)
-      deps = tsortable_hash(dependencies).tsort
+    def initialize
+      @nodes = Set.new
       @jobs = Set.new
       @edges = Set.new
+    end
+
+    def build(job_net, dependencies)
+      deps = tsortable_hash(dependencies).tsort
 
       deps.each do |job_class|
         job = job_class.new(job_net)
-        @jobs << job
+        @nodes << job
+        @jobs << job if job.is_a?(Job)
 
         dependencies[job_class].each do |depend_job_class|
-          depend_job = @jobs.find { |j| j.instance_of?(depend_job_class) }
+          depend_job = @nodes.find { |j| j.instance_of?(depend_job_class) }
 
-          depend_job.nodes_as_from.product(job.nodes_as_to).each do |from, to|
+          depend_job.jobs_as_from.product(job.jobs_as_to).each do |from, to|
+            @jobs << from
+            @jobs << to
             edge = Edge.new(from, to, job_net)
             @edges << edge
             from.out_goings << edge
@@ -26,24 +33,20 @@ module Rukawa
           end
         end
       end
+
+      if job_net.parent_job_net
+        job_net.parent_job_net.dag.jobs.merge(@jobs)
+      end
     end
 
     def each
       if block_given?
-        @jobs.each { |j| yield j }
+        @nodes.each { |j| yield j }
       else
-        @jobs.each
+        @nodes.each
       end
     end
 
-    def roots
-      select(&:root?)
-    end
-
-    def leaves
-      select(&:leaf?)
-    end
-
     private
 
     def tsortable_hash(hash)

data/lib/rukawa/job.rb

@@ -4,21 +4,21 @@ require 'rukawa/abstract_job'
 module Rukawa
   class Job < AbstractJob
     attr_accessor :in_comings, :out_goings
-    attr_reader :state
+    attr_reader :parent_job_net, :state
 
-    def initialize(job_net, variables = {})
-      @job_net = job_net
+    def initialize(parent_job_net)
+      @parent_job_net = parent_job_net
       @in_comings = Set.new
       @out_goings = Set.new
       set_state(:waiting)
     end
 
     def root?
-      in_comings.select { |edge| edge.cluster == @job_net }.empty?
+      in_comings.select { |edge| edge.cluster == @parent_job_net }.empty?
     end
 
     def leaf?
-      out_goings.select { |edge| edge.cluster == @job_net }.empty?
+      out_goings.select { |edge| edge.cluster == @parent_job_net }.empty?
     end
 
     def dataflow
@@ -28,7 +28,7 @@ module Rukawa
         begin
           raise DependentJobFailure unless results.all? { |r| !r.nil? }
 
-          if skip? || @job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
+          if skip? || @parent_job_net.skip? || results.any? { |r| r == Rukawa::State.get(:skipped) }
             Rukawa.logger.info("Skip #{self.class}")
             set_state(:skipped)
           else
@@ -51,10 +51,10 @@ module Rukawa
     def run
     end
 
-    def nodes_as_from
+    def jobs_as_from
       [self]
     end
-    alias :nodes_as_to :nodes_as_from
+    alias :jobs_as_to :jobs_as_from
 
     def to_dot_def
       if state == Rukawa::State::Waiting

data/lib/rukawa/job_net.rb

@@ -3,7 +3,7 @@ require 'rukawa/abstract_job'
 module Rukawa
   class JobNet < AbstractJob
     include Enumerable
-    attr_reader :dag
+    attr_reader :parent_job_net, :dag
 
     class << self
       def dependencies
@@ -11,9 +11,18 @@ module Rukawa
       end
     end
 
-    def initialize(_job_net, variables = {})
-      @variables = variables
-      @dag = Dag.new(self, self.class.dependencies)
+    def initialize(parent_job_net, *resume_job_classes)
+      @parent_job_net = parent_job_net
+      @dag = Dag.new
+      @dag.build(self, self.class.dependencies)
+    end
+
+    def toplevel?
+      @parent_job_net.nil?
+    end
+
+    def subgraph?
+      !toplevel?
     end
 
     def dataflows
@@ -36,14 +45,6 @@ module Rukawa
       File.open(filename, 'w') { |f| f.write(to_dot) }
     end
 
-    def nodes_as_from
-      leaves
-    end
-
-    def nodes_as_to
-      roots
-    end
-
     def to_dot(subgraph = false)
       graphdef = subgraph ? "subgraph" : "digraph"
       buf = %Q|#{graphdef} "#{subgraph ? "cluster_" : ""}#{name}" {\n|
@@ -65,12 +66,12 @@ module Rukawa
       to_dot(true)
     end
 
-    def roots
-      @dag.roots
+    def jobs_as_to
+      @dag.jobs.select { |j| j.in_comings.select { |edge| edge.cluster == self }.empty? && j.root? }
     end
 
-    def leaves
-      @dag.leaves
+    def jobs_as_from
+      @dag.jobs.select { |j| j.out_goings.select { |edge| edge.cluster == self }.empty? && j.leaf? }
     end
 
     def each(&block)

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

data/sample/job_nets/sample_job_net.rb

@@ -42,6 +42,18 @@ class InnerJobNet4 < Rukawa::JobNet
         InnerJob11 => [],
         InnerJob12 => [],
         InnerJob13 => [],
+        NestedJobNet => [InnerJob11, InnerJob12],
+      }
+    end
+  end
+end
+
+class NestedJobNet < Rukawa::JobNet
+  class << self
+    def dependencies
+      {
+        NestedJob1 => [],
+        NestedJob2 => [NestedJob1],
       }
     end
   end

data/sample/jobs/sample_job.rb

@@ -73,3 +73,9 @@ end
 
 class InnerJob13 < SampleJob
 end
+
+class NestedJob1 < SampleJob
+end
+
+class NestedJob2 < SampleJob
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - joker1007