rukawa 0.7.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 8087ac19883cd6f2ab570ab99494db140cd86670
-  data.tar.gz: 4b86cea0a91ba32c838a7b80fc7b6f7dc2334666
+SHA256:
+  metadata.gz: f433f168b831f2b1c30a55129c711382b2519c5e2de79f669486cff353aa344a
+  data.tar.gz: 28c8226bf65d6f747723650a60786fd966e1e758a5d3145efcde7318e19208ea
 SHA512:
-  metadata.gz: 074ca450e282d2a63198f357df4f0fb1b5c694c04734dea25b23ceb4ca4b8f5e7e5bb90807950ebc2b95c562a08503fa5af8627b61bd37fa266d15aa0223dbb3
-  data.tar.gz: 4847ee20b6163067927180c183ee4f98686f35a8e86375d7035612f04d1dbfdb11ca549dac16c2f6ef2775c1211f48a53f670ded8ba067fd6e59c04d1fa47367
+  metadata.gz: 452a7bfd1726c3992f1788ae457a1208247ccdac4e6e7ce9b79b069c942c8d7a176ea1908ef481479e730d751089b9880ec39eb66b0449f018ab1fac9b6b9bda
+  data.tar.gz: 8a86b7ca28ea37638c0b92bb5747bd55f6ca347683d9285a368898c8a92a569b72b8e2a387a701e4292a1a887302880f4bade96c2808f061f726b0a2ddb8748f

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: test
+
+on:
+  push:
+    branches:
+      - '*'
+
+  pull_request:
+  # schedule:
+  #   cron: '0 0 * * 0' # every Sunday at 00:00 (UTC)
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby:
+          - '3.0'
+          - '3.1'
+          - '3.2'
+    services:
+      redis:
+        image: redis
+        options: >-
+            --health-cmd "redis-cli ping"
+            --health-interval 10s
+            --health-timeout 5s
+            --health-retries 5
+        ports:
+          - 6379:6379
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - run: bundle exec rake spec
+

data/LICENSE.txt

@@ -0,0 +1,7 @@
+Copyright 2016 Tomohiro Hashidate (joker1007)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,7 +1,6 @@
-[![Stories in Ready](https://badge.waffle.io/joker1007/rukawa.png?label=ready&title=Ready)](https://waffle.io/joker1007/rukawa)
 # Rukawa
 [![Gem Version](https://badge.fury.io/rb/rukawa.svg)](https://badge.fury.io/rb/rukawa)
-[![Build Status](https://travis-ci.org/joker1007/rukawa.svg?branch=master)](https://travis-ci.org/joker1007/rukawa)
+[![test](https://github.com/joker1007/rukawa/actions/workflows/test.yml/badge.svg)](https://github.com/joker1007/rukawa/actions/workflows/test.yml)
 [![Code Climate](https://codeclimate.com/github/joker1007/rukawa/badges/gpa.svg)](https://codeclimate.com/github/joker1007/rukawa)
 
 Rukawa = (流川)
@@ -44,7 +43,7 @@ See [sample/job_nets/sample_job_net.rb](https://github.com/joker1007/rukawa/blob
 ```
 % cd rukawa/sample
 
-# load ./jobs/**/*.rb, ./job_net/**/*.rb automatically
+# load ./jobs/**/*.rb, ./job_nets/**/*.rb automatically
 % bundle exec rukawa run SampleJobNet -r 10 -c 10
 +----------------+----------+
 | Job            | Status   |
@@ -331,6 +330,37 @@ end
 
 see. [Rukawa::Configuration](https://github.com/joker1007/rukawa/blob/master/lib/rukawa/configuration.rb)
 
+### ActiveJob Integration
+
+```ruby
+class SampleJobNet < Rukawa::JobNet
+  class << self
+    def dependencies
+      # Generate Wrapper class
+      wrapped1 = Rukawa::Wrapper::ActiveJob[ActiveJobSample1] # named Rukawa::Wrapper::ActiveJobSample1Wrapper
+      wrapped2 = Rukawa::Wrapper::ActiveJob[ActiveJobSample2] # named Rukawa::Wrapper::ActiveJobSample2Wrapper
+      {
+        Job1 => [],
+        wrapped1 => [Job1],
+        wrapped2 => [wrapped1],
+      }
+    end
+  end
+end
+```
+
+And write config to define status store for tracking remote job status"
+
+```ruby
+redis_host = ENV["REDIS_HOST"] || "localhost:6379"
+Rukawa.configure do |c|
+  c.status_store = ActiveSupport::Cache::RedisStore.new(redis_host)
+  c.status_expire_duration = 72 * 60 * 60 # default is 24 hours
+end
+```
+
+__Caution: When rukawa runs wrapper job, base ActiveJob class includes hook modules automatically in order to track job running status.__
+
 ### help
 ```
 % bundle exec rukawa help run
@@ -392,7 +422,7 @@ Output jobnet graph. If JOB_NET is set, simulate resumed job sequence
 
 ```ruby
 currency = 4
-job_net = YourJobNetClass.new(nil, {"var1" => "value1"}, Context.new(currency))
+job_net = YourJobNetClass.new(variables: {"var1" => "value1"})
 promise = job_net.run do
   puts "Job Running"
 end
@@ -417,4 +447,3 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/joker1007/rukawa.
-

data/lib/rukawa/abstract_job.rb

@@ -26,7 +26,7 @@ module Rukawa
     end
 
     def name
-      self.class.to_s
+      self.class.name || self.class.to_s
     end
 
     def inspect

data/lib/rukawa/builtins/base.rb

@@ -0,0 +1,22 @@
+require 'rukawa/job'
+
+module Rukawa
+  module Builtins
+    class Base < ::Rukawa::Job
+      class << self
+        def [](**params)
+          Class.new(self) do
+            handle_parameters(params)
+
+            def self.name
+              super || "#{superclass.name}_#{object_id}"
+            end
+          end
+        end
+
+        def handle_parameters(**params)
+        end
+      end
+    end
+  end
+end

data/lib/rukawa/builtins/embulk.rb

@@ -0,0 +1,60 @@
+require 'rukawa/builtins/shell'
+
+module Rukawa
+  module Builtins
+    class Embulk < Shell
+      class_attribute :config, :embulk_bin, :embulk_bundle, :embulk_vm_options, :jvm_options, :preview
+
+      self.embulk_bin = "embulk"
+      self.embulk_bundle = nil
+      self.embulk_vm_options = []
+      self.jvm_options = []
+      self.preview = false
+
+      class << self
+        def handle_parameters(config:, embulk_bin: nil, embulk_bundle: nil, embulk_vm_options: nil, jvm_options: nil, stdout: nil, stderr: nil, env: nil, chdir: nil, preview: false)
+          self.config = config
+          self.embulk_bin = embulk_bin if embulk_bin
+          self.embulk_bundle = embulk_bundle if embulk_bundle
+          self.embulk_vm_options = embulk_vm_options if embulk_vm_options
+          self.jvm_options = jvm_options if jvm_options
+          self.stdout = stdout if stdout
+          self.stderr = stderr if stderr
+          self.env = env if env
+          self.chdir = chdir if chdir
+          self.preview = preview
+        end
+      end
+
+      def run
+        process = -> do
+          if preview
+            stdout.puts "Config:\n#{File.read(config)}"
+            cmds = [embulk_bin, "preview", *embulk_bundle, config].compact
+            stdout.puts cmds.join(" ")
+          else
+            cmds = [embulk_bin, *embulk_vm_options, *jvm_options, "run", *embulk_bundle, config].compact
+            stdout.puts cmds.join(" ")
+          end
+
+          stdout.flush
+
+          cmdenv = env || {}
+          opts = chdir ? {chdir: chdir} : {}
+          result, log = execute_command(cmds, cmdenv, opts)
+
+          unless result.success?
+            next if log =~ /NoSampleException/
+            raise "embulk error"
+          end
+        end
+
+        if defined?(Bundler)
+          Bundler.with_clean_env(&process)
+        else
+          process.call
+        end
+      end
+    end
+  end
+end

data/lib/rukawa/builtins/shell.rb

@@ -0,0 +1,72 @@
+require 'rukawa/builtins/base'
+require 'active_support/core_ext/class'
+require 'open3'
+
+module Rukawa
+  module Builtins
+    class Shell < Base
+      class_attribute :command, :args, :env, :chdir, :stdout, :stderr
+
+      self.args = []
+      self.stdout = $stdout
+      self.stderr = $stderr
+
+      class << self
+        def handle_parameters(command:, args: [], env: nil, chdir: nil, stdout: nil, stderr: nil, **rest)
+          self.command = command
+          self.args = args
+          self.env = env if env
+          self.chdir = chdir if chdir
+          self.stdout = stdout if stdout
+          self.stderr = stderr if stderr
+        end
+      end
+
+      def run
+        cmdenv = env || {}
+        opts = chdir ? {chdir: chdir} : {}
+
+        if defined?(Bundler)
+          result, log = nil
+          Bundler.with_clean_env do
+            result, log = execute_command([command, *args], cmdenv, opts)
+          end
+        else
+          result, log = execute_command([command, *args], cmdenv, opts)
+        end
+
+        unless result.success?
+          raise "command error"
+        end
+      end
+
+      def execute_command(command, env, opts)
+        log = "".dup
+        result = Open3.popen3(env, *command, opts) do |stdin, out, err, wait_th|
+          stdin.close
+          until out.eof? && err.eof?
+            rs, = IO.select([out, err])
+            rs.each do |rio|
+              line = rio.gets
+              if line
+                log << line
+                if rio == out
+                  stdout.write(line)
+                else
+                  stderr.write(line)
+                end
+              end
+            end
+          end
+
+          wait_th.value
+        end
+
+        stdout.flush
+        stderr.flush unless stdout == stderr
+
+        return result, log
+      end
+    end
+  end
+end

data/lib/rukawa/builtins/waiter.rb

@@ -0,0 +1,208 @@
+require 'rukawa/builtins/base'
+require 'timeout'
+
+module Rukawa
+  module Builtins
+    class Waiter < Base
+      class_attribute :timeout, :poll_interval
+
+      self.timeout = 1800
+      self.poll_interval = 1
+
+      class << self
+        def handle_parameters(timeout: nil, poll_interval: nil, **rest)
+          self.timeout = timeout if timeout
+          self.poll_interval = poll_interval if poll_interval
+        end
+      end
+
+      def run
+        Timeout.timeout(timeout) do
+          wait_until do
+            fetch_condition
+          end
+        end
+      end
+
+      private
+
+      def wait_until
+        until yield
+          sleep poll_interval
+        end
+      end
+
+      def fetch_condition
+        raise NotImplementedError
+      end
+    end
+
+    class SleepWaiter < Waiter
+      class_attribute :sec
+
+      class << self
+        def handle_parameters(sec:, **rest)
+          self.sec = sec
+        end
+      end
+
+      private
+
+      def fetch_condition
+        sleep sec
+      end
+    end
+
+    class LocalFileWaiter < Waiter
+      class_attribute :path, :if_modified_since, :if_unmodified_since
+
+      class << self
+        def handle_parameters(path:, if_modified_since: nil, if_unmodified_since: nil, **rest)
+          self.path = path
+          self.if_modified_since = if_modified_since if if_modified_since
+          self.if_unmodified_since = if_unmodified_since if if_unmodified_since
+          super(**rest)
+        end
+      end
+
+      private
+
+      def fetch_condition
+        if path.respond_to?(:all?)
+          get_stat_and_check_condition(p)
+          path.all?(&method(:get_stat_and_check_condition))
+        else
+          get_stat_and_check_condition(path)
+        end
+      end
+
+      def get_stat_and_check_condition(path)
+        stat = File.stat(path)
+
+        if if_modified_since
+          stat.mtime > if_modified_since
+        elsif if_unmodified_since
+          stat.mtime <= if_unmodified_since
+        else
+          true
+        end
+      rescue
+        false
+      end
+    end
+
+    class S3Waiter < Waiter
+      class_attribute :url, :aws_access_key_id, :aws_secret_access_key, :region, :if_modified_since, :if_unmodified_since
+
+      class << self
+        def handle_parameters(url:, aws_access_key_id: nil, aws_secret_access_key: nil, region: nil, if_modified_since: nil, if_unmodified_since: nil, **rest)
+          require 'aws-sdk'
+
+          self.url = url
+          self.aws_access_key_id = aws_access_key_id if aws_access_key_id
+          self.aws_secret_access_key = aws_secret_access_key if aws_secret_access_key
+          self.region = region if region
+          self.if_modified_since = if_modified_since if if_modified_since
+          self.if_unmodified_since = if_unmodified_since if if_unmodified_since
+          super(**rest)
+        end
+      end
+
+      private
+
+      def fetch_condition
+        opts = {if_modified_since: if_modified_since, if_unmodified_since: if_unmodified_since}.reject do |_, v|
+          v.nil?
+        end
+
+        if url.respond_to?(:all?)
+          url.all? do |u|
+            s3url = URI.parse(u)
+            client.head_object(bucket: s3url.host, key: s3url.path[1..-1], **opts) rescue false
+          end
+        else
+          s3url = URI.parse(url)
+          client.head_object(bucket: s3url.host, key: s3url.path[1..-1], **opts) rescue false
+        end
+      end
+
+      def client
+        return @client if @client
+
+        if aws_secret_access_key || aws_secret_access_key || region
+          options = {access_key_id: aws_access_key_id, secret_access_key: aws_secret_access_key, region: region}.reject do |_, v|
+            v.nil?
+          end
+          @client = Aws::S3::Client.new(options)
+        else
+          @client = Aws::S3::Client.new
+        end
+      end
+    end
+
+    class GCSWaiter < Waiter
+      class_attribute :url, :json_key, :if_modified_since, :if_unmodified_since
+
+      class << self
+        def handle_parameters(url:, json_key: nil, if_modified_since: nil, if_unmodified_since: nil, **rest)
+          require 'google/apis/storage_v1'
+          require 'googleauth'
+
+          self.url = url
+          self.json_key = json_key if json_key
+          self.if_modified_since = if_modified_since if if_modified_since
+          self.if_unmodified_since = if_unmodified_since if if_unmodified_since
+          super(**rest)
+        end
+      end
+
+      private
+
+      def fetch_condition
+        if url.respond_to?(:all?)
+          url.all? do |u|
+            get_object_and_check_condition(URI.parse(u))
+          end
+        else
+          get_object_and_check_condition(URI.parse(url))
+        end
+      end
+
+      def get_object_and_check_condition(url)
+        obj = client.get_object(url.host, url.path[1..-1])
+        if if_modified_since
+          obj.updated.to_time > if_modified_since
+        elsif if_unmodified_since
+          obj.updated.to_time <= if_unmodified_since
+        else
+          true
+        end
+      rescue
+        false
+      end
+
+      def client
+        return @client if @client
+
+        client = Google::Apis::StorageV1::StorageService.new
+        scope = "https://www.googleapis.com/auth/devstorage.read_only"
+
+        if json_key
+          begin
+            JSON.parse(json_key)
+            key = StringIO.new(json_key)
+            client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: key, scope: scope)
+          rescue JSON::ParserError
+            key = json_key
+            File.open(json_key) do |f|
+              client.authorization = Google::Auth::ServiceAccountCredentials.make_creds(json_key_io: f, scope: scope)
+            end
+          end
+        else
+          client.authorization = Google::Auth.get_application_default([scope])
+        end
+        client
+      end
+    end
+  end
+end

data/lib/rukawa/cli.rb

@@ -35,7 +35,7 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, variables, Context.new, *job_classes)
+      job_net = job_net_class.new(variables: variables, resume_job_classes: job_classes)
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]
@@ -60,7 +60,7 @@ module Rukawa
 
       job_net_class = get_class(job_net_name)
       job_classes = job_name.map { |name| get_class(name) }
-      job_net = job_net_class.new(nil, {}, Context.new, *job_classes)
+      job_net = job_net_class.new(resume_job_classes: job_classes)
       job_net.output_dot(options[:output], format: options[:format])
     end
 
@@ -75,7 +75,7 @@ module Rukawa
 
       job_classes = job_name.map { |name| get_class(name) }
       job_net_class = anonymous_job_net_class(*job_classes)
-      job_net = job_net_class.new(nil, variables, Context.new)
+      job_net = job_net_class.new
       result = Runner.run(job_net, options[:batch], options[:refresh_interval])
 
       if options[:dot]
@@ -131,8 +131,6 @@ module Rukawa
           c.logger = Syslog::Logger.new('rukawa')
         elsif options[:log]
           c.logger = Logger.new(options[:log])
-        else
-          c.logger ||= Logger.new('./rukawa.log');
         end
       end
     end

data/lib/rukawa/configuration.rb

@@ -6,13 +6,15 @@ require 'concurrent'
 module Rukawa
   class Configuration < Delegator
     include Singleton
-    attr_accessor :logger
 
     def initialize
       @config = OpenStruct.new(
         concurrency: Concurrent.processor_count,
         dot_command: "dot",
-        job_dirs: [File.join(Dir.pwd, "job_nets"), File.join(Dir.pwd, "jobs")]
+        job_dirs: [File.join(Dir.pwd, "job_nets"), File.join(Dir.pwd, "jobs")],
+        status_store: nil,
+        status_expire_duration: 24 * 60 * 60,
+        logger: Logger.new('./rukawa.log')
       )
       @config.graph = GraphConfig.new.tap { |c| c.rankdir = "LR" }
     end

data/lib/rukawa/dag.rb

@@ -18,7 +18,7 @@ module Rukawa
       deps = tsortable_hash(dependencies).tsort
 
       deps.each do |job_class|
-        job = job_class.new(job_net, variables, context)
+        job = job_class.new(variables: variables, context: context, parent_job_net: job_net)
         @nodes << job
         @jobs << job if job.is_a?(Job)
 
@@ -92,6 +92,7 @@ module Rukawa
     private
 
     def tsortable_hash(hash)
+      ensure_dependencies_have_all_jobs_as_key!(hash)
       class << hash
         include TSort
         alias :tsort_each_node :each_key
@@ -102,6 +103,14 @@ module Rukawa
       hash
     end
 
+    def ensure_dependencies_have_all_jobs_as_key!(hash)
+      hash.values.each do |parents|
+        parents.each do |j|
+          hash[j] ||= []
+        end
+      end
+    end
+
     class Edge
       attr_reader :from, :to, :cluster
 

data/lib/rukawa/job.rb

@@ -70,6 +70,17 @@ module Rukawa
       def around_run(*args, **options, &block)
         set_callback :run, :around, *args, **options, &block
       end
+
+      def wrappers
+        @@wrappers ||= {}
+      end
+
+      def wrapper_for(*classes)
+        classes.each do |c|
+          raise "Wrapper for #{c} is already defined" if wrappers[c]
+          wrappers[c] = self
+        end
+      end
     end
 
     around_run do |_, blk|
@@ -84,7 +95,7 @@ module Rukawa
       set_state(:finished)
     end
 
-    def initialize(parent_job_net, variables, context)
+    def initialize(variables: {}, context: Context.new, parent_job_net: nil)
       @parent_job_net = parent_job_net
       @variables = variables
       @context = context
@@ -150,9 +161,9 @@ module Rukawa
 
     def to_dot_def
       if state == Rukawa::State::Waiting
-        "#{name};\n"
+        "\"#{name}\";\n"
       else
-        "#{name} [style = filled,fillcolor = #{state.color}];\n"
+        "\"#{name}\" [style = filled,fillcolor = #{state.color}];\n"
       end
     end
 
@@ -218,6 +229,15 @@ module Rukawa
       @context.store[self.class][key] = value
     end
 
+    def fetch(job, key)
+      job = job.is_a?(String) ? Object.const_get(job) : job
+      raise TypeError, "job must be a Class" unless job.is_a?(Class)
+
+      if @context.store[job]
+        @context.store[job][key]
+      end
+    end
+
     def acquire_resource
       @context.semaphore.acquire(resource_count) if resource_count > 0
       yield

data/lib/rukawa/job_net.rb

@@ -11,7 +11,7 @@ module Rukawa
       end
     end
 
-    def initialize(parent_job_net, variables, context, *resume_job_classes)
+    def initialize(variables: {}, context: Context.new, parent_job_net: nil, resume_job_classes: [])
       @parent_job_net = parent_job_net
       @variables = variables
       @context = context

data/lib/rukawa/remote/status_store.rb

@@ -0,0 +1,46 @@
+module Rukawa
+  module Remote
+    class StatusStore
+      ENQUEUED = "enqueued".freeze
+      PERFORMING = "performing".freeze
+      COMPLETED = "completed".freeze
+      FAILED = "failed".freeze
+
+      # default expire duration is 24 hours.
+      def initialize(job_id:, expire_duration: Rukawa.config.status_expire_duration)
+        @job_id = job_id
+        @expire_duration = expire_duration
+      end
+
+      def fetch
+        Rukawa.config.status_store.fetch(store_key)
+      end
+
+      def enqueued
+        Rukawa.config.status_store.write(store_key, ENQUEUED, expires_in: @expire_duration)
+      end
+
+      def performing
+        Rukawa.config.status_store.write(store_key, PERFORMING, expires_in: @expire_duration)
+      end
+
+      def completed
+        Rukawa.config.status_store.write(store_key, COMPLETED, expires_in: @expire_duration)
+      end
+
+      def failed
+        Rukawa.config.status_store.write(store_key, FAILED, expires_in: @expire_duration)
+      end
+
+      def delete
+        Rukawa.config.status_store.delete(store_key)
+      end
+
+      private
+
+      def store_key
+        "rukawa.remote_job.status.#{@job_id}"
+      end
+    end
+  end
+end

data/lib/rukawa/remote.rb

@@ -0,0 +1,9 @@
+module Rukawa
+  module Remote
+    class << self
+      def store
+        Rukawa.config.status_store
+      end
+    end
+  end
+end

data/lib/rukawa/version.rb

@@ -1,3 +1,3 @@
 module Rukawa
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

data/lib/rukawa/wrapper/active_job.rb

@@ -0,0 +1,75 @@
+require 'rukawa/job'
+require 'rukawa/wrapper'
+require 'rukawa/remote/status_store'
+
+module Rukawa
+  module Wrapper
+    module ActiveJob
+      def self.[](job_class)
+        raise "Please set Rukawa.config.status_store subclass of ActiveSupport::Cache::Store" unless Rukawa.config.status_store
+        @wrapper_classes ||= {}
+        return @wrapper_classes[job_class] if @wrapper_classes[job_class]
+
+        wrapper = Class.new(Rukawa::Job) do
+          set_resource_count 0
+
+          define_singleton_method(:origin_class) do
+            job_class
+          end
+
+          def initialize(variables: {}, context: Context.new, parent_job_net: nil)
+            super
+            @job_class = self.class.origin_class
+          end
+
+          def run
+            @job_class.include(Hooks) unless @job_class.include?(Hooks)
+            @job_class.prepend(HooksForFailure) unless @job_class.include?(HooksForFailure)
+            job = @job_class.perform_later
+
+            status_store = Rukawa::Remote::StatusStore.new(job_id: job.job_id)
+            finish_statuses = [Rukawa::Remote::StatusStore::COMPLETED, Rukawa::Remote::StatusStore::FAILED]
+            until finish_statuses.include?(last_status = status_store.fetch)
+              sleep 0.1
+            end
+
+            status_store.delete
+
+            raise WrappedJobError if last_status == Rukawa::Remote::StatusStore::FAILED
+          end
+        end
+
+        @wrapper_classes[job_class] = wrapper
+        Rukawa::Wrapper.const_set("#{job_class.to_s.gsub(/::/, "_")}Wrapper", wrapper)
+        wrapper
+      end
+    end
+
+    module Hooks
+      def self.included(base)
+        base.class_eval do
+          before_enqueue { status_store.enqueued }
+
+          before_perform { status_store.performing }
+
+          after_perform { status_store.completed }
+        end
+      end
+
+      private
+
+      def status_store
+        @status_store ||= Rukawa::Remote::StatusStore.new(job_id: job_id)
+      end
+    end
+
+    module HooksForFailure
+      def perform(*args)
+        super
+      rescue Exception
+        status_store.failed
+        raise
+      end
+    end
+  end
+end

data/lib/rukawa/wrapper.rb

@@ -0,0 +1,5 @@
+module Rukawa
+  module Wrapper
+    class WrappedJobError < StandardError; end
+  end
+end

data/lib/rukawa.rb

@@ -33,3 +33,4 @@ require 'rukawa/configuration'
 require 'rukawa/job_net'
 require 'rukawa/job'
 require 'rukawa/dag'
+require 'rukawa/wrapper/active_job'

data/rukawa.gemspec

@@ -18,15 +18,20 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "activesupport", ">= 4"
+  spec.add_runtime_dependency "activesupport", ">= 6"
+  spec.add_runtime_dependency "redis"
   spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "thor"
   spec.add_runtime_dependency "terminal-table"
   spec.add_runtime_dependency "paint"
 
-  spec.add_development_dependency "bundler", "~> 1.11"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "rspec-power_assert"
   spec.add_development_dependency "rspec-parameterized"
+  spec.add_development_dependency "activejob"
+  spec.add_development_dependency "sucker_punch"
+  spec.add_development_dependency "aws-sdk", "~> 2.0"
+  spec.add_development_dependency "google-api-client", "~> 0.9"
 end

data/sample/job_nets/sample_job_net.rb

@@ -2,8 +2,6 @@ class InnerJobNet < Rukawa::JobNet
   class << self
     def dependencies
       {
-        InnerJob3 => [],
-        InnerJob1 => [],
         InnerJob2 => [InnerJob1, InnerJob3],
       }
     end
@@ -14,7 +12,6 @@ class InnerJobNet2 < Rukawa::JobNet
   class << self
     def dependencies
       {
-        InnerJob4 => [],
         InnerJob5 => [InnerJob4],
         InnerJob6 => [InnerJob4, InnerJob5],
       }
@@ -26,8 +23,6 @@ class InnerJobNet3 < Rukawa::JobNet
   class << self
     def dependencies
       {
-        InnerJob7 => [],
-        InnerJob8 => [],
         InnerJob9 => [InnerJob7, InnerJob8],
         InnerJob10 => [InnerJob7, InnerJob8],
       }
@@ -39,9 +34,6 @@ class InnerJobNet4 < Rukawa::JobNet
   class << self
     def dependencies
       {
-        InnerJob11 => [],
-        InnerJob12 => [],
-        InnerJob13 => [],
         NestedJobNet => [InnerJob11, InnerJob12],
       }
     end
@@ -52,7 +44,6 @@ class NestedJobNet < Rukawa::JobNet
   class << self
     def dependencies
       {
-        NestedJob1 => [],
         NestedJob2 => [NestedJob1],
       }
     end
@@ -62,9 +53,13 @@ end
 class SampleJobNet < Rukawa::JobNet
   class << self
     def dependencies
+      wrapped1 = Rukawa::Wrapper::ActiveJob[ActiveJobSample1]
+      wrapped2 = Rukawa::Wrapper::ActiveJob[ActiveJobSample2]
       {
         Job1 => [],
-        Job2 => [Job1], Job3 => [Job1],
+        wrapped1 => [Job1],
+        wrapped2 => [wrapped1],
+        Job2 => [Job1], Job3 => [Job1, wrapped1],
         Job4 => [Job2, Job3],
         InnerJobNet => [Job3],
         Job8 => [InnerJobNet],

data/sample/jobs/active_job.rb

@@ -0,0 +1,19 @@
+require 'active_job'
+
+class ActiveJobSample1 < ActiveJob::Base
+  queue_as :default
+
+  def perform
+    sleep 10
+    p "active_job1"
+  end
+end
+
+class ActiveJobSample2 < ActiveJob::Base
+  queue_as :default
+
+  def perform
+    sleep 10
+    raise "active_job2 is failed"
+  end
+end

data/sample/result.dot

@@ -1,31 +1,33 @@
 digraph "SampleJobNet" {
 label = "SampleJobNet";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
-Job1 [style = filled,fillcolor = green];
-Job2 [style = filled,fillcolor = red];
-Job3 [style = filled,fillcolor = green];
-Job4 [style = filled,fillcolor = green];
+"Job1" [style = filled,fillcolor = green];
+"Rukawa::Wrapper::ActiveJobSample1Wrapper" [style = filled,fillcolor = green];
+"Rukawa::Wrapper::ActiveJobSample2Wrapper" [style = filled,fillcolor = red];
+"Job2" [style = filled,fillcolor = red];
+"Job3" [style = filled,fillcolor = green];
+"Job4" [style = filled,fillcolor = green];
 subgraph "cluster_InnerJobNet" {
 label = "InnerJobNet";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
-InnerJob3 [style = filled,fillcolor = green];
-InnerJob1 [style = filled,fillcolor = green];
-InnerJob2 [style = filled,fillcolor = red];
+"InnerJob3" [style = filled,fillcolor = green];
+"InnerJob1" [style = filled,fillcolor = green];
+"InnerJob2" [style = filled,fillcolor = red];
 "InnerJob1" -> "InnerJob2";
 "InnerJob3" -> "InnerJob2";
 }
-Job8 [style = filled,fillcolor = magenta];
-Job5 [style = filled,fillcolor = red];
-Job6 [style = filled,fillcolor = green];
-Job7 [style = filled,fillcolor = green];
+"Job8" [style = filled,fillcolor = magenta];
+"Job5" [style = filled,fillcolor = red];
+"Job6" [style = filled,fillcolor = green];
+"Job7" [style = filled,fillcolor = green];
 subgraph "cluster_InnerJobNet2" {
 label = "InnerJobNet2";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
-InnerJob4 [style = filled,fillcolor = green];
-InnerJob5 [style = filled,fillcolor = yellow];
-InnerJob6 [style = filled,fillcolor = magenta];
+"InnerJob4" [style = filled,fillcolor = green];
+"InnerJob5" [style = filled,fillcolor = yellow];
+"InnerJob6" [style = filled,fillcolor = magenta];
 "InnerJob4" -> "InnerJob5";
 "InnerJob4" -> "InnerJob6";
 "InnerJob5" -> "InnerJob6";
@@ -34,10 +36,10 @@ subgraph "cluster_InnerJobNet3" {
 label = "InnerJobNet3";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
-InnerJob7 [style = filled,fillcolor = magenta];
-InnerJob8 [style = filled,fillcolor = magenta];
-InnerJob9 [style = filled,fillcolor = magenta];
-InnerJob10 [style = filled,fillcolor = magenta];
+"InnerJob7" [style = filled,fillcolor = magenta];
+"InnerJob8" [style = filled,fillcolor = magenta];
+"InnerJob9" [style = filled,fillcolor = magenta];
+"InnerJob10" [style = filled,fillcolor = magenta];
 "InnerJob7" -> "InnerJob9";
 "InnerJob8" -> "InnerJob9";
 "InnerJob7" -> "InnerJob10";
@@ -47,22 +49,25 @@ subgraph "cluster_InnerJobNet4" {
 label = "InnerJobNet4";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
-InnerJob11 [style = filled,fillcolor = magenta];
-InnerJob12 [style = filled,fillcolor = magenta];
-InnerJob13 [style = filled,fillcolor = magenta];
+"InnerJob11" [style = filled,fillcolor = magenta];
+"InnerJob12" [style = filled,fillcolor = magenta];
+"InnerJob13" [style = filled,fillcolor = magenta];
 subgraph "cluster_NestedJobNet" {
 label = "NestedJobNet";
 graph [rankdir = LR,nodesep = 0.8,concentrate = true];
 color = blue;
-NestedJob1 [style = filled,fillcolor = magenta];
-NestedJob2 [style = filled,fillcolor = magenta];
+"NestedJob1" [style = filled,fillcolor = magenta];
+"NestedJob2" [style = filled,fillcolor = magenta];
 "NestedJob1" -> "NestedJob2";
 }
 "InnerJob11" -> "NestedJob1";
 "InnerJob12" -> "NestedJob1";
 }
+"Job1" -> "Rukawa::Wrapper::ActiveJobSample1Wrapper";
+"Rukawa::Wrapper::ActiveJobSample1Wrapper" -> "Rukawa::Wrapper::ActiveJobSample2Wrapper";
 "Job1" -> "Job2";
 "Job1" -> "Job3";
+"Rukawa::Wrapper::ActiveJobSample1Wrapper" -> "Job3";
 "Job2" -> "Job4";
 "Job3" -> "Job4";
 "Job3" -> "InnerJob3";

data/sample/rukawa.rb

@@ -2,3 +2,11 @@ Rukawa.configure do |c|
   c.graph.concentrate = true
   c.graph.nodesep = 0.8
 end
+
+redis_host = ENV["REDIS_HOST"] || "localhost:6379"
+Rukawa.configure do |c|
+  c.status_store = ActiveSupport::Cache::RedisStore.new(redis_host)
+end
+
+require 'active_job'
+ActiveJob::Base.queue_adapter = :sucker_punch

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rukawa
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.9.0
 platform: ruby
 authors:
 - joker1007
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2016-07-29 00:00:00.000000000 Z
+date: 2023-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '4'
+        version: '6'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -86,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.11'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '13.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -150,6 +164,62 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: activejob
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sucker_punch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: google-api-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
 description: Hyper simple job workflow engine
 email:
 - kakyoin.hierophant@gmail.com
@@ -158,10 +228,11 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - Gemfile
+- LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
@@ -169,6 +240,10 @@ files:
 - exe/rukawa
 - lib/rukawa.rb
 - lib/rukawa/abstract_job.rb
+- lib/rukawa/builtins/base.rb
+- lib/rukawa/builtins/embulk.rb
+- lib/rukawa/builtins/shell.rb
+- lib/rukawa/builtins/waiter.rb
 - lib/rukawa/cli.rb
 - lib/rukawa/configuration.rb
 - lib/rukawa/context.rb
@@ -178,12 +253,17 @@ files:
 - lib/rukawa/job.rb
 - lib/rukawa/job_net.rb
 - lib/rukawa/overview.rb
+- lib/rukawa/remote.rb
+- lib/rukawa/remote/status_store.rb
 - lib/rukawa/runner.rb
 - lib/rukawa/state.rb
 - lib/rukawa/version.rb
+- lib/rukawa/wrapper.rb
+- lib/rukawa/wrapper/active_job.rb
 - rukawa.gemspec
 - sample/job_nets/sample_job_net.rb
 - sample/jobnet.png
+- sample/jobs/active_job.rb
 - sample/jobs/sample_job.rb
 - sample/result.dot
 - sample/result.png
@@ -193,7 +273,7 @@ files:
 homepage: https://github.com/joker1007/rukawa
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -208,9 +288,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.4
-signing_key: 
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: Hyper simple job workflow engine
 test_files: []

data/.travis.yml

@@ -1,6 +0,0 @@
-language: ruby
-sudo: false
-rvm:
-  - 2.2.5
-  - 2.3.0
-before_install: gem install bundler -v 1.11.2