abid 0.3.0.pre.alpha.3 → 0.3.0.pre.alpha.4

data/lib/abid/dsl/rake_job.rb

@@ -0,0 +1,41 @@
+require 'abid/dsl/job'
+
+module Abid
+  module DSL
+    # Rake::Task wrapper.
+    class RakeJob < Job
+      def volatile?
+        true
+      end
+
+      def worker
+        :default
+      end
+
+      def params
+        {}
+      end
+
+      def concerned?
+        true
+      end
+
+      def needed?
+        task.needed?
+      end
+
+      def execute(args)
+        task.application.trace "** Execute (dry run) #{task.name}" if dryrun?
+        return if dryrun? || preview?
+
+        task.execute(args)
+      end
+
+      def prerequisites
+        task.prerequisite_tasks.map do |preq|
+          task.application.job_manager.bind(preq.name, {})
+        end
+      end
+    end
+  end
+end

data/lib/abid/dsl/syntax.rb

@@ -0,0 +1,34 @@
+module Abid
+  module DSL
+    module Syntax
+      def play(*args, &block)
+        Task.define_task(*args, &block)
+      end
+
+      def mixin(*args, &block)
+        MixinTask.define_task(*args, &block)
+      end
+
+      def define_worker(name, thread_count)
+        Abid.global.engine.worker_manager.define(name, thread_count)
+      end
+
+      def global_mixin(&block)
+        Abid.application.global_mixin.class_eval(&block)
+      end
+
+      def invoke(task, *args, **params)
+        Abid.global.engine.invoke(task, *args, **params)
+      end
+
+      # @yieldparam top_level_tasks [Array<String>]
+      # @yieldparam summary [Hash]
+      # @yieldparam errors [Array<Exception>]
+      def after_all(&block)
+        Abid.application.after_all_actions << block
+      end
+    end
+  end
+end
+
+extend Abid::DSL::Syntax

data/lib/abid/dsl/task.rb

@@ -0,0 +1,53 @@
+module Abid
+  module DSL
+    # `play` task is defined as an instance of Abid::DSL::Task.
+    #
+    #     play(:foo) { ... } #=> #<Abid::DSL::Task ...>
+    #
+    class Task < Rake::Task
+      attr_reader :internal
+
+      def initialize(*args)
+        super
+        initialize_internal
+      end
+
+      def initialize_internal
+        @internal = Play.create(self)
+      end
+
+      def enhance(deps = nil, &block)
+        @internal.module_eval(&block) if block_given?
+        super(deps)
+      end
+
+      def clear
+        @internal_definitions.clear
+        initialize_internal
+        super
+      end
+
+      def bind(params = {})
+        AbidJob.new(self, params)
+      end
+
+      def params_spec
+        internal.params_spec
+      end
+    end
+
+    # `mixin` task is defined as an instance of Abid::DSL::MixinTask.
+    #
+    #     mixin(:foo) { ... } #=> #<Abid::DSL::MixinTask ...>
+    #
+    class MixinTask < Task
+      def initialize_internal
+        @internal = Mixin.create(self)
+      end
+
+      def bind(*_args)
+        raise 'mixin is not executable'
+      end
+    end
+  end
+end

data/lib/abid/engine.rb

@@ -1,13 +1,43 @@
+require 'forwardable'
+
+require 'abid/engine/executor'
+require 'abid/engine/process_manager'
+require 'abid/engine/process'
+require 'abid/engine/scheduler'
+require 'abid/engine/worker_manager'
+require 'abid/engine/waiter'
+
 module Abid
   # Engine module operates task execution.
-  module Engine
-    def self.kill(error)
-      Abid.global.worker_manager.kill
-      Abid.global.process_manager.kill(error)
+  class Engine
+    extend Forwardable
+
+    def initialize(env)
+      @env = env
+      @process_manager = ProcessManager.new(self)
+      @worker_manager = WorkerManager.new(self)
+    end
+    attr_reader :process_manager, :worker_manager
+    def_delegators :@env, :options, :state_manager, :logger
+    def_delegators :process_manager, :summary, :errors
+
+    # @param name [String] task name
+    # @param params [Hash]
+    # @param args [Array]
+    # @return [(Symbol, Exception)] pair of result and error.
+    def invoke(name, *args, **params)
+      task = @env.application.job_manager[name, params]
+      process_manager.invoke(task, args)
+    end
+
+    def kill(error)
+      worker_manager.kill
+      process_manager.kill(error)
     end
 
-    def self.shutdown
-      Abid.global.worker_manager.shutdown
+    def shutdown
+      process_manager.shutdown
+      worker_manager.shutdown
     end
   end
 end

data/lib/abid/engine/executor.rb

@@ -1,21 +1,22 @@
 module Abid
-  module Engine
+  class Engine
     # @!visibility private
 
-    # Executor operates each task execution.
+    # Executor operates each job execution.
     class Executor
-      def initialize(job, args)
-        @job = job
+      def initialize(process, args)
+        @process = process
+        @job = process.job
         @args = args
 
-        @process = job.process
-        @state = job.state.find
-        @prerequisites = job.prerequisites.map(&:process)
+        @state = @process.state_service.find
+        @prerequisites = process.prerequisites
+        @worker = @process.engine.worker_manager[@job.worker]
       end
 
-      # Check if the task should be executed.
+      # Check if the job should be executed.
       #
-      # @return [Boolean] false if the task should not be executed.
+      # @return [Boolean] false if the job should not be executed.
       def prepare
         return unless @process.prepare
         return false if precheck_to_cancel
@@ -23,11 +24,11 @@ module Abid
         true
       end
 
-      # Start processing the task.
+      # Start processing the job.
       #
-      # The task is executed asynchronously.
+      # The job is executed asynchronously.
       #
-      # @return [Boolean] false if the task is not executed
+      # @return [Boolean] false if the job is not executed
       def start
         return false unless @prerequisites.all?(&:complete?)
 
@@ -39,37 +40,28 @@ module Abid
         true
       end
 
-      def capture_exception
-        yield
-      rescue StandardError, ScriptError => error
-        @process.quit(error)
-      rescue Exception => exception
-        # TODO: exit immediately when fatal error occurs.
-        @process.quit(exception)
-      end
-
       private
 
-      # Cancel the task if it should be.
+      # Cancel the job if it should be.
       # @return [Boolean] true if cancelled
       def precheck_to_cancel
-        return false if @job.env.options.repair
+        return false if @job.repair?
         return false unless @state.failed?
-        return false if @job.task.top_level?
+        return false if @process.root?
         @process.cancel(Error.new('task has been failed'))
       end
 
-      # Skip the task if it should be.
+      # Skip the job if it should be.
       # @return [Boolean] true if skipped
       def precheck_to_skip
-        return @process.skip unless @job.task.concerned?
+        return @process.skip unless @job.concerned?
 
-        return false if @job.env.options.repair && !@prerequisites.empty?
+        return false if @job.repair? && !@prerequisites.empty?
         return false unless @state.successed?
         @process.skip
       end
 
-      # Cancel the task if it should be.
+      # Cancel the job if it should be.
       # @return [Boolean] true if cancelled
       def check_to_cancel
         return false if @prerequisites.empty?
@@ -77,50 +69,40 @@ module Abid
         @process.cancel
       end
 
-      # Skip the task if it should be.
+      # Skip the job if it should be.
       # @return [Boolean] true if skipped
       def check_to_skip
-        return @process.skip unless @job.task.needed?
+        return @process.skip unless @job.needed?
 
         return false if @prerequisites.empty?
-        return false unless @job.env.options.repair
+        return false unless @job.repair?
         return false if @prerequisites.any?(&:successed?)
         @process.skip
       end
 
-      # Post the task if no external process executing the same task, wait the
-      # task finished otherwise.
+      # Post the job if no external process executing the same job, wait the
+      # job finished otherwise.
       def execute_or_wait
-        if @job.state.try_start
-          @job.worker.post { capture_exception { execute } }
+        if @process.state_service.try_start
+          @worker.post { @process.capture_exception { execute } }
         else
-          Waiter.new(@job).wait
+          Waiter.new(@process).wait
         end
       end
 
       def execute
         _, error = safe_execute
 
-        call_after_hooks(error)
-        @job.state.finish(error)
+        @process.state_service.finish(error)
         @process.finish(error)
       end
 
       def safe_execute
-        @job.task.call_hooks(:before_execute)
-        @job.task.execute(@args)
+        @job.execute(@args)
         true
       rescue => error
         [false, error]
       end
-
-      def call_after_hooks(error)
-        @job.task.call_hooks(:after_invoke, error)
-        true
-      rescue
-        # TODO: Error logging
-        false
-      end
     end
   end
 end

data/lib/abid/engine/process.rb

@@ -1,136 +1,170 @@
-require 'concurrent/ivar'
 require 'forwardable'
-require 'monitor'
 
 module Abid
-  module Engine
+  class Engine
     # @!visibility private
 
-    # Process object manages the task execution status.
+    # Process object manages the job execution status.
     #
     # You should retrive a process object via Job#process.
     # Do not create a process object by Process.new constructor.
     #
-    # A process object has an internal status of the task execution and
-    # the task result (Process#result).
+    # A process object has an internal status of the job execution.
     #
     # An initial status is :unscheduled.
     # When Process#prepare is called, the status gets :pending.
-    # When Process#execute is called and the task is posted to a thread pool,
-    # the status gets :running. When the task is finished, the status gets
-    # :complete and the result is assigned to :successed or :failed.
+    # When Process#execute is called and the job is posted to a thread pool,
+    # the status gets :running. When the job is finished, the status gets
+    # :successed or :failed.
     #
-    #     process = Job['task_name'].process
+    #     process = Job['job_name'].process
     #     process.prepare
     #     process.start
     #     process.wait
-    #     process.result #=> :successed or :failed
+    #     process.status #=> :successed or :failed
     #
     # Possible status are:
     #
     # <dl>
     #   <dt>:unscheduled</dt>
-    #   <dd>The task is not invoked yet.</dd>
+    #   <dd>The job is not invoked yet.</dd>
     #   <dt>:pending</dt>
-    #   <dd>The task is waiting for prerequisites complete.</dd>
+    #   <dd>The job is waiting for prerequisites complete.</dd>
     #   <dt>:running</dt>
-    #   <dd>The task is running.</dd>
-    #   <dt>:complete</dt>
-    #   <dd>The task is finished.</dd>
-    # </dl>
-    #
-    # Possible results are:
-    #
-    # <dl>
+    #   <dd>The job is running.</dd>
     #   <dt>:successed</dt>
-    #   <dd>The task is successed.</dd>
+    #   <dd>The job is successed.</dd>
     #   <dt>:failed</dt>
-    #   <dd>The task is failed.</dd>
+    #   <dd>The job is failed.</dd>
     #   <dt>:cancelled</dt>
-    #   <dd>The task is not executed because of some problems.</dd>
+    #   <dd>The job is not executed because of some problems.</dd>
     #   <dt>:skipped</dt>
-    #   <dd>The task is not executed because already successed.</dd>
+    #   <dd>The job is not executed because already successed.</dd>
     # </dl>
     class Process
       extend Forwardable
 
-      attr_reader :status, :error
+      def initialize(engine, job)
+        @engine = engine
+        @job = job
+        @status = Status.new(:unscheduled)
+        @error = nil
+        initialize_logger(job)
+        initialize_state_service(job)
+      end
+      attr_reader :error, :engine, :job
+      def_delegators :@status, :on_update, :on_complete, :wait, :complete?
+
+      def initialize_logger(job)
+        @logger = @engine.logger.clone
+        pn = @logger.progname
+        @logger.progname = pn ? "#{pn}: #{job}" : job.to_s
+      end
+      private :initialize_logger
+      attr_reader :logger
 
-      def_delegators :@result_ivar, :add_observer, :wait, :complete?
+      def initialize_state_service(job)
+        @state_service = @engine.state_manager.state(
+          job.name, job.params,
+          dryrun: job.dryrun? || job.preview?,
+          volatile: job.volatile?
+        )
+      end
+      private :initialize_state_service
+      attr_reader :state_service
 
-      def initialize(process_manager)
-        @process_manager = process_manager
-        @result_ivar = Concurrent::IVar.new
-        @status = :unscheduled
-        @error = nil
-        @mon = Monitor.new
+      def prerequisites
+        @job.prerequisites.map { |preq| @engine.process_manager[preq] }
       end
 
-      %w(successed failed cancelled skipped).each do |meth|
+      #
+      # State predicates
+      #
+      %w(unscheduled pending running
+         successed failed cancelled skipped).each do |meth|
         class_eval <<-RUBY, __FILE__, __LINE__ + 1
         def #{meth}?
-          result == :#{meth}
+          @status.get == :#{meth}
         end
         RUBY
       end
 
-      def result
-        @result_ivar.value if @result_ivar.complete?
+      def root?
+        @engine.process_manager.root?(self)
+      end
+
+      def status
+        @status.get
       end
 
       def prepare
-        compare_and_set_status(:pending, :unscheduled)
+        @status.compare_and_set(:unscheduled, :pending)
       end
 
       def start
-        compare_and_set_status(:running, :pending)
+        return unless @status.compare_and_set(:pending, :running)
+        @logger.info('start.')
       end
 
       def finish(error = nil)
-        return unless compare_and_set_status(:complete, :running)
-        @error = error if error
-        @result_ivar.set(error.nil? ? :successed : :failed)
-        true
+        error.nil? ? successed : failed(error)
+      end
+
+      def successed
+        return unless @status.compare_and_set(:running, :successed, true)
+        @logger.info('successed.')
+      end
+
+      def failed(error)
+        log_error(error)
+        return unless @status.compare_and_set(:running, :failed, true)
+        @logger.error('failed.')
       end
 
       def cancel(error = nil)
-        return false unless compare_and_set_status(:complete, :pending)
-        @error = error if error
-        @result_ivar.set :cancelled
-        true
+        log_error(error) if error
+        return unless @status.compare_and_set(:pending, :cancelled, true)
+        @logger.info('cancelled.')
       end
 
       def skip
-        return false unless compare_and_set_status(:complete, :pending)
-        @result_ivar.set :skipped
-        true
+        return unless @status.compare_and_set(:pending, :skipped, true)
+        @logger.info('skipped')
       end
 
-      # Force fail the task.
+      # Force fail the job.
       # @return [void]
       def quit(error)
-        @status = :complete
-        @error = error
-        @result_ivar.try_set(:failed)
+        log_error(error)
+        @status.try_set(:failed, true)
+      end
+
+      def capture_exception
+        yield
+      rescue StandardError, ScriptError => error
+        quit(error)
+      rescue Exception => exception
+        # kill from independent thread
+        Thread.start { @engine.kill(exception) }
       end
 
       private
 
-      # Atomic compare and set operation.
-      # State is set to `next_state` only if
-      # `current state == expected_current`.
-      #
-      # @param [Symbol] next_state
-      # @param [Symbol] expected_current
-      #
-      # @return [Boolean] true if state is changed, false otherwise
-      def compare_and_set_status(next_state, *expected_current)
-        @mon.synchronize do
-          return unless expected_current.include? @status
-          @status = next_state
-          @process_manager.update(self)
-          true
+      def log_error(error)
+        @error = error
+        @logger.error(format_error_backtrace(error))
+      end
+
+      def format_error_backtrace(error)
+        if error.backtrace.nil? || error.backtrace.empty?
+          return "#{error.message} (#{error.class})"
         end
+
+        bt = error.backtrace
+        ret = ''
+        ret << "#{bt.first}: #{error.message} (#{error.class})\n"
+        bt.each { |b| ret << "    from #{b}\n" }
+        ret
       end
     end
   end