abid 0.1.1

data/lib/abid/dsl_definition.rb

@@ -0,0 +1,21 @@
+module Abid
+  module DSL
+    def play(*args, &block)
+      Abid::Task.define_play(*args, &block)
+    end
+
+    def define_worker(name, thread_count)
+      Rake.application.worker.define(name, thread_count)
+    end
+
+    def default_play_class(&block)
+      Rake.application.default_play_class(&block)
+    end
+
+    def helpers(*extensions, &block)
+      Abid::Play.helpers(*extensions, &block)
+    end
+  end
+end
+
+extend Abid::DSL

data/lib/abid/params_parser.rb

@@ -0,0 +1,50 @@
+module Abid
+  module ParamsParser
+    class <<self
+      def parse(params, specs)
+        specs.map do |name, spec|
+          if params.include?(name)
+            value = type_cast(params[name], spec[:type])
+          elsif ENV.include?(name.to_s)
+            value = type_cast(ENV[name.to_s], spec[:type])
+          elsif spec.key?(:default)
+            value = spec[:default]
+          else
+            fail "param #{name} is not specified"
+          end
+
+          [name, value]
+        end.to_h
+      end
+
+      def type_cast(value, type)
+        case type
+        when :boolean then value == 'true'
+        when :int then value.to_i
+        when :float then value.to_f
+        when :string then value.to_s
+        when :date then type_cast_date(value)
+        when :datetime, :time then type_cast_time(value)
+        when nil then value
+        else fail "invalid type: #{type}"
+        end
+      end
+
+      def type_cast_date(value)
+        case value
+        when Date then value
+        when Time, DateTime then value.to_date
+        else Date.parse(value.to_s)
+        end
+      end
+
+      def type_cast_time(value)
+        case value
+        when Date then value.to_time
+        when Time, DateTime then value
+        else Time.parse(value.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/abid/play.rb

@@ -0,0 +1,139 @@
+require 'forwardable'
+
+module Abid
+  class Play
+    class << self
+      attr_accessor :task
+
+      def inherited(child)
+        params_spec.each { |k, v| child.params_spec[k] = v.dup }
+        hooks.each { |k, v| child.hooks[k] = v.dup }
+      end
+
+      def params_spec
+        @params_spec ||= {}
+      end
+
+      def param(name, **param_spec)
+        params_spec[name] = { significant: true }.merge(param_spec)
+
+        define_method(name) { params[name] }
+      end
+
+      def hooks
+        @hooks ||= {
+          before: [],
+          after: [],
+          around: []
+        }
+      end
+
+      def set(name, value = nil, &block)
+        var = :"@#{name}"
+        define_method(name) do
+          unless instance_variable_defined?(var)
+            if !value.nil?
+              instance_variable_set(var, value)
+            elsif block_given?
+              instance_variable_set(var, instance_eval(&block))
+            end
+          end
+          instance_variable_get(var)
+        end
+      end
+
+      def helpers(*extensions, &block)
+        class_eval(&block) if block_given?
+        include(*extensions) if extensions.any?
+      end
+
+      def before(&block)
+        (hooks[:before] ||= []) << block
+      end
+
+      def after(&block)
+        (hooks[:after] ||= []) << block
+      end
+
+      def around(&block)
+        (hooks[:around] ||= []) << block
+      end
+    end
+
+    set :worker, :default
+    set :volatile, false
+
+    extend Forwardable
+    def_delegators :task, :application, :name, :scope
+    def_delegators 'self.class', :params_spec
+
+    attr_reader :prerequisites
+    attr_reader :params
+
+    def initialize(params)
+      @prerequisites = []
+
+      @params = ParamsParser.parse(params, params_spec)
+      @params = @params.sort.to_h # avoid ambiguity of keys order
+      @params.freeze
+    end
+
+    def task
+      self.class.task
+    end
+
+    def setup
+      # noop
+    end
+
+    def run
+      # noop
+    end
+
+    def needs(task_name, **params)
+      @prerequisites |= [[task_name, params]]
+    end
+
+    def significant_params
+      [
+        name,
+        params.select { |p, _| params_spec[p][:significant] }
+      ]
+    end
+
+    def hash
+      significant_params.hash
+    end
+
+    def eql?(other)
+      other.is_a?(Abid::Play) && \
+        significant_params.eql?(other.significant_params)
+    end
+
+    def volatile?
+      volatile
+    end
+
+    def preview?
+      application.options.preview
+    end
+
+    def invoke
+      self.class.hooks[:before].each { |blk| instance_eval(&blk) }
+
+      call_around_hooks(self.class.hooks[:around]) { run }
+
+      self.class.hooks[:after].each { |blk| instance_eval(&blk) }
+    end
+
+    def call_around_hooks(hooks, &body)
+      if hooks.empty?
+        body.call
+      else
+        h, *rest = hooks
+        instance_exec(-> { call_around_hooks(rest, &body) }, &h)
+      end
+    end
+    private :call_around_hooks
+  end
+end

data/lib/abid/rake_extensions.rb

@@ -0,0 +1,6 @@
+module Abid
+  module RakeExtensions
+    require 'abid/rake_extensions/task'
+    Rake::Task.include Task
+  end
+end

data/lib/abid/rake_extensions/task.rb

@@ -0,0 +1,135 @@
+module Abid
+  module RakeExtensions
+    module Task
+      def volatile?
+        true
+      end
+
+      def worker
+        :default
+      end
+
+      def state
+        @state ||= State.find(self)
+      end
+
+      def async_invoke(*args)
+        task_args = Rake::TaskArguments.new(arg_names, args)
+        async_invoke_with_call_chain(task_args, Rake::InvocationChain::EMPTY)
+      end
+
+      def async_invoke_with_call_chain(task_args, invocation_chain)
+        state.reload
+        new_chain = Rake::InvocationChain.append(self, invocation_chain)
+        @lock.synchronize do
+          if application.futures.include?(object_id)
+            return application.futures[object_id]
+          end
+
+          application.trace "** Invoke #{name}" if application.options.trace
+
+          preq_futures = async_invoke_prerequisites(task_args, new_chain)
+
+          future = async_invoke_after_prerequisites(task_args, preq_futures)
+
+          application.futures[object_id] = future
+        end
+      end
+
+      def async_invoke_prerequisites(task_args, invocation_chain)
+        # skip if successed
+        if state.successed?
+          if !application.options.check_prerequisites
+            preqs = []
+          else
+            preqs = prerequisite_tasks.reject(&:volatile?)
+          end
+        else
+          preqs = prerequisite_tasks
+        end
+
+        preqs.map do |p|
+          preq_args = task_args.new_scope(p.arg_names)
+          p.async_invoke_with_call_chain(preq_args, invocation_chain)
+        end
+      end
+
+      def async_invoke_after_prerequisites(task_args, preq_futures)
+        if preq_futures.empty?
+          async_execute_with_session(task_args, false)
+        else
+          result = Concurrent::IVar.new
+          counter = Concurrent::DependencyCounter.new(preq_futures.size) do
+            begin
+              failed_preq = preq_futures.find(&:rejected?)
+              next result.fail(failed_preq.reason) if failed_preq
+
+              preq_updated = preq_futures.map(&:value!).any?
+
+              future = async_execute_with_session(task_args, preq_updated)
+
+              future.add_observer do |_time, value, reason|
+                reason.nil? ? result.set(value) : result.fail(reason)
+              end
+            rescue Exception => err
+              result.fail(err)
+            end
+          end
+          preq_futures.each { |p| p.add_observer counter }
+          result
+        end
+      end
+
+      def async_execute_with_session(task_args, prerequisites_updated = false)
+        if (state.successed? && !prerequisites_updated) || !needed?
+          application.trace "** Skip #{name}" if application.options.trace
+          return Concurrent::IVar.new(false)
+        end
+
+        session_started = state.start_session
+
+        return async_wait_complete unless session_started
+
+        pool = application.worker[worker]
+        future = Concurrent::Future.execute(executor: pool) do
+          begin
+            execute(task_args)
+            true
+          ensure
+            state.close_session($ERROR_INFO)
+          end
+        end
+      ensure
+        # close session if error occurred outside the future
+        if session_started && future.nil? && $ERROR_INFO
+          state.close_session($ERROR_INFO)
+        end
+      end
+
+      def async_wait_complete
+        unless application.options.wait_external_task
+          err = RuntimeError.new("task #{name} already running")
+          return Concurrent::IVar.new.fail(err)
+        end
+
+        application.trace "** Wait #{name}" if application.options.trace
+
+        pool = application.worker[:waiter]
+        Concurrent::Future.execute(executor: pool) do
+          interval = application.options.wait_external_task_interval || 10
+          timeout = application.options.wait_external_task_timeout || 3600
+          timeout_tm = Time.now.to_f + timeout
+
+          loop do
+            state.reload
+            break unless state.running?
+
+            sleep interval
+            break if Time.now.to_f >= timeout_tm
+          end
+          true
+        end
+      end
+    end
+  end
+end

data/lib/abid/state.rb

@@ -0,0 +1,153 @@
+module Abid
+  class State
+    extend Forwardable
+
+    RUNNING = 1
+    SUCCESSED = 2
+    FAILED = 3
+
+    STATES = constants.map { |c| [const_get(c), c] }.to_h
+
+    class <<self
+      def find(task)
+        new(task)
+      end
+
+      def list(pattern: nil, started_before: nil, started_after: nil)
+        dataset = Rake.application.database[:states]
+
+        dataset = dataset.where { start_time < started_before } if started_before
+        dataset = dataset.where { start_time > started_after } if started_after
+        dataset = dataset.order(:start_time)
+
+        dataset.map do |record|
+          next if pattern && record[:name] !~ pattern
+          {
+            id: record[:id],
+            name: record[:name],
+            params: deserialize(record[:params]),
+            state: STATES[record[:state]],
+            start_time: record[:start_time],
+            end_time: record[:end_time]
+          }
+        end.compact
+      end
+
+      def serialize(params)
+        YAML.dump(params)
+      end
+
+      def deserialize(bytes)
+        YAML.load(bytes)
+      end
+    end
+
+    def_delegators 'self.class', :serialize, :deserialize
+
+    def initialize(task)
+      @task = task
+      reload
+    end
+
+    def database
+      Rake.application.database
+    end
+
+    def dataset
+      database[:states]
+    end
+
+    def disabled?
+      @task.volatile? || Rake.application.options.disable_state
+    end
+
+    def reload
+      return if disabled?
+
+      if @record
+        id = @record[:id]
+        @record = dataset.where(id: id).first
+      else
+        @record = dataset.where(digest: digest).to_a.find do |r|
+          [@task.name, @task.params].eql? [r[:name], deserialize(r[:params])]
+        end
+      end
+    end
+
+    def id
+      @record[:id] if @record
+    end
+
+    def state
+      @record[:state] if @record
+    end
+
+    def running?
+      state == RUNNING
+    end
+
+    def successed?
+      state == SUCCESSED
+    end
+
+    def failed?
+      state == FAILED
+    end
+
+    def revoke
+      fail 'cannot revoke volatile task' if disabled?
+
+      database.transaction do
+        reload
+        fail 'task is not executed yet' if id.nil?
+        fail 'task is now running' if running?
+        dataset.where(id: id).delete
+      end
+
+      @record = nil
+    end
+
+    def start_session
+      return true if disabled?
+
+      database.transaction do
+        reload
+
+        return false if running?
+
+        new_state = {
+          state: RUNNING,
+          start_time: Time.now,
+          end_time: nil
+        }
+
+        if @record
+          dataset.where(id: @record[:id]).update(new_state)
+          @record = @record.merge(new_state)
+        else
+          id = dataset.insert(
+            digest: digest,
+            name: @task.name,
+            params: serialize(@task.params),
+            **new_state
+          )
+          @record = { id: id, **new_state }
+        end
+
+        true
+      end
+    end
+
+    def close_session(error = nil)
+      return if disabled?
+      return unless @record
+      state = error ? FAILED : SUCCESSED
+      dataset.where(id: @record[:id]).update(state: state, end_time: Time.now)
+      reload
+    end
+
+    def digest
+      Digest::MD5.hexdigest(@task.name + "\n" + serialize(@task.params))
+    end
+  end
+end