abid 0.3.0.pre.alpha.3 → 0.3.0.pre.alpha.4

data/lib/abid/environment.rb

@@ -1,7 +1,10 @@
+require 'forwardable'
 require 'monitor'
 
 module Abid
   class Environment
+    extend Forwardable
+
     def initialize
       @mon = Monitor.new
     end
@@ -10,39 +13,21 @@ module Abid
       @application ||= Abid::Application.new(self)
     end
     attr_writer :application
-
-    def options
-      application.options
-    end
+    def_delegators :application, :options, :logger
 
     def config
-      @mon.synchronize do
-        @cofig ||= Config.new
-      end
-    end
-
-    def job_manager
-      @mon.synchronize do
-        @job_manager ||= JobManager.new(self)
-      end
-    end
-
-    def process_manager
-      @mon.synchronize do
-        @process_manager ||= Engine::ProcessManager.new(self)
-      end
+      return @config if @config
+      @mon.synchronize { @cofig ||= Config.new }
     end
 
-    def worker_manager
-      @mon.synchronize do
-        @worker_manager ||= Engine::WorkerManager.new(self)
-      end
+    def engine
+      return @engine if @engine
+      @mon.synchronize { @engine ||= Engine.new(self) }
     end
 
-    def db
-      @mon.synchronize do
-        @db ||= StateManager::Database.new(self)
-      end
+    def state_manager
+      return @state_manager if @state_manager
+      @mon.synchronize { @state_manager ||= StateManager.new(self) }
     end
   end
 end

data/lib/abid/error.rb

@@ -2,4 +2,6 @@ module Abid
   class Error < StandardError; end
 
   class AlreadyRunningError < Error; end
+
+  class NoParamError < NameError; end
 end

data/lib/abid/params_format.rb

@@ -1,3 +1,5 @@
+require 'date'
+require 'time'
 require 'shellwords'
 
 module Abid
@@ -10,6 +12,10 @@ module Abid
       end.join(' ')
     end
 
+    def self.format_with_name(name, params)
+      name + (params.empty? ? '' : ' ' + format(params))
+    end
+
     def self.format_value(value)
       case value
       when Numeric, TrueClass, FalseClass
@@ -25,20 +31,17 @@ module Abid
       end
     end
 
-    def self.parse_args(args)
-      tasks = []
-      params = {}
-
-      args.each do |arg|
-        m = arg.match(/^(\w+)=(.*)$/m)
-        if m
-          params.store(m[1].to_sym, parse_value(m[2]))
-        else
-          tasks << arg unless arg =~ /^-/
-        end
+    def self.collect_params(args)
+      args.each_with_object([{}, []]) do |arg, (params, extras)|
+        key, val = parse_pair(arg)
+        key.nil? ? extras << arg : params[key] = val
       end
+    end
 
-      [tasks, params]
+    def self.parse_pair(str)
+      m = str.match(/^(\w+)=(.*)$/)
+      return unless m
+      [m[1].to_sym, parse_value(m[2])]
     end
 
     def self.parse_value(value)
@@ -53,9 +56,23 @@ module Abid
         Date.parse(value)
       when /\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}( \d{4})?\z/
         Time.parse(value)
+      when /\A(["']).*\1\z/
+        value[1..-2]
       else
         value
       end
     end
+
+    SUPPORTED_TYPES = [
+      Numeric, TrueClass, FalseClass, Date, Time, DateTime, String
+    ].freeze
+
+    def self.validate_params!(params)
+      params.values.each do |value|
+        valid = SUPPORTED_TYPES.any? { |t| value.is_a? t }
+        raise Error, "#{value.class} class is not supported" unless valid
+      end
+      params
+    end
   end
 end

data/lib/abid/rake_extensions.rb

@@ -1,5 +1,3 @@
-require 'abid/rake_extensions/task'
-
 # Delegate Rake.application to Abid.global.application
 class << Rake
   def application
@@ -11,4 +9,14 @@ class << Rake
   end
 end
 
-Rake::Task.include Abid::RakeExtensions::Task
+module Rake
+  class Task
+    def bind(params = {})
+      Abid::DSL::RakeJob.new(self, params)
+    end
+
+    def params_spec
+      {}
+    end
+  end
+end

data/lib/abid/state_manager.rb

@@ -0,0 +1,40 @@
+require 'sequel'
+require 'abid/state_manager/state'
+require 'abid/state_manager/state_service'
+
+module Abid
+  class StateManager
+    Sequel.extension :migration
+
+    MIGRATIONS_PATH = File.expand_path('../../../migrations', __FILE__)
+
+    # Creates a new database object and checks schema version.
+    #
+    # @return [Sequel::Database] database object
+    # @see Sequel.connect
+    def self.connect(*args)
+      db = Sequel.connect(*args)
+      Sequel::Migrator.check_current(db, MIGRATIONS_PATH)
+      db
+    rescue Sequel::Migrator::NotCurrentError
+      raise Error, 'current schema is out of date'
+    end
+
+    def initialize(env)
+      @env = env
+      @db = self.class.connect(@env.config.database)
+      @states = State.connect(@db)
+    end
+    attr_reader :db, :states
+
+    # Returns StateService object from name and params.
+    # @param name [String] task name
+    # @param params [Hash]
+    # @return [StateManager::StateService]
+    def state(name, params, dryrun: false, volatile: false)
+      return VolatileStateService.new(@states, name, params) if volatile
+      return NullStateService.new(@states, name, params) if dryrun
+      StateService.new(@states, name, params)
+    end
+  end
+end

data/lib/abid/state_manager/state.rb

@@ -1,26 +1,26 @@
+require 'yaml'
+
 module Abid
-  module StateManager
-    State = Class.new(Sequel::Model)
+  class StateManager
+    State = Class.new(Sequel::Model) do # delay Sequel::Model.set_dataset
+      def self.connect(database)
+        Class.new(self.Model(database[:states]))
+      end
+    end
 
     # O/R Mapper for `states` table.
-    #
-    # Use #init_by_job to initialize a state object.
     class State
       RUNNING = 1
       SUCCESSED = 2
       FAILED = 3
 
-      # @!method self.filter_by_start_time(after: nil, before: nil)
-      #   @param after [Time] lower bound of start_time
-      #   @param before [Time] upper bound of start_time
-      #   @return [Sequel::Dataset<State>] a set of states started between
-      #     the given range.
-      #
-      # @!method self.filter_by_prefix(prefix)
-      #   @param prefix [String] the prefix of task names
-      #   @return [Sequel::Dataset<State>] a set of states which name starts
-      #     with the given prefix.
+      @dataset = nil
+
       dataset_module do
+        #   @param after [Time] lower bound of start_time
+        #   @param before [Time] upper bound of start_time
+        #   @return [Sequel::Dataset<State>] a set of states started between
+        #     the given range.
         def filter_by_start_time(after: nil, before: nil)
           dataset = self
           dataset = dataset.where { start_time >= after } if after
@@ -28,112 +28,15 @@ module Abid
           dataset
         end
 
+        #   @param prefix [String] the prefix of task names
+        #   @return [Sequel::Dataset<State>] a set of states which name starts
+        #     with the given prefix.
         def filter_by_prefix(prefix)
           return self if prefix.nil?
           where { Sequel.like(:name, prefix + '%') }
         end
       end
 
-      # Find a state by the job.
-      #
-      # @param job [Job] job
-      # @return [State] state object
-      def self.find_by_job(job)
-        where(
-          name: job.name,
-          params: job.params_str,
-          digest: job.digest
-        ).first
-      end
-
-      # Initialize a state by a job.
-      #
-      # @param job [Job] job
-      # @return [State] state object
-      def self.init_by_job(job)
-        new(
-          name: job.name,
-          params: job.params_str,
-          digest: job.digest
-        )
-      end
-
-      # Find or initialize a state by a job.
-      #
-      # @param job [Job] job
-      # @return [State] state object
-      def self.find_or_init_by_job(job)
-        find_by_job(job) || init_by_job(job)
-      end
-
-      # Update the state to RUNNING.
-      #
-      # @param job [Job] job
-      def self.start(job)
-        db.transaction do
-          state = find_or_init_by_job(job)
-          state.check_running!
-          state.state = RUNNING
-          state.start_time = Time.now
-          state.end_time = nil
-          state.save
-        end
-      end
-
-      # Update the state to SUCCESSED or FAILED.
-      #
-      # If error is given, the state will be FAILED.
-      #
-      # @param job [Job] job
-      # @param error [Error] error object
-      def self.finish(job, error = nil)
-        db.transaction do
-          state = find_or_init_by_job(job)
-          return unless state.running?
-
-          state.state = error ? FAILED : SUCCESSED
-          state.end_time = Time.now
-          state.save
-        end
-      end
-
-      # Assume the job to be successed
-      #
-      # If the force option is true, update the state to SUCCESSED even if the
-      # task is running.
-      #
-      # @param job [Job] job
-      # @param force [Boolean] force update the state
-      # @return [void]
-      def self.assume(job, force: false)
-        db.transaction do
-          state = find_or_init_by_job(job)
-          return state if state.successed?
-          state.check_running! unless force
-
-          state.state = SUCCESSED
-          state.start_time = Time.now
-          state.end_time = Time.now
-          state.save
-          state
-        end
-      end
-
-      # Delete the state.
-      #
-      # @param state_id [Integer] State ID
-      # @param force [Boolean] If true, delete the state even if running
-      # @return [void]
-      def self.revoke(state_id, force: false)
-        db.transaction do
-          state = self[state_id]
-          return false if state.nil?
-          state.check_running! unless force
-          state.delete
-          true
-        end
-      end
-
       # check if the state is running
       def check_running!
         raise AlreadyRunningError, 'job already running' if running?
@@ -164,6 +67,41 @@ module Abid
         return unless start_time && end_time
         end_time - start_time
       end
+
+      # Update the state to RUNNING.
+      def start
+        check_running!
+        update(state: RUNNING, start_time: Time.now, end_time: nil)
+      end
+
+      # Update the state to SUCCESSED or FAILED.
+      # If error is given, the state will be FAILED.
+      def finish(error = nil)
+        return unless running?
+        update(state: error ? FAILED : SUCCESSED, end_time: Time.now)
+      end
+
+      # Assume the job to be successed
+      #
+      # If the `force` option is true, update the state to SUCCESSED even if the
+      # task is running.
+      def assume(force: false)
+        return if successed?
+        check_running! unless force
+        time = Time.now
+        update(state: SUCCESSED, start_time: time, end_time: time)
+      end
+
+      # Delete the state.
+      # @param force [Boolean] If true, delete the state even if running
+      def revoke(force: false)
+        check_running! unless force
+        delete
+      end
+
+      def params_hash
+        YAML.load(params)
+      end
     end
   end
 end

data/lib/abid/state_manager/state_service.rb

@@ -0,0 +1,88 @@
+require 'yaml'
+
+module Abid
+  class StateManager
+    # StateService updates the job status in a transaction.
+    class StateService
+      def initialize(model, name, params)
+        @model = model
+        @name = name
+        @params = params
+      end
+      attr_reader :name, :params
+
+      # @return [State] state
+      def find
+        cond = { name: name, params: params_text, digest: digest }
+        @model.where(cond).first || @model.new(cond)
+      end
+
+      # @see State#start
+      def start
+        transaction { find.start }
+        self
+      end
+
+      # Try to update the state to started unless volatile.
+      # @see State#start
+      # @return [true,false] false if already running
+      def try_start
+        start
+        true
+      rescue AlreadyRunningError
+        false
+      end
+
+      # @see State#finish
+      def finish(error = nil)
+        transaction { find.finish(error) }
+        self
+      end
+
+      def assume(force: false)
+        find.assume(force: force)
+        self
+      end
+
+      private
+
+      def transaction(&block)
+        @model.db.transaction(
+          isolation: :serializable,
+          retry_on: Sequel::SerializationFailure,
+          &block
+        )
+      end
+
+      def params_text
+        @params_text ||= YAML.dump(params.sort.to_h)
+      end
+
+      def digest
+        @digest ||= Digest::MD5.hexdigest(name + "\n" + params_text)
+      end
+    end
+
+    # NullStateService does not update job status.
+    class NullStateService < StateService
+      def start
+        self
+      end
+
+      def finish(_ = nil)
+        self
+      end
+
+      def assume(_ = {})
+        self
+      end
+    end
+
+    # VolatileStateService never access to database.
+    class VolatileStateService < NullStateService
+      def find
+        @model.new(name: name, params: params_text, digest: digest)
+      end
+    end
+  end
+end