abid 0.3.0.pre.alpha.3 → 0.3.0.pre.alpha.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 53398cf57eddf9c3cc6de80acce35fa592c4d00a
-  data.tar.gz: a89e82cc2adb5d59cc1bbf2697ae21f850f5fa29
+  metadata.gz: 1596801ffb3b1eb65f3afcabef8320657e25b53a
+  data.tar.gz: d1eca9882b7e33183b8c7fe1664249e10e7823f6
 SHA512:
-  metadata.gz: 06e97ecf6eeb098f101386351b081224b0a665278088e4b266f059603b69ffe8778485a8d9519c18367ee5c964f9c8c8a9c6a56b167da352d2131f38d2ecc883
-  data.tar.gz: 0d20bb8a5032cbec0b2f89416cf8dd872dcce97585685b59447cfb716915423cdd9f25355518e18dc5fc0e2c0e68f0358dad945aa551d79769846366ac5ef9d9
+  metadata.gz: 08fd18e4a5280106ee3e23a2e422692428b211f3f35de959512f9bf1a4cde6b75408dbfdbd7ba32d37db6c56177a891bd105fbf25ff05a76865a8975fecef5ad
+  data.tar.gz: 0df0a4df266bf2e69f755a1aff2458b2bbbb7d94f3fffc106ae476c2bff241e1f86d336158cb03de19ed0698700f5f43e7b1ca8e4272b945a630ced6cfc30ae2

data/README.md

@@ -131,7 +131,7 @@ Abid plays can also be volatile.
 
 ```
 play :voaltile_play do
-  set :volatile, true
+  volatile
   def run
     ...
   end
@@ -164,14 +164,14 @@ define_worker :copy, 2
 define_worker :hive, 4
 
 play :copy_source_1 do
-  set :worker, :copy
+  worker :copy
   def run
     ...
   end
 end
 
 play :hive_query_1 do
-  set :worker, :hive
+  worker :hive
   def run
     ...
   end

data/abid.gemspec

@@ -24,10 +24,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'pry-byebug'
   spec.add_development_dependency 'yard'
 
-  spec.add_dependency 'rake', '~> 10.0'
+  spec.add_dependency 'rake'
   spec.add_dependency 'concurrent-ruby-ext'
   spec.add_dependency 'sequel'
   spec.add_dependency 'sqlite3'
-  spec.add_dependency 'rbtree'
   spec.add_dependency 'thor'
 end

data/exe/abidsc

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-require 'abid/cli'
 require 'abid'
+require 'abid/cli'
 
 Abid::CLI.start(ARGV)

data/lib/abid.rb

@@ -1,45 +1,18 @@
 require 'rake'
-require 'date'
-require 'digest/md5'
 require 'English'
-require 'forwardable'
-require 'monitor'
-require 'time'
-require 'yaml'
-require 'concurrent'
-require 'rbtree'
-require 'sqlite3'
-require 'sequel'
 
+require 'abid/application'
 require 'abid/config'
 require 'abid/environment'
 require 'abid/error'
 require 'abid/engine'
-require 'abid/engine/process'
-require 'abid/engine/process_manager'
-require 'abid/engine/executor'
-require 'abid/engine/scheduler'
-require 'abid/engine/worker_manager'
-require 'abid/engine/waiter'
 require 'abid/params_format'
-require 'abid/state_manager/database'
-require 'abid/job'
-require 'abid/job_manager'
-
 require 'abid/rake_extensions'
+require 'abid/state_manager'
+require 'abid/status'
 require 'abid/version'
-require 'abid/params_parser'
-require 'abid/play_core'
-require 'abid/play'
-require 'abid/task'
-require 'abid/mixin_task'
-require 'abid/task_manager'
-require 'abid/dsl_definition'
-require 'abid/application'
 
 module Abid
-  FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) # :nodoc:
-
   class << self
     def global
       @global ||= Environment.new

data/lib/abid/application.rb

@@ -1,30 +1,60 @@
+require 'logger'
+require 'abid/dsl/abid_job'
+require 'abid/dsl/actions'
+require 'abid/dsl/mixin'
+require 'abid/dsl/params_spec'
+require 'abid/dsl/play_core'
+require 'abid/dsl/play'
+require 'abid/dsl/rake_job'
+require 'abid/dsl/syntax'
+require 'abid/dsl/job'
+require 'abid/dsl/job_manager'
+require 'abid/dsl/task'
+
 module Abid
   class Application < Rake::Application
-    include Abid::TaskManager
-
     def initialize(env)
       super()
       @rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb)
       @env = env
+      @global_params = {}
+      @global_mixin = DSL::Mixin.create_global_mixin
+      @job_manager = DSL::JobManager.new(self)
+      @after_all_actions = []
     end
+    attr_reader :global_params, :global_mixin, :job_manager, :after_all_actions
 
     def init
       super
       @env.config.load(options.config_file)
     end
 
-    def run_with_threads
-      yield
-    rescue Exception => err
-      Engine.kill(err)
-      raise err
-    else
-      Engine.shutdown
+    def top_level
+      if options.show_tasks || options.show_prereqs
+        super
+      else
+        run_with_engine { invoke_top_level_tasks }
+      end
     end
 
-    def invoke_task(task_string) # :nodoc:
-      name, args = parse_task_string(task_string)
-      Job.find_by_task(self[name]).invoke(*args)
+    def invoke_top_level_tasks
+      top_level_tasks.each do |task_string|
+        name, args = parse_task_string(task_string)
+        @env.engine.invoke(name, *args)
+        break unless @env.engine.errors.empty?
+      end
+    end
+
+    def run_with_engine
+      yield
+      @env.engine.shutdown
+    rescue Exception => exception
+      @env.engine.kill(exception)
+      raise
+    else
+      raise @env.engine.errors.first unless @env.engine.errors.empty?
+    ensure
+      call_after_all_actions
     end
 
     def standard_rake_options
@@ -56,7 +86,16 @@ module Abid
            proc { options.preview = true }],
           ['--wait-external-task',
            'Wait a task finished if it is running in externl process',
-           proc { options.wait_external_task = true }]
+           proc { options.wait_external_task = true }],
+          ['--log-level LEVEL',
+           'Specifies the log level. LEVEL can be error, warn, info or debug.' \
+           ' (default: info)',
+           proc do |v|
+             options.log_level = Logger::Severity.const_get(v.upcase)
+           end],
+          ['--[no-]logging',
+           'Enable logging. (default: on)',
+           proc { |v| options.logging = v }]
         ]
       )
     end
@@ -64,6 +103,8 @@ module Abid
     def handle_options
       options.rakelib = %w(rakelib tasks)
       options.trace_output = $stderr
+      options.log_level = Logger::Severity::INFO
+      options.logging = true
 
       OptionParser.new do |opts|
         opts.banner = 'See full documentation at https://github.com/ojima-h/abid.'
@@ -88,5 +129,34 @@ module Abid
         opts.environment('RAKEOPT')
       end.parse!
     end
+
+    def collect_command_line_tasks(args)
+      params, = ParamsFormat.collect_params(args)
+      @global_params.update(params)
+      super
+    end
+
+    #
+    # Abid Extentions
+    #
+    def logger
+      return @logger if @logger
+      logdev = options.logging ? $stderr : nil
+      @logger = Logger.new(logdev).tap do |l|
+        l.progname = 'abid'
+        l.level = options.log_level || Logger::Severity::INFO
+      end
+    end
+    attr_writer :logger
+
+    def call_after_all_actions
+      @after_all_actions.each do |block|
+        block.call(
+          top_level_tasks,
+          @env.engine.summary,
+          @env.engine.errors
+        )
+      end
+    end
   end
 end

data/lib/abid/cli/assume.rb

@@ -10,21 +10,20 @@ module Abid
       end
 
       def run
-        tasks, params = ParamsFormat.parse_args(@args)
+        params, tasks = ParamsFormat.collect_params(@args)
 
         tasks.each { |task| assume(task, params) }
       end
 
       def assume(task, params)
-        params_str = ParamsFormat.format(params)
+        state = @env.state_manager.state(task, params)
+        state.assume(force: @force)
 
-        job = @env.job_manager[task, params]
-        state = job.state.assume(force: @force)
-
-        puts "#{task} #{params_str} (id: #{state.id})" \
-             ' is assumed to be SUCCESSED.'
+        s = state.find
+        n = ParamsFormat.format_with_name(task, params)
+        puts "#{n} (id: #{s.id}) is assumed to be SUCCESSED."
       rescue AlreadyRunningError
-        $stderr.puts "#{task} #{params_str} already running.\n" \
+        $stderr.puts "#{n} already running.\n" \
                      'Use -f option if you want to force assume.'
         raise
       end

data/lib/abid/cli/list.rb

@@ -17,7 +17,7 @@ module Abid
       end
 
       def build_table
-        states = @env.db.states
+        states = @env.state_manager.states
                      .filter_by_prefix(@prefix)
                      .filter_by_start_time(after: @after, before: @before)
         table = states.map { |state| format_state(state) }
@@ -31,7 +31,7 @@ module Abid
           t,
           state.state_label,
           format_exec_time(state.exec_time),
-          state.name + ' ' + ParamsFormat.format(YAML.load(state.params))
+          ParamsFormat.format_with_name(state.name, state.params_hash)
         ]
       end
 

data/lib/abid/cli/migrate.rb

@@ -8,7 +8,7 @@ module Abid
 
       def run
         db = Sequel.connect(@env.config.database)
-        dir = StateManager::Database::MIGRATIONS_PATH
+        dir = StateManager::MIGRATIONS_PATH
 
         if Sequel::Migrator.is_current?(db, dir)
           puts 'Schema is latest.'

data/lib/abid/cli/revoke.rb

@@ -14,30 +14,29 @@ module Abid
 
       def run
         @job_ids.each do |job_id|
-          state = @env.db.states[job_id]
+          state = @env.state_manager.states[job_id]
           if state.nil?
             $stderr.puts "#{job_id} is not found"
             next
           end
 
-          next if !@quiet && !ask(state)
+          text = ParamsFormat.format_with_name(state.name, state.params_hash)
+          next if !@quiet && !ask(text)
 
-          revoke(state)
+          revoke(state, text)
         end
       end
 
-      def revoke(state)
-        @env.db.states.revoke(state.id, force: @force)
+      def revoke(state, text)
+        state.revoke(force: @force)
         puts "revoked #{state.id}"
       rescue AlreadyRunningError
-        params = ParamsFormat.format(YAML.load(state.params))
-        $stderr.puts "#{state.name} #{params} already running.\n" \
+        $stderr.puts "#{text} already running.\n" \
                      'Use -f option if you want to force assume.'
       end
 
-      def ask(state)
-        params = ParamsFormat.format(YAML.load(state.params))
-        print "revoke task \`#{state.name} #{params}'? "
+      def ask(text)
+        print "revoke task \`#{text}'? "
         $stdout.flush
         ret = $stdin.gets
         ret.match(/y(es)?/i)

data/lib/abid/config.rb

@@ -1,3 +1,5 @@
+require 'yaml'
+
 module Abid
   class Config < Hash
     DEFAULT_DATABASE_CONFIG = {

data/lib/abid/dsl/abid_job.rb

@@ -0,0 +1,58 @@
+require 'abid/dsl/job'
+
+module Abid
+  module DSL
+    # Abid::DSL::Task wrapper
+    class AbidJob < Job
+      def_delegators :play, :worker
+      def_delegator :play, :concerned, :concerned?
+      def_delegator :play, :needed,    :needed?
+
+      def initialize(task, params)
+        super
+        @play = task.internal.new(params)
+        @play.call_action(:setup)
+      end
+      attr_reader :play
+      private :play
+
+      def execute(args)
+        if dryrun?
+          trace_execute
+          return
+        end
+        play.call_action(:safe_action)
+        play.call_action(:action, args) unless preview?
+        run(args)
+      ensure
+        play.call_action(:after, $ERROR_INFO) unless dryrun? || preview?
+      end
+
+      def run(args)
+        if play.method(:run).arity.zero?
+          play.run
+        else
+          play.run(args)
+        end
+      end
+
+      def prerequisites
+        ps = task.prerequisite_tasks.map { |preq| [preq, params] } \
+           + play.prerequisite_tasks
+
+        ps.map do |preq, params|
+          task.application.job_manager[preq.name, params]
+        end
+      end
+
+      def volatile?
+        play.volatile || task.application.options.disable_state
+      end
+
+      def trace_execute
+        task.application.trace "** Execute (dry run) #{task.name}"
+      end
+      private :trace_execute
+    end
+  end
+end

data/lib/abid/dsl/actions.rb

@@ -0,0 +1,36 @@
+module Abid
+  module DSL
+    # Actions manages action index of a play.
+    #
+    # It merges all actions of the play's ancestors.
+    class Actions
+      def initialize(play_class)
+        @play_class = play_class
+        @index = Hash.new { |h, k| h[k] = [] }
+      end
+
+      attr_reader :index
+      protected :index
+
+      # @param key [Symbol] action_name
+      # @return [Enumerator<Proc>]
+      def [](key)
+        Enumerator.new do |y|
+          # search key over including mixins
+          @play_class.superplays.reverse.each do |sp|
+            sp.actions.index[key].each { |block| y << block }
+          end
+        end
+      end
+
+      # @param key [Symbol] action name
+      # @param scope [Rake::Scope] task scope where the action is declared
+      # @param block [Proc] action body
+      # @return self
+      def add(key, scope, block)
+        @index[key] << [scope, block]
+        self
+      end
+    end
+  end
+end