abid 0.3.0.pre.alpha.2 → 0.3.0.pre.alpha.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f4721d7f5310427976f61dd1673b774c9881bfeb
4
- data.tar.gz: aa2d329aae16145039c4bb28b57d50d3c98dd76b
3
+ metadata.gz: 53398cf57eddf9c3cc6de80acce35fa592c4d00a
4
+ data.tar.gz: a89e82cc2adb5d59cc1bbf2697ae21f850f5fa29
5
5
  SHA512:
6
- metadata.gz: 095800154b3abfccf282351e73b66e1d02e11bdb27e01d0fc54707aa533d4da0e8745c70ecf6f0275f338afa2e3846573cab8ba0c4e25547389888f6957e812b
7
- data.tar.gz: fbd9d3d6b3e1975310a39e59d0b7a0b8a22974ec0eb74fe674f3b6702d1f62cd9537259cacb3714c5cd743830a595c4ce6f57a5c4f316068e8d06b0e22a1389e
6
+ metadata.gz: 06e97ecf6eeb098f101386351b081224b0a665278088e4b266f059603b69ffe8778485a8d9519c18367ee5c964f9c8c8a9c6a56b167da352d2131f38d2ecc883
7
+ data.tar.gz: 0d20bb8a5032cbec0b2f89416cf8dd872dcce97585685b59447cfb716915423cdd9f25355518e18dc5fc0e2c0e68f0358dad945aa551d79769846366ac5ef9d9
data/exe/abid CHANGED
@@ -1,4 +1,4 @@
1
1
  #!/usr/bin/env ruby
2
2
  require 'abid'
3
3
 
4
- Abid::Application.new.run
4
+ Abid.global.application.run
data/lib/abid.rb CHANGED
@@ -12,21 +12,25 @@ require 'sqlite3'
12
12
  require 'sequel'
13
13
 
14
14
  require 'abid/config'
15
+ require 'abid/environment'
15
16
  require 'abid/error'
17
+ require 'abid/engine'
18
+ require 'abid/engine/process'
19
+ require 'abid/engine/process_manager'
20
+ require 'abid/engine/executor'
21
+ require 'abid/engine/scheduler'
22
+ require 'abid/engine/worker_manager'
23
+ require 'abid/engine/waiter'
16
24
  require 'abid/params_format'
17
- require 'abid/state_manager'
25
+ require 'abid/state_manager/database'
18
26
  require 'abid/job'
27
+ require 'abid/job_manager'
19
28
 
20
29
  require 'abid/rake_extensions'
21
30
  require 'abid/version'
22
- require 'abid/abid_module'
23
- require 'abid/waiter'
24
- require 'abid/worker'
25
31
  require 'abid/params_parser'
26
32
  require 'abid/play_core'
27
33
  require 'abid/play'
28
- require 'abid/session'
29
- require 'abid/state'
30
34
  require 'abid/task'
31
35
  require 'abid/mixin_task'
32
36
  require 'abid/task_manager'
@@ -35,4 +39,15 @@ require 'abid/application'
35
39
 
36
40
  module Abid
37
41
  FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) # :nodoc:
42
+
43
+ class << self
44
+ def global
45
+ @global ||= Environment.new
46
+ end
47
+ attr_writer :global
48
+
49
+ def application
50
+ global.application
51
+ end
52
+ end
38
53
  end
@@ -2,55 +2,36 @@ module Abid
2
2
  class Application < Rake::Application
3
3
  include Abid::TaskManager
4
4
 
5
- attr_reader :worker
6
-
7
- def initialize
8
- super
9
- @rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb) << abidfile
10
- @worker = Worker.new(self)
11
-
12
- Abid.config.load
13
- end
14
-
15
- def run
16
- Abid.application = self
17
- super
18
- end
19
-
20
- # allows the built-in tasks to load without a abidfile
21
- def abidfile
22
- File.expand_path(File.join(File.dirname(__FILE__), '..', 'Abidfile.rb'))
5
+ def initialize(env)
6
+ super()
7
+ @rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb)
8
+ @env = env
23
9
  end
24
10
 
25
- # load built-in tasks
26
- def load_rakefile
27
- standard_exception_handling do
28
- glob(File.expand_path('../tasks/*.rake', __FILE__)) do |name|
29
- Rake.load_rakefile name
30
- end
31
- end
11
+ def init
32
12
  super
13
+ @env.config.load(options.config_file)
33
14
  end
34
15
 
35
16
  def run_with_threads
36
17
  yield
37
18
  rescue Exception => err
38
- worker.kill
19
+ Engine.kill(err)
39
20
  raise err
40
21
  else
41
- worker.shutdown
22
+ Engine.shutdown
42
23
  end
43
24
 
44
25
  def invoke_task(task_string) # :nodoc:
45
26
  name, args = parse_task_string(task_string)
46
- self[name].async_invoke(*args).wait!
27
+ Job.find_by_task(self[name]).invoke(*args)
47
28
  end
48
29
 
49
30
  def standard_rake_options
50
31
  super.each do |opt|
51
32
  case opt.first
52
33
  when '--execute-print'
53
- # disable short option
34
+ # disable short option (-p)
54
35
  opt.delete_at(1)
55
36
  when '--version'
56
37
  opt[-1] = lambda do |_value|
@@ -61,25 +42,21 @@ module Abid
61
42
  end
62
43
  end
63
44
 
64
- def abid_options # :nodoc:
45
+ def abid_options
65
46
  sort_options(
66
47
  [
48
+ ['--config-file', '-C CONFIG_FILE',
49
+ 'Config file path',
50
+ proc { |v| options.config_file = v }],
67
51
  ['--repair',
68
52
  'Run the task in repair mode.',
69
- proc { options.repair = true }
70
- ],
53
+ proc { options.repair = true }],
71
54
  ['--preview', '-p',
72
55
  'Run tasks in preview mode.',
73
- proc do
74
- options.preview = true
75
- end
76
- ],
56
+ proc { options.preview = true }],
77
57
  ['--wait-external-task',
78
58
  'Wait a task finished if it is running in externl process',
79
- proc do
80
- options.wait_external_task_interval = true
81
- end
82
- ]
59
+ proc { options.wait_external_task = true }]
83
60
  ]
84
61
  )
85
62
  end
data/lib/abid/cli.rb CHANGED
@@ -6,18 +6,19 @@ module Abid
6
6
 
7
7
  def initialize(*args)
8
8
  super(*args)
9
- Abid.config.load(options[:config_file])
9
+ @env = Abid.global
10
+ @env.config.load(options[:config_file])
10
11
  end
11
12
 
12
13
  desc 'config', 'Show current config'
13
14
  def config
14
- puts Abid.config.to_yaml
15
+ puts @env.config.to_yaml
15
16
  end
16
17
 
17
18
  desc 'migrate', 'Run database migration'
18
19
  def migrate
19
20
  require 'abid/cli/migrate'
20
- Migrate.new(options).run
21
+ Migrate.new(@env, options).run
21
22
  end
22
23
 
23
24
  desc 'assume TASK [TASKS..] [PARAMS]', 'Assume the job to be SUCCESSED'
@@ -25,9 +26,9 @@ module Abid
25
26
  desc: 'set the state even if the job is running'
26
27
  def assume(task, *rest_args)
27
28
  require 'abid/cli/assume'
28
- Assume.new(options, [task, *rest_args]).run
29
+ Assume.new(@env, options, [task, *rest_args]).run
29
30
  rescue AlreadyRunningError
30
- exit 1
31
+ $stderr.puts '[WARN] task alread running.'
31
32
  end
32
33
 
33
34
  desc 'list [PREFIX]', 'List jobs'
@@ -35,7 +36,7 @@ module Abid
35
36
  option :before, type: :string, aliases: '-b', desc: 'start time filter'
36
37
  def list(prefix = nil)
37
38
  require 'abid/cli/list'
38
- List.new(options, prefix).run
39
+ List.new(@env, options, prefix).run
39
40
  end
40
41
  map ls: :list
41
42
 
@@ -46,7 +47,7 @@ module Abid
46
47
  desc: 'no prompt before removal'
47
48
  def revoke(job_id, *rest_args)
48
49
  require 'abid/cli/revoke'
49
- Revoke.new(options, [job_id, *rest_args]).run
50
+ Revoke.new(@env, options, [job_id, *rest_args]).run
50
51
  end
51
52
  map rm: :revoke
52
53
  end
@@ -1,7 +1,8 @@
1
1
  module Abid
2
2
  class CLI
3
3
  class Assume
4
- def initialize(options, args)
4
+ def initialize(env, options, args)
5
+ @env = env
5
6
  @options = options
6
7
  @args = args
7
8
 
@@ -17,8 +18,8 @@ module Abid
17
18
  def assume(task, params)
18
19
  params_str = ParamsFormat.format(params)
19
20
 
20
- job = Job.new(task, params)
21
- state = StateManager::State.assume(job, force: @force)
21
+ job = @env.job_manager[task, params]
22
+ state = job.state.assume(force: @force)
22
23
 
23
24
  puts "#{task} #{params_str} (id: #{state.id})" \
24
25
  ' is assumed to be SUCCESSED.'
data/lib/abid/cli/list.rb CHANGED
@@ -3,7 +3,8 @@ require 'abid/cli/table_formatter'
3
3
  module Abid
4
4
  class CLI
5
5
  class List
6
- def initialize(options, prefix)
6
+ def initialize(env, options, prefix)
7
+ @env = env
7
8
  @options = options
8
9
  @prefix = prefix
9
10
 
@@ -16,9 +17,9 @@ module Abid
16
17
  end
17
18
 
18
19
  def build_table
19
- states = StateManager::State
20
- .filter_by_prefix(@prefix)
21
- .filter_by_start_time(after: @after, before: @before)
20
+ states = @env.db.states
21
+ .filter_by_prefix(@prefix)
22
+ .filter_by_start_time(after: @after, before: @before)
22
23
  table = states.map { |state| format_state(state) }
23
24
  TableFormatter.new(table).format
24
25
  end
@@ -1,15 +1,14 @@
1
- require 'abid/state_manager'
2
-
3
1
  module Abid
4
2
  class CLI
5
3
  class Migrate
6
- def initialize(options)
4
+ def initialize(env, options)
5
+ @env = env
7
6
  @options = options
8
7
  end
9
8
 
10
9
  def run
11
- db = StateManager::Database.connect!
12
- dir = StateManager::Database.migrations_path
10
+ db = Sequel.connect(@env.config.database)
11
+ dir = StateManager::Database::MIGRATIONS_PATH
13
12
 
14
13
  if Sequel::Migrator.is_current?(db, dir)
15
14
  puts 'Schema is latest.'
@@ -3,7 +3,8 @@ require 'abid/cli/table_formatter'
3
3
  module Abid
4
4
  class CLI
5
5
  class Revoke
6
- def initialize(options, job_ids)
6
+ def initialize(env, options, job_ids)
7
+ @env = env
7
8
  @options = options
8
9
  @job_ids = job_ids.map(&:to_i)
9
10
 
@@ -13,9 +14,9 @@ module Abid
13
14
 
14
15
  def run
15
16
  @job_ids.each do |job_id|
16
- state = StateManager::State[job_id]
17
+ state = @env.db.states[job_id]
17
18
  if state.nil?
18
- $stderr.puts "state_id #{job_id} not found"
19
+ $stderr.puts "#{job_id} is not found"
19
20
  next
20
21
  end
21
22
 
@@ -26,7 +27,7 @@ module Abid
26
27
  end
27
28
 
28
29
  def revoke(state)
29
- state.revoke(force: @force)
30
+ @env.db.states.revoke(state.id, force: @force)
30
31
  puts "revoked #{state.id}"
31
32
  rescue AlreadyRunningError
32
33
  params = ParamsFormat.format(YAML.load(state.params))
data/lib/abid/config.rb CHANGED
@@ -21,7 +21,7 @@ module Abid
21
21
 
22
22
  # @return [Hash] database configuration
23
23
  def database
24
- self['database']
24
+ self['database'].each_with_object({}) { |(k, v), h| h[k.to_sym] = v }
25
25
  end
26
26
 
27
27
  # Load config file.
@@ -29,9 +29,9 @@ module Abid
29
29
  # If `config_file` is specified and does not exist, it raises an error.
30
30
  #
31
31
  # If `config_file` is not specified, it searches config file in
32
- # Config.earch_path.
32
+ # Config.search_path.
33
33
  #
34
- # When #load is called again, original cofnigurations is cleared.
34
+ # When #load is called again, original configurations is cleared.
35
35
  #
36
36
  # @param config_file [String] config file
37
37
  # @return [Config] self
@@ -5,11 +5,11 @@ module Abid
5
5
  end
6
6
 
7
7
  def define_worker(name, thread_count)
8
- Rake.application.worker.define(name, thread_count)
8
+ Abid.global.worker_manager.define(name, thread_count)
9
9
  end
10
10
 
11
11
  def play_base(&block)
12
- Rake.application.play_base(&block)
12
+ Abid.application.play_base(&block)
13
13
  end
14
14
 
15
15
  def helpers(*extensions, &block)
@@ -17,7 +17,7 @@ module Abid
17
17
  end
18
18
 
19
19
  def invoke(task, *args, **params)
20
- Rake.application[task, **params].async_invoke(*args).wait!
20
+ Job.find_by_task(Abid.application[task, **params]).invoke(*args)
21
21
  end
22
22
 
23
23
  def mixin(*args, &block)
@@ -0,0 +1,13 @@
1
+ module Abid
2
+ # Engine module operates task execution.
3
+ module Engine
4
+ def self.kill(error)
5
+ Abid.global.worker_manager.kill
6
+ Abid.global.process_manager.kill(error)
7
+ end
8
+
9
+ def self.shutdown
10
+ Abid.global.worker_manager.shutdown
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,126 @@
1
+ module Abid
2
+ module Engine
3
+ # @!visibility private
4
+
5
+ # Executor operates each task execution.
6
+ class Executor
7
+ def initialize(job, args)
8
+ @job = job
9
+ @args = args
10
+
11
+ @process = job.process
12
+ @state = job.state.find
13
+ @prerequisites = job.prerequisites.map(&:process)
14
+ end
15
+
16
+ # Check if the task should be executed.
17
+ #
18
+ # @return [Boolean] false if the task should not be executed.
19
+ def prepare
20
+ return unless @process.prepare
21
+ return false if precheck_to_cancel
22
+ return false if precheck_to_skip
23
+ true
24
+ end
25
+
26
+ # Start processing the task.
27
+ #
28
+ # The task is executed asynchronously.
29
+ #
30
+ # @return [Boolean] false if the task is not executed
31
+ def start
32
+ return false unless @prerequisites.all?(&:complete?)
33
+
34
+ return false if check_to_cancel
35
+ return false if check_to_skip
36
+
37
+ return false unless @process.start
38
+ execute_or_wait
39
+ true
40
+ end
41
+
42
+ def capture_exception
43
+ yield
44
+ rescue StandardError, ScriptError => error
45
+ @process.quit(error)
46
+ rescue Exception => exception
47
+ # TODO: exit immediately when fatal error occurs.
48
+ @process.quit(exception)
49
+ end
50
+
51
+ private
52
+
53
+ # Cancel the task if it should be.
54
+ # @return [Boolean] true if cancelled
55
+ def precheck_to_cancel
56
+ return false if @job.env.options.repair
57
+ return false unless @state.failed?
58
+ return false if @job.task.top_level?
59
+ @process.cancel(Error.new('task has been failed'))
60
+ end
61
+
62
+ # Skip the task if it should be.
63
+ # @return [Boolean] true if skipped
64
+ def precheck_to_skip
65
+ return @process.skip unless @job.task.concerned?
66
+
67
+ return false if @job.env.options.repair && !@prerequisites.empty?
68
+ return false unless @state.successed?
69
+ @process.skip
70
+ end
71
+
72
+ # Cancel the task if it should be.
73
+ # @return [Boolean] true if cancelled
74
+ def check_to_cancel
75
+ return false if @prerequisites.empty?
76
+ return false if @prerequisites.all? { |p| !p.failed? && !p.cancelled? }
77
+ @process.cancel
78
+ end
79
+
80
+ # Skip the task if it should be.
81
+ # @return [Boolean] true if skipped
82
+ def check_to_skip
83
+ return @process.skip unless @job.task.needed?
84
+
85
+ return false if @prerequisites.empty?
86
+ return false unless @job.env.options.repair
87
+ return false if @prerequisites.any?(&:successed?)
88
+ @process.skip
89
+ end
90
+
91
+ # Post the task if no external process executing the same task, wait the
92
+ # task finished otherwise.
93
+ def execute_or_wait
94
+ if @job.state.try_start
95
+ @job.worker.post { capture_exception { execute } }
96
+ else
97
+ Waiter.new(@job).wait
98
+ end
99
+ end
100
+
101
+ def execute
102
+ _, error = safe_execute
103
+
104
+ call_after_hooks(error)
105
+ @job.state.finish(error)
106
+ @process.finish(error)
107
+ end
108
+
109
+ def safe_execute
110
+ @job.task.call_hooks(:before_execute)
111
+ @job.task.execute(@args)
112
+ true
113
+ rescue => error
114
+ [false, error]
115
+ end
116
+
117
+ def call_after_hooks(error)
118
+ @job.task.call_hooks(:after_invoke, error)
119
+ true
120
+ rescue
121
+ # TODO: Error logging
122
+ false
123
+ end
124
+ end
125
+ end
126
+ end