abid 0.3.0.pre.alpha.3 → 0.3.0.pre.alpha.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/abid.gemspec +1 -2
  4. data/exe/abidsc +1 -1
  5. data/lib/abid.rb +3 -30
  6. data/lib/abid/application.rb +83 -13
  7. data/lib/abid/cli/assume.rb +7 -8
  8. data/lib/abid/cli/list.rb +2 -2
  9. data/lib/abid/cli/migrate.rb +1 -1
  10. data/lib/abid/cli/revoke.rb +9 -10
  11. data/lib/abid/config.rb +2 -0
  12. data/lib/abid/dsl/abid_job.rb +58 -0
  13. data/lib/abid/dsl/actions.rb +36 -0
  14. data/lib/abid/dsl/job.rb +58 -0
  15. data/lib/abid/dsl/job_manager.rb +53 -0
  16. data/lib/abid/dsl/mixin.rb +52 -0
  17. data/lib/abid/dsl/params_spec.rb +64 -0
  18. data/lib/abid/dsl/play.rb +35 -0
  19. data/lib/abid/dsl/play_core.rb +354 -0
  20. data/lib/abid/dsl/rake_job.rb +41 -0
  21. data/lib/abid/dsl/syntax.rb +34 -0
  22. data/lib/abid/dsl/task.rb +53 -0
  23. data/lib/abid/engine.rb +36 -6
  24. data/lib/abid/engine/executor.rb +30 -48
  25. data/lib/abid/engine/process.rb +102 -68
  26. data/lib/abid/engine/process_manager.rb +72 -28
  27. data/lib/abid/engine/scheduler.rb +24 -30
  28. data/lib/abid/engine/waiter.rb +20 -30
  29. data/lib/abid/engine/worker_manager.rb +26 -60
  30. data/lib/abid/environment.rb +12 -27
  31. data/lib/abid/error.rb +2 -0
  32. data/lib/abid/params_format.rb +29 -12
  33. data/lib/abid/rake_extensions.rb +11 -3
  34. data/lib/abid/state_manager.rb +40 -0
  35. data/lib/abid/state_manager/state.rb +52 -114
  36. data/lib/abid/state_manager/state_service.rb +88 -0
  37. data/lib/abid/status.rb +63 -0
  38. data/lib/abid/version.rb +1 -1
  39. metadata +19 -32
  40. data/lib/Abidfile.rb +0 -1
  41. data/lib/abid/dsl_definition.rb +0 -29
  42. data/lib/abid/job.rb +0 -67
  43. data/lib/abid/job_manager.rb +0 -22
  44. data/lib/abid/mixin_task.rb +0 -29
  45. data/lib/abid/params_parser.rb +0 -50
  46. data/lib/abid/play.rb +0 -66
  47. data/lib/abid/play_core.rb +0 -53
  48. data/lib/abid/rake_extensions/task.rb +0 -41
  49. data/lib/abid/state_manager/database.rb +0 -40
  50. data/lib/abid/state_manager/state_proxy.rb +0 -65
  51. data/lib/abid/task.rb +0 -123
  52. data/lib/abid/task_manager.rb +0 -61
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 53398cf57eddf9c3cc6de80acce35fa592c4d00a
4
- data.tar.gz: a89e82cc2adb5d59cc1bbf2697ae21f850f5fa29
3
+ metadata.gz: 1596801ffb3b1eb65f3afcabef8320657e25b53a
4
+ data.tar.gz: d1eca9882b7e33183b8c7fe1664249e10e7823f6
5
5
  SHA512:
6
- metadata.gz: 06e97ecf6eeb098f101386351b081224b0a665278088e4b266f059603b69ffe8778485a8d9519c18367ee5c964f9c8c8a9c6a56b167da352d2131f38d2ecc883
7
- data.tar.gz: 0d20bb8a5032cbec0b2f89416cf8dd872dcce97585685b59447cfb716915423cdd9f25355518e18dc5fc0e2c0e68f0358dad945aa551d79769846366ac5ef9d9
6
+ metadata.gz: 08fd18e4a5280106ee3e23a2e422692428b211f3f35de959512f9bf1a4cde6b75408dbfdbd7ba32d37db6c56177a891bd105fbf25ff05a76865a8975fecef5ad
7
+ data.tar.gz: 0df0a4df266bf2e69f755a1aff2458b2bbbb7d94f3fffc106ae476c2bff241e1f86d336158cb03de19ed0698700f5f43e7b1ca8e4272b945a630ced6cfc30ae2
data/README.md CHANGED
@@ -131,7 +131,7 @@ Abid plays can also be volatile.
131
131
 
132
132
  ```
133
133
  play :voaltile_play do
134
- set :volatile, true
134
+ volatile
135
135
  def run
136
136
  ...
137
137
  end
@@ -164,14 +164,14 @@ define_worker :copy, 2
164
164
  define_worker :hive, 4
165
165
 
166
166
  play :copy_source_1 do
167
- set :worker, :copy
167
+ worker :copy
168
168
  def run
169
169
  ...
170
170
  end
171
171
  end
172
172
 
173
173
  play :hive_query_1 do
174
- set :worker, :hive
174
+ worker :hive
175
175
  def run
176
176
  ...
177
177
  end
data/abid.gemspec CHANGED
@@ -24,10 +24,9 @@ Gem::Specification.new do |spec|
24
24
  spec.add_development_dependency 'pry-byebug'
25
25
  spec.add_development_dependency 'yard'
26
26
 
27
- spec.add_dependency 'rake', '~> 10.0'
27
+ spec.add_dependency 'rake'
28
28
  spec.add_dependency 'concurrent-ruby-ext'
29
29
  spec.add_dependency 'sequel'
30
30
  spec.add_dependency 'sqlite3'
31
- spec.add_dependency 'rbtree'
32
31
  spec.add_dependency 'thor'
33
32
  end
data/exe/abidsc CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
- require 'abid/cli'
3
2
  require 'abid'
3
+ require 'abid/cli'
4
4
 
5
5
  Abid::CLI.start(ARGV)
data/lib/abid.rb CHANGED
@@ -1,45 +1,18 @@
1
1
  require 'rake'
2
- require 'date'
3
- require 'digest/md5'
4
2
  require 'English'
5
- require 'forwardable'
6
- require 'monitor'
7
- require 'time'
8
- require 'yaml'
9
- require 'concurrent'
10
- require 'rbtree'
11
- require 'sqlite3'
12
- require 'sequel'
13
3
 
4
+ require 'abid/application'
14
5
  require 'abid/config'
15
6
  require 'abid/environment'
16
7
  require 'abid/error'
17
8
  require 'abid/engine'
18
- require 'abid/engine/process'
19
- require 'abid/engine/process_manager'
20
- require 'abid/engine/executor'
21
- require 'abid/engine/scheduler'
22
- require 'abid/engine/worker_manager'
23
- require 'abid/engine/waiter'
24
9
  require 'abid/params_format'
25
- require 'abid/state_manager/database'
26
- require 'abid/job'
27
- require 'abid/job_manager'
28
-
29
10
  require 'abid/rake_extensions'
11
+ require 'abid/state_manager'
12
+ require 'abid/status'
30
13
  require 'abid/version'
31
- require 'abid/params_parser'
32
- require 'abid/play_core'
33
- require 'abid/play'
34
- require 'abid/task'
35
- require 'abid/mixin_task'
36
- require 'abid/task_manager'
37
- require 'abid/dsl_definition'
38
- require 'abid/application'
39
14
 
40
15
  module Abid
41
- FIXNUM_MAX = (2**(0.size * 8 - 2) - 1) # :nodoc:
42
-
43
16
  class << self
44
17
  def global
45
18
  @global ||= Environment.new
@@ -1,30 +1,60 @@
1
+ require 'logger'
2
+ require 'abid/dsl/abid_job'
3
+ require 'abid/dsl/actions'
4
+ require 'abid/dsl/mixin'
5
+ require 'abid/dsl/params_spec'
6
+ require 'abid/dsl/play_core'
7
+ require 'abid/dsl/play'
8
+ require 'abid/dsl/rake_job'
9
+ require 'abid/dsl/syntax'
10
+ require 'abid/dsl/job'
11
+ require 'abid/dsl/job_manager'
12
+ require 'abid/dsl/task'
13
+
1
14
  module Abid
2
15
  class Application < Rake::Application
3
- include Abid::TaskManager
4
-
5
16
  def initialize(env)
6
17
  super()
7
18
  @rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb)
8
19
  @env = env
20
+ @global_params = {}
21
+ @global_mixin = DSL::Mixin.create_global_mixin
22
+ @job_manager = DSL::JobManager.new(self)
23
+ @after_all_actions = []
9
24
  end
25
+ attr_reader :global_params, :global_mixin, :job_manager, :after_all_actions
10
26
 
11
27
  def init
12
28
  super
13
29
  @env.config.load(options.config_file)
14
30
  end
15
31
 
16
- def run_with_threads
17
- yield
18
- rescue Exception => err
19
- Engine.kill(err)
20
- raise err
21
- else
22
- Engine.shutdown
32
+ def top_level
33
+ if options.show_tasks || options.show_prereqs
34
+ super
35
+ else
36
+ run_with_engine { invoke_top_level_tasks }
37
+ end
23
38
  end
24
39
 
25
- def invoke_task(task_string) # :nodoc:
26
- name, args = parse_task_string(task_string)
27
- Job.find_by_task(self[name]).invoke(*args)
40
+ def invoke_top_level_tasks
41
+ top_level_tasks.each do |task_string|
42
+ name, args = parse_task_string(task_string)
43
+ @env.engine.invoke(name, *args)
44
+ break unless @env.engine.errors.empty?
45
+ end
46
+ end
47
+
48
+ def run_with_engine
49
+ yield
50
+ @env.engine.shutdown
51
+ rescue Exception => exception
52
+ @env.engine.kill(exception)
53
+ raise
54
+ else
55
+ raise @env.engine.errors.first unless @env.engine.errors.empty?
56
+ ensure
57
+ call_after_all_actions
28
58
  end
29
59
 
30
60
  def standard_rake_options
@@ -56,7 +86,16 @@ module Abid
56
86
  proc { options.preview = true }],
57
87
  ['--wait-external-task',
58
88
  'Wait a task finished if it is running in externl process',
59
- proc { options.wait_external_task = true }]
89
+ proc { options.wait_external_task = true }],
90
+ ['--log-level LEVEL',
91
+ 'Specifies the log level. LEVEL can be error, warn, info or debug.' \
92
+ ' (default: info)',
93
+ proc do |v|
94
+ options.log_level = Logger::Severity.const_get(v.upcase)
95
+ end],
96
+ ['--[no-]logging',
97
+ 'Enable logging. (default: on)',
98
+ proc { |v| options.logging = v }]
60
99
  ]
61
100
  )
62
101
  end
@@ -64,6 +103,8 @@ module Abid
64
103
  def handle_options
65
104
  options.rakelib = %w(rakelib tasks)
66
105
  options.trace_output = $stderr
106
+ options.log_level = Logger::Severity::INFO
107
+ options.logging = true
67
108
 
68
109
  OptionParser.new do |opts|
69
110
  opts.banner = 'See full documentation at https://github.com/ojima-h/abid.'
@@ -88,5 +129,34 @@ module Abid
88
129
  opts.environment('RAKEOPT')
89
130
  end.parse!
90
131
  end
132
+
133
+ def collect_command_line_tasks(args)
134
+ params, = ParamsFormat.collect_params(args)
135
+ @global_params.update(params)
136
+ super
137
+ end
138
+
139
+ #
140
+ # Abid Extentions
141
+ #
142
+ def logger
143
+ return @logger if @logger
144
+ logdev = options.logging ? $stderr : nil
145
+ @logger = Logger.new(logdev).tap do |l|
146
+ l.progname = 'abid'
147
+ l.level = options.log_level || Logger::Severity::INFO
148
+ end
149
+ end
150
+ attr_writer :logger
151
+
152
+ def call_after_all_actions
153
+ @after_all_actions.each do |block|
154
+ block.call(
155
+ top_level_tasks,
156
+ @env.engine.summary,
157
+ @env.engine.errors
158
+ )
159
+ end
160
+ end
91
161
  end
92
162
  end
@@ -10,21 +10,20 @@ module Abid
10
10
  end
11
11
 
12
12
  def run
13
- tasks, params = ParamsFormat.parse_args(@args)
13
+ params, tasks = ParamsFormat.collect_params(@args)
14
14
 
15
15
  tasks.each { |task| assume(task, params) }
16
16
  end
17
17
 
18
18
  def assume(task, params)
19
- params_str = ParamsFormat.format(params)
19
+ state = @env.state_manager.state(task, params)
20
+ state.assume(force: @force)
20
21
 
21
- job = @env.job_manager[task, params]
22
- state = job.state.assume(force: @force)
23
-
24
- puts "#{task} #{params_str} (id: #{state.id})" \
25
- ' is assumed to be SUCCESSED.'
22
+ s = state.find
23
+ n = ParamsFormat.format_with_name(task, params)
24
+ puts "#{n} (id: #{s.id}) is assumed to be SUCCESSED."
26
25
  rescue AlreadyRunningError
27
- $stderr.puts "#{task} #{params_str} already running.\n" \
26
+ $stderr.puts "#{n} already running.\n" \
28
27
  'Use -f option if you want to force assume.'
29
28
  raise
30
29
  end
data/lib/abid/cli/list.rb CHANGED
@@ -17,7 +17,7 @@ module Abid
17
17
  end
18
18
 
19
19
  def build_table
20
- states = @env.db.states
20
+ states = @env.state_manager.states
21
21
  .filter_by_prefix(@prefix)
22
22
  .filter_by_start_time(after: @after, before: @before)
23
23
  table = states.map { |state| format_state(state) }
@@ -31,7 +31,7 @@ module Abid
31
31
  t,
32
32
  state.state_label,
33
33
  format_exec_time(state.exec_time),
34
- state.name + ' ' + ParamsFormat.format(YAML.load(state.params))
34
+ ParamsFormat.format_with_name(state.name, state.params_hash)
35
35
  ]
36
36
  end
37
37
 
@@ -8,7 +8,7 @@ module Abid
8
8
 
9
9
  def run
10
10
  db = Sequel.connect(@env.config.database)
11
- dir = StateManager::Database::MIGRATIONS_PATH
11
+ dir = StateManager::MIGRATIONS_PATH
12
12
 
13
13
  if Sequel::Migrator.is_current?(db, dir)
14
14
  puts 'Schema is latest.'
@@ -14,30 +14,29 @@ module Abid
14
14
 
15
15
  def run
16
16
  @job_ids.each do |job_id|
17
- state = @env.db.states[job_id]
17
+ state = @env.state_manager.states[job_id]
18
18
  if state.nil?
19
19
  $stderr.puts "#{job_id} is not found"
20
20
  next
21
21
  end
22
22
 
23
- next if !@quiet && !ask(state)
23
+ text = ParamsFormat.format_with_name(state.name, state.params_hash)
24
+ next if !@quiet && !ask(text)
24
25
 
25
- revoke(state)
26
+ revoke(state, text)
26
27
  end
27
28
  end
28
29
 
29
- def revoke(state)
30
- @env.db.states.revoke(state.id, force: @force)
30
+ def revoke(state, text)
31
+ state.revoke(force: @force)
31
32
  puts "revoked #{state.id}"
32
33
  rescue AlreadyRunningError
33
- params = ParamsFormat.format(YAML.load(state.params))
34
- $stderr.puts "#{state.name} #{params} already running.\n" \
34
+ $stderr.puts "#{text} already running.\n" \
35
35
  'Use -f option if you want to force assume.'
36
36
  end
37
37
 
38
- def ask(state)
39
- params = ParamsFormat.format(YAML.load(state.params))
40
- print "revoke task \`#{state.name} #{params}'? "
38
+ def ask(text)
39
+ print "revoke task \`#{text}'? "
41
40
  $stdout.flush
42
41
  ret = $stdin.gets
43
42
  ret.match(/y(es)?/i)
data/lib/abid/config.rb CHANGED
@@ -1,3 +1,5 @@
1
+ require 'yaml'
2
+
1
3
  module Abid
2
4
  class Config < Hash
3
5
  DEFAULT_DATABASE_CONFIG = {
@@ -0,0 +1,58 @@
1
+ require 'abid/dsl/job'
2
+
3
+ module Abid
4
+ module DSL
5
+ # Abid::DSL::Task wrapper
6
+ class AbidJob < Job
7
+ def_delegators :play, :worker
8
+ def_delegator :play, :concerned, :concerned?
9
+ def_delegator :play, :needed, :needed?
10
+
11
+ def initialize(task, params)
12
+ super
13
+ @play = task.internal.new(params)
14
+ @play.call_action(:setup)
15
+ end
16
+ attr_reader :play
17
+ private :play
18
+
19
+ def execute(args)
20
+ if dryrun?
21
+ trace_execute
22
+ return
23
+ end
24
+ play.call_action(:safe_action)
25
+ play.call_action(:action, args) unless preview?
26
+ run(args)
27
+ ensure
28
+ play.call_action(:after, $ERROR_INFO) unless dryrun? || preview?
29
+ end
30
+
31
+ def run(args)
32
+ if play.method(:run).arity.zero?
33
+ play.run
34
+ else
35
+ play.run(args)
36
+ end
37
+ end
38
+
39
+ def prerequisites
40
+ ps = task.prerequisite_tasks.map { |preq| [preq, params] } \
41
+ + play.prerequisite_tasks
42
+
43
+ ps.map do |preq, params|
44
+ task.application.job_manager[preq.name, params]
45
+ end
46
+ end
47
+
48
+ def volatile?
49
+ play.volatile || task.application.options.disable_state
50
+ end
51
+
52
+ def trace_execute
53
+ task.application.trace "** Execute (dry run) #{task.name}"
54
+ end
55
+ private :trace_execute
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,36 @@
1
+ module Abid
2
+ module DSL
3
+ # Actions manages action index of a play.
4
+ #
5
+ # It merges all actions of the play's ancestors.
6
+ class Actions
7
+ def initialize(play_class)
8
+ @play_class = play_class
9
+ @index = Hash.new { |h, k| h[k] = [] }
10
+ end
11
+
12
+ attr_reader :index
13
+ protected :index
14
+
15
+ # @param key [Symbol] action_name
16
+ # @return [Enumerator<Proc>]
17
+ def [](key)
18
+ Enumerator.new do |y|
19
+ # search key over including mixins
20
+ @play_class.superplays.reverse.each do |sp|
21
+ sp.actions.index[key].each { |block| y << block }
22
+ end
23
+ end
24
+ end
25
+
26
+ # @param key [Symbol] action name
27
+ # @param scope [Rake::Scope] task scope where the action is declared
28
+ # @param block [Proc] action body
29
+ # @return self
30
+ def add(key, scope, block)
31
+ @index[key] << [scope, block]
32
+ self
33
+ end
34
+ end
35
+ end
36
+ end