abid 0.1.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,12 +4,10 @@ module Abid
4
4
 
5
5
  attr_reader :worker
6
6
  attr_reader :config
7
- attr_reader :futures
8
7
 
9
8
  def initialize
10
9
  super
11
10
  @rakefiles = %w(abidfile Abidfile abidfile.rb Abidfile.rb) << abidfile
12
- @futures = {}
13
11
  @worker = Worker.new(self)
14
12
  end
15
13
 
@@ -21,26 +19,39 @@ module Abid
21
19
  def init(app_name = 'abid')
22
20
  standard_exception_handling do
23
21
  @config = IniFile.new(content: default_config)
24
- @config.merge!(IniFile.load('config/abid.cfg'))
22
+ @config.merge!(IniFile.load('config/abid.conf'))
25
23
  end
26
24
 
27
25
  super(app_name)
28
26
  end
29
27
 
30
- # allows the `abid db:migrate` task to load without a abidfile
28
+ # allows the built-in tasks to load without a abidfile
31
29
  def abidfile
32
30
  File.expand_path(File.join(File.dirname(__FILE__), '..', 'Abidfile.rb'))
33
31
  end
34
32
 
33
+ # load built-in tasks
34
+ def load_rakefile
35
+ standard_exception_handling do
36
+ glob(File.expand_path('../tasks/*.rake', __FILE__)) do |name|
37
+ Rake.load_rakefile name
38
+ end
39
+ end
40
+ super
41
+ end
42
+
35
43
  def run_with_threads
36
44
  yield
37
- ensure
45
+ rescue Exception => err
46
+ worker.kill
47
+ raise err
48
+ else
38
49
  worker.shutdown
39
50
  end
40
51
 
41
52
  def invoke_task(task_string) # :nodoc:
42
53
  name, args = parse_task_string(task_string)
43
- self[name].async_invoke(*args).value!
54
+ self[name].async_invoke(*args).wait!
44
55
  end
45
56
 
46
57
  def default_config
@@ -61,12 +72,6 @@ module Abid
61
72
  when '--execute-print'
62
73
  # disable short option
63
74
  opt.delete_at(1)
64
- when '--dry-run'
65
- h = opt.last
66
- opt[-1] = lambda do |value|
67
- h.call(value)
68
- options.disable_state = true
69
- end
70
75
  when '--version'
71
76
  opt[-1] = lambda do |_value|
72
77
  puts "Abid Version: #{Abid::VERSION} (Rake Version: #{RAKEVERSION})"
@@ -79,14 +84,13 @@ module Abid
79
84
  def abid_options # :nodoc:
80
85
  sort_options(
81
86
  [
82
- ['--check-parents', '-c',
83
- 'Run the task if the parents was updated.',
84
- proc { options.check_prerequisites = true }
87
+ ['--repair',
88
+ 'Run the task in repair mode.',
89
+ proc { options.repair = true }
85
90
  ],
86
91
  ['--preview', '-p',
87
92
  'Run tasks in preview mode.',
88
93
  proc do
89
- options.disable_state = true
90
94
  options.preview = true
91
95
  end
92
96
  ],
@@ -101,7 +105,7 @@ module Abid
101
105
  end
102
106
 
103
107
  def handle_options
104
- options.rakelib = ['rakelib']
108
+ options.rakelib = %w(rakelib tasks)
105
109
  options.trace_output = $stderr
106
110
 
107
111
  OptionParser.new do |opts|
@@ -0,0 +1,6 @@
1
+ module Abid
2
+ module ConcurrentExtention
3
+ require 'abid/concurrent_extention/ivar'
4
+ Concurrent::IVar.include IVar
5
+ end
6
+ end
@@ -0,0 +1,12 @@
1
+ module Abid
2
+ module ConcurrentExtention
3
+ module IVar
4
+ def try_fail(reason = StandardError.new)
5
+ self.fail(reason)
6
+ true
7
+ rescue Concurrent::MultipleAssignmentError
8
+ false
9
+ end
10
+ end
11
+ end
12
+ end
@@ -8,8 +8,8 @@ module Abid
8
8
  Rake.application.worker.define(name, thread_count)
9
9
  end
10
10
 
11
- def default_play_class(&block)
12
- Rake.application.default_play_class(&block)
11
+ def play_base(&block)
12
+ Rake.application.play_base(&block)
13
13
  end
14
14
 
15
15
  def helpers(*extensions, &block)
@@ -19,7 +19,7 @@ module Abid
19
19
 
20
20
  def type_cast(value, type)
21
21
  case type
22
- when :boolean then value == 'true'
22
+ when :boolean then value == true || value == 'true'
23
23
  when :int then value.to_i
24
24
  when :float then value.to_f
25
25
  when :string then value.to_s
data/lib/abid/play.rb CHANGED
@@ -17,11 +17,17 @@ module Abid
17
17
  def param(name, **param_spec)
18
18
  params_spec[name] = { significant: true }.merge(param_spec)
19
19
 
20
- define_method(name) { params[name] }
20
+ define_method(name) { task.params[name] }
21
+ end
22
+
23
+ def undef_param(name)
24
+ params_spec.delete(name)
25
+ undef_method(name) if method_defined?(name)
21
26
  end
22
27
 
23
28
  def hooks
24
29
  @hooks ||= {
30
+ setup: [],
25
31
  before: [],
26
32
  after: [],
27
33
  around: []
@@ -47,43 +53,30 @@ module Abid
47
53
  include(*extensions) if extensions.any?
48
54
  end
49
55
 
56
+ def setup(&block)
57
+ hooks[:setup] << block
58
+ end
59
+
50
60
  def before(&block)
51
- (hooks[:before] ||= []) << block
61
+ hooks[:before] << block
52
62
  end
53
63
 
54
64
  def after(&block)
55
- (hooks[:after] ||= []) << block
65
+ hooks[:after] << block
56
66
  end
57
67
 
58
68
  def around(&block)
59
- (hooks[:around] ||= []) << block
69
+ hooks[:around] << block
60
70
  end
61
71
  end
62
72
 
63
73
  set :worker, :default
64
74
  set :volatile, false
65
75
 
66
- extend Forwardable
67
- def_delegators :task, :application, :name, :scope
68
- def_delegators 'self.class', :params_spec
69
-
70
- attr_reader :prerequisites
71
- attr_reader :params
72
-
73
- def initialize(params)
74
- @prerequisites = []
76
+ attr_reader :task
75
77
 
76
- @params = ParamsParser.parse(params, params_spec)
77
- @params = @params.sort.to_h # avoid ambiguity of keys order
78
- @params.freeze
79
- end
80
-
81
- def task
82
- self.class.task
83
- end
84
-
85
- def setup
86
- # noop
78
+ def initialize(task)
79
+ @task = task
87
80
  end
88
81
 
89
82
  def run
@@ -91,23 +84,7 @@ module Abid
91
84
  end
92
85
 
93
86
  def needs(task_name, **params)
94
- @prerequisites |= [[task_name, params]]
95
- end
96
-
97
- def significant_params
98
- [
99
- name,
100
- params.select { |p, _| params_spec[p][:significant] }
101
- ]
102
- end
103
-
104
- def hash
105
- significant_params.hash
106
- end
107
-
108
- def eql?(other)
109
- other.is_a?(Abid::Play) && \
110
- significant_params.eql?(other.significant_params)
87
+ task.enhance([[task_name, params]])
111
88
  end
112
89
 
113
90
  def volatile?
@@ -115,25 +92,7 @@ module Abid
115
92
  end
116
93
 
117
94
  def preview?
118
- application.options.preview
119
- end
120
-
121
- def invoke
122
- self.class.hooks[:before].each { |blk| instance_eval(&blk) }
123
-
124
- call_around_hooks(self.class.hooks[:around]) { run }
125
-
126
- self.class.hooks[:after].each { |blk| instance_eval(&blk) }
127
- end
128
-
129
- def call_around_hooks(hooks, &body)
130
- if hooks.empty?
131
- body.call
132
- else
133
- h, *rest = hooks
134
- instance_exec(-> { call_around_hooks(rest, &body) }, &h)
135
- end
95
+ task.application.options.preview
136
96
  end
137
- private :call_around_hooks
138
97
  end
139
98
  end
@@ -10,7 +10,11 @@ module Abid
10
10
  end
11
11
 
12
12
  def state
13
- @state ||= State.find(self)
13
+ State.find(self)
14
+ end
15
+
16
+ def name_with_params
17
+ name
14
18
  end
15
19
 
16
20
  def async_invoke(*args)
@@ -20,114 +24,121 @@ module Abid
20
24
 
21
25
  def async_invoke_with_call_chain(task_args, invocation_chain)
22
26
  state.reload
23
- new_chain = Rake::InvocationChain.append(self, invocation_chain)
24
- @lock.synchronize do
25
- if application.futures.include?(object_id)
26
- return application.futures[object_id]
27
- end
28
-
29
- application.trace "** Invoke #{name}" if application.options.trace
30
27
 
31
- preq_futures = async_invoke_prerequisites(task_args, new_chain)
32
-
33
- future = async_invoke_after_prerequisites(task_args, preq_futures)
28
+ new_chain = Rake::InvocationChain.append(self, invocation_chain)
34
29
 
35
- application.futures[object_id] = future
30
+ state.only_once do
31
+ if !application.options.repair && state.successed?
32
+ # skip if successed
33
+ state.ivar.try_set(false)
34
+ elsif !application.options.repair && state.failed? && !invocation_chain.empty?
35
+ # fail if not top level
36
+ fail "#{name} -- task has been failed" rescue state.ivar.try_fail($ERROR_INFO)
37
+ else
38
+ async_invoke_with_prerequisites(task_args, new_chain)
39
+ end
36
40
  end
41
+ state.ivar
42
+ ensure
43
+ state.ivar.try_fail($ERROR_INFO) if $ERROR_INFO
37
44
  end
38
45
 
39
- def async_invoke_prerequisites(task_args, invocation_chain)
40
- # skip if successed
41
- if state.successed?
42
- if !application.options.check_prerequisites
43
- preqs = []
46
+ def async_invoke_with_prerequisites(task_args, invocation_chain)
47
+ application.trace "** Invoke #{name_with_params}" if application.options.trace
48
+
49
+ volatiles, non_volatiles = prerequisite_tasks.partition(&:volatile?)
50
+
51
+ async_invoke_tasks(non_volatiles, task_args, invocation_chain) do |updated|
52
+ if state.successed? && !updated
53
+ application.trace "** Skip #{name_with_params}" if application.options.trace
54
+ state.ivar.try_set(false)
44
55
  else
45
- preqs = prerequisite_tasks.reject(&:volatile?)
56
+ async_invoke_tasks(volatiles, task_args, invocation_chain) do
57
+ async_execute_with_session(task_args)
58
+ end
46
59
  end
47
- else
48
- preqs = prerequisite_tasks
49
60
  end
61
+ end
50
62
 
51
- preqs.map do |p|
52
- preq_args = task_args.new_scope(p.arg_names)
53
- p.async_invoke_with_call_chain(preq_args, invocation_chain)
63
+ def async_invoke_tasks(tasks, task_args, invocation_chain, &block)
64
+ ivars = tasks.map do |t|
65
+ args = task_args.new_scope(t.arg_names)
66
+ t.async_invoke_with_call_chain(args, invocation_chain)
54
67
  end
55
- end
56
68
 
57
- def async_invoke_after_prerequisites(task_args, preq_futures)
58
- if preq_futures.empty?
59
- async_execute_with_session(task_args, false)
69
+ if ivars.empty?
70
+ block.call(false)
60
71
  else
61
- result = Concurrent::IVar.new
62
- counter = Concurrent::DependencyCounter.new(preq_futures.size) do
72
+ counter = Concurrent::DependencyCounter.new(ivars.size) do
63
73
  begin
64
- failed_preq = preq_futures.find(&:rejected?)
65
- next result.fail(failed_preq.reason) if failed_preq
66
-
67
- preq_updated = preq_futures.map(&:value!).any?
68
-
69
- future = async_execute_with_session(task_args, preq_updated)
70
-
71
- future.add_observer do |_time, value, reason|
72
- reason.nil? ? result.set(value) : result.fail(reason)
74
+ if ivars.any?(&:rejected?)
75
+ state.ivar.try_fail(ivars.find(&:rejected?).reason)
76
+ else
77
+ updated = ivars.map(&:value).any?
78
+ block.call(updated)
73
79
  end
74
80
  rescue Exception => err
75
- result.fail(err)
81
+ state.ivar.try_fail(err)
76
82
  end
77
83
  end
78
- preq_futures.each { |p| p.add_observer counter }
79
- result
84
+ ivars.each { |i| i.add_observer counter }
80
85
  end
81
86
  end
82
87
 
83
- def async_execute_with_session(task_args, prerequisites_updated = false)
84
- if (state.successed? && !prerequisites_updated) || !needed?
85
- application.trace "** Skip #{name}" if application.options.trace
86
- return Concurrent::IVar.new(false)
87
- end
88
-
89
- session_started = state.start_session
90
-
91
- return async_wait_complete unless session_started
92
-
93
- pool = application.worker[worker]
94
- future = Concurrent::Future.execute(executor: pool) do
88
+ def async_execute_with_session(task_args)
89
+ async_execute_in_worker do
95
90
  begin
96
- execute(task_args)
97
- true
98
- ensure
99
- state.close_session($ERROR_INFO)
91
+ state.session do
92
+ begin
93
+ execute(task_args) if needed?
94
+ finished = true
95
+ ensure
96
+ fail "#{name} -- thread killed" if $ERROR_INFO.nil? && !finished
97
+ end
98
+ end
99
+
100
+ state.ivar.try_set(true)
101
+ rescue AbidErrorTaskAlreadyRunning
102
+ async_wait_complete
100
103
  end
101
104
  end
102
- ensure
103
- # close session if error occurred outside the future
104
- if session_started && future.nil? && $ERROR_INFO
105
- state.close_session($ERROR_INFO)
106
- end
107
105
  end
108
106
 
109
107
  def async_wait_complete
110
108
  unless application.options.wait_external_task
111
- err = RuntimeError.new("task #{name} already running")
112
- return Concurrent::IVar.new.fail(err)
109
+ err = RuntimeError.new("task #{name_with_params} already running")
110
+ return state.ivar.try_fail(err)
113
111
  end
114
112
 
115
- application.trace "** Wait #{name}" if application.options.trace
113
+ application.trace "** Wait #{name_with_params}" if application.options.trace
116
114
 
117
- pool = application.worker[:waiter]
118
- Concurrent::Future.execute(executor: pool) do
115
+ async_execute_in_worker(:waiter) do
119
116
  interval = application.options.wait_external_task_interval || 10
120
117
  timeout = application.options.wait_external_task_timeout || 3600
121
118
  timeout_tm = Time.now.to_f + timeout
122
119
 
123
120
  loop do
124
121
  state.reload
125
- break unless state.running?
122
+ if !state.running?
123
+ state.ivar.try_set(true)
124
+ break
125
+ elsif Time.now.to_f >= timeout_tm
126
+ fail "#{name} -- timeout exceeded" rescue state.ivar.try_fail($ERROR_INFO)
127
+ break
128
+ else
129
+ sleep interval
130
+ end
131
+ end
132
+ end
133
+ end
126
134
 
127
- sleep interval
128
- break if Time.now.to_f >= timeout_tm
135
+ def async_execute_in_worker(worker = nil, &block)
136
+ application.worker[worker || self.worker].post do
137
+ begin
138
+ block.call
139
+ rescue Exception => err
140
+ state.ivar.try_fail(err)
129
141
  end
130
- true
131
142
  end
132
143
  end
133
144
  end