abid 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,21 @@
1
+ module Abid
2
+ module DSL
3
+ def play(*args, &block)
4
+ Abid::Task.define_play(*args, &block)
5
+ end
6
+
7
+ def define_worker(name, thread_count)
8
+ Rake.application.worker.define(name, thread_count)
9
+ end
10
+
11
+ def default_play_class(&block)
12
+ Rake.application.default_play_class(&block)
13
+ end
14
+
15
+ def helpers(*extensions, &block)
16
+ Abid::Play.helpers(*extensions, &block)
17
+ end
18
+ end
19
+ end
20
+
21
+ extend Abid::DSL
@@ -0,0 +1,50 @@
1
+ module Abid
2
+ module ParamsParser
3
+ class <<self
4
+ def parse(params, specs)
5
+ specs.map do |name, spec|
6
+ if params.include?(name)
7
+ value = type_cast(params[name], spec[:type])
8
+ elsif ENV.include?(name.to_s)
9
+ value = type_cast(ENV[name.to_s], spec[:type])
10
+ elsif spec.key?(:default)
11
+ value = spec[:default]
12
+ else
13
+ fail "param #{name} is not specified"
14
+ end
15
+
16
+ [name, value]
17
+ end.to_h
18
+ end
19
+
20
+ def type_cast(value, type)
21
+ case type
22
+ when :boolean then value == 'true'
23
+ when :int then value.to_i
24
+ when :float then value.to_f
25
+ when :string then value.to_s
26
+ when :date then type_cast_date(value)
27
+ when :datetime, :time then type_cast_time(value)
28
+ when nil then value
29
+ else fail "invalid type: #{type}"
30
+ end
31
+ end
32
+
33
+ def type_cast_date(value)
34
+ case value
35
+ when Date then value
36
+ when Time, DateTime then value.to_date
37
+ else Date.parse(value.to_s)
38
+ end
39
+ end
40
+
41
+ def type_cast_time(value)
42
+ case value
43
+ when Date then value.to_time
44
+ when Time, DateTime then value
45
+ else Time.parse(value.to_s)
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,139 @@
1
+ require 'forwardable'
2
+
3
+ module Abid
4
+ class Play
5
+ class << self
6
+ attr_accessor :task
7
+
8
+ def inherited(child)
9
+ params_spec.each { |k, v| child.params_spec[k] = v.dup }
10
+ hooks.each { |k, v| child.hooks[k] = v.dup }
11
+ end
12
+
13
+ def params_spec
14
+ @params_spec ||= {}
15
+ end
16
+
17
+ def param(name, **param_spec)
18
+ params_spec[name] = { significant: true }.merge(param_spec)
19
+
20
+ define_method(name) { params[name] }
21
+ end
22
+
23
+ def hooks
24
+ @hooks ||= {
25
+ before: [],
26
+ after: [],
27
+ around: []
28
+ }
29
+ end
30
+
31
+ def set(name, value = nil, &block)
32
+ var = :"@#{name}"
33
+ define_method(name) do
34
+ unless instance_variable_defined?(var)
35
+ if !value.nil?
36
+ instance_variable_set(var, value)
37
+ elsif block_given?
38
+ instance_variable_set(var, instance_eval(&block))
39
+ end
40
+ end
41
+ instance_variable_get(var)
42
+ end
43
+ end
44
+
45
+ def helpers(*extensions, &block)
46
+ class_eval(&block) if block_given?
47
+ include(*extensions) if extensions.any?
48
+ end
49
+
50
+ def before(&block)
51
+ (hooks[:before] ||= []) << block
52
+ end
53
+
54
+ def after(&block)
55
+ (hooks[:after] ||= []) << block
56
+ end
57
+
58
+ def around(&block)
59
+ (hooks[:around] ||= []) << block
60
+ end
61
+ end
62
+
63
+ set :worker, :default
64
+ set :volatile, false
65
+
66
+ extend Forwardable
67
+ def_delegators :task, :application, :name, :scope
68
+ def_delegators 'self.class', :params_spec
69
+
70
+ attr_reader :prerequisites
71
+ attr_reader :params
72
+
73
+ def initialize(params)
74
+ @prerequisites = []
75
+
76
+ @params = ParamsParser.parse(params, params_spec)
77
+ @params = @params.sort.to_h # avoid ambiguity of keys order
78
+ @params.freeze
79
+ end
80
+
81
+ def task
82
+ self.class.task
83
+ end
84
+
85
+ def setup
86
+ # noop
87
+ end
88
+
89
+ def run
90
+ # noop
91
+ end
92
+
93
+ def needs(task_name, **params)
94
+ @prerequisites |= [[task_name, params]]
95
+ end
96
+
97
+ def significant_params
98
+ [
99
+ name,
100
+ params.select { |p, _| params_spec[p][:significant] }
101
+ ]
102
+ end
103
+
104
+ def hash
105
+ significant_params.hash
106
+ end
107
+
108
+ def eql?(other)
109
+ other.is_a?(Abid::Play) && \
110
+ significant_params.eql?(other.significant_params)
111
+ end
112
+
113
+ def volatile?
114
+ volatile
115
+ end
116
+
117
+ def preview?
118
+ application.options.preview
119
+ end
120
+
121
+ def invoke
122
+ self.class.hooks[:before].each { |blk| instance_eval(&blk) }
123
+
124
+ call_around_hooks(self.class.hooks[:around]) { run }
125
+
126
+ self.class.hooks[:after].each { |blk| instance_eval(&blk) }
127
+ end
128
+
129
+ def call_around_hooks(hooks, &body)
130
+ if hooks.empty?
131
+ body.call
132
+ else
133
+ h, *rest = hooks
134
+ instance_exec(-> { call_around_hooks(rest, &body) }, &h)
135
+ end
136
+ end
137
+ private :call_around_hooks
138
+ end
139
+ end
@@ -0,0 +1,6 @@
1
+ module Abid
2
+ module RakeExtensions
3
+ require 'abid/rake_extensions/task'
4
+ Rake::Task.include Task
5
+ end
6
+ end
@@ -0,0 +1,135 @@
1
+ module Abid
2
+ module RakeExtensions
3
+ module Task
4
+ def volatile?
5
+ true
6
+ end
7
+
8
+ def worker
9
+ :default
10
+ end
11
+
12
+ def state
13
+ @state ||= State.find(self)
14
+ end
15
+
16
+ def async_invoke(*args)
17
+ task_args = Rake::TaskArguments.new(arg_names, args)
18
+ async_invoke_with_call_chain(task_args, Rake::InvocationChain::EMPTY)
19
+ end
20
+
21
+ def async_invoke_with_call_chain(task_args, invocation_chain)
22
+ state.reload
23
+ new_chain = Rake::InvocationChain.append(self, invocation_chain)
24
+ @lock.synchronize do
25
+ if application.futures.include?(object_id)
26
+ return application.futures[object_id]
27
+ end
28
+
29
+ application.trace "** Invoke #{name}" if application.options.trace
30
+
31
+ preq_futures = async_invoke_prerequisites(task_args, new_chain)
32
+
33
+ future = async_invoke_after_prerequisites(task_args, preq_futures)
34
+
35
+ application.futures[object_id] = future
36
+ end
37
+ end
38
+
39
+ def async_invoke_prerequisites(task_args, invocation_chain)
40
+ # skip if successed
41
+ if state.successed?
42
+ if !application.options.check_prerequisites
43
+ preqs = []
44
+ else
45
+ preqs = prerequisite_tasks.reject(&:volatile?)
46
+ end
47
+ else
48
+ preqs = prerequisite_tasks
49
+ end
50
+
51
+ preqs.map do |p|
52
+ preq_args = task_args.new_scope(p.arg_names)
53
+ p.async_invoke_with_call_chain(preq_args, invocation_chain)
54
+ end
55
+ end
56
+
57
+ def async_invoke_after_prerequisites(task_args, preq_futures)
58
+ if preq_futures.empty?
59
+ async_execute_with_session(task_args, false)
60
+ else
61
+ result = Concurrent::IVar.new
62
+ counter = Concurrent::DependencyCounter.new(preq_futures.size) do
63
+ begin
64
+ failed_preq = preq_futures.find(&:rejected?)
65
+ next result.fail(failed_preq.reason) if failed_preq
66
+
67
+ preq_updated = preq_futures.map(&:value!).any?
68
+
69
+ future = async_execute_with_session(task_args, preq_updated)
70
+
71
+ future.add_observer do |_time, value, reason|
72
+ reason.nil? ? result.set(value) : result.fail(reason)
73
+ end
74
+ rescue Exception => err
75
+ result.fail(err)
76
+ end
77
+ end
78
+ preq_futures.each { |p| p.add_observer counter }
79
+ result
80
+ end
81
+ end
82
+
83
+ def async_execute_with_session(task_args, prerequisites_updated = false)
84
+ if (state.successed? && !prerequisites_updated) || !needed?
85
+ application.trace "** Skip #{name}" if application.options.trace
86
+ return Concurrent::IVar.new(false)
87
+ end
88
+
89
+ session_started = state.start_session
90
+
91
+ return async_wait_complete unless session_started
92
+
93
+ pool = application.worker[worker]
94
+ future = Concurrent::Future.execute(executor: pool) do
95
+ begin
96
+ execute(task_args)
97
+ true
98
+ ensure
99
+ state.close_session($ERROR_INFO)
100
+ end
101
+ end
102
+ ensure
103
+ # close session if error occurred outside the future
104
+ if session_started && future.nil? && $ERROR_INFO
105
+ state.close_session($ERROR_INFO)
106
+ end
107
+ end
108
+
109
+ def async_wait_complete
110
+ unless application.options.wait_external_task
111
+ err = RuntimeError.new("task #{name} already running")
112
+ return Concurrent::IVar.new.fail(err)
113
+ end
114
+
115
+ application.trace "** Wait #{name}" if application.options.trace
116
+
117
+ pool = application.worker[:waiter]
118
+ Concurrent::Future.execute(executor: pool) do
119
+ interval = application.options.wait_external_task_interval || 10
120
+ timeout = application.options.wait_external_task_timeout || 3600
121
+ timeout_tm = Time.now.to_f + timeout
122
+
123
+ loop do
124
+ state.reload
125
+ break unless state.running?
126
+
127
+ sleep interval
128
+ break if Time.now.to_f >= timeout_tm
129
+ end
130
+ true
131
+ end
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,153 @@
1
+ module Abid
2
+ class State
3
+ extend Forwardable
4
+
5
+ RUNNING = 1
6
+ SUCCESSED = 2
7
+ FAILED = 3
8
+
9
+ STATES = constants.map { |c| [const_get(c), c] }.to_h
10
+
11
+ class <<self
12
+ def find(task)
13
+ new(task)
14
+ end
15
+
16
+ def list(pattern: nil, started_before: nil, started_after: nil)
17
+ dataset = Rake.application.database[:states]
18
+
19
+ dataset = dataset.where { start_time < started_before } if started_before
20
+ dataset = dataset.where { start_time > started_after } if started_after
21
+ dataset = dataset.order(:start_time)
22
+
23
+ dataset.map do |record|
24
+ next if pattern && record[:name] !~ pattern
25
+ {
26
+ id: record[:id],
27
+ name: record[:name],
28
+ params: deserialize(record[:params]),
29
+ state: STATES[record[:state]],
30
+ start_time: record[:start_time],
31
+ end_time: record[:end_time]
32
+ }
33
+ end.compact
34
+ end
35
+
36
+ def serialize(params)
37
+ YAML.dump(params)
38
+ end
39
+
40
+ def deserialize(bytes)
41
+ YAML.load(bytes)
42
+ end
43
+ end
44
+
45
+ def_delegators 'self.class', :serialize, :deserialize
46
+
47
+ def initialize(task)
48
+ @task = task
49
+ reload
50
+ end
51
+
52
+ def database
53
+ Rake.application.database
54
+ end
55
+
56
+ def dataset
57
+ database[:states]
58
+ end
59
+
60
+ def disabled?
61
+ @task.volatile? || Rake.application.options.disable_state
62
+ end
63
+
64
+ def reload
65
+ return if disabled?
66
+
67
+ if @record
68
+ id = @record[:id]
69
+ @record = dataset.where(id: id).first
70
+ else
71
+ @record = dataset.where(digest: digest).to_a.find do |r|
72
+ [@task.name, @task.params].eql? [r[:name], deserialize(r[:params])]
73
+ end
74
+ end
75
+ end
76
+
77
+ def id
78
+ @record[:id] if @record
79
+ end
80
+
81
+ def state
82
+ @record[:state] if @record
83
+ end
84
+
85
+ def running?
86
+ state == RUNNING
87
+ end
88
+
89
+ def successed?
90
+ state == SUCCESSED
91
+ end
92
+
93
+ def failed?
94
+ state == FAILED
95
+ end
96
+
97
+ def revoke
98
+ fail 'cannot revoke volatile task' if disabled?
99
+
100
+ database.transaction do
101
+ reload
102
+ fail 'task is not executed yet' if id.nil?
103
+ fail 'task is now running' if running?
104
+ dataset.where(id: id).delete
105
+ end
106
+
107
+ @record = nil
108
+ end
109
+
110
+ def start_session
111
+ return true if disabled?
112
+
113
+ database.transaction do
114
+ reload
115
+
116
+ return false if running?
117
+
118
+ new_state = {
119
+ state: RUNNING,
120
+ start_time: Time.now,
121
+ end_time: nil
122
+ }
123
+
124
+ if @record
125
+ dataset.where(id: @record[:id]).update(new_state)
126
+ @record = @record.merge(new_state)
127
+ else
128
+ id = dataset.insert(
129
+ digest: digest,
130
+ name: @task.name,
131
+ params: serialize(@task.params),
132
+ **new_state
133
+ )
134
+ @record = { id: id, **new_state }
135
+ end
136
+
137
+ true
138
+ end
139
+ end
140
+
141
+ def close_session(error = nil)
142
+ return if disabled?
143
+ return unless @record
144
+ state = error ? FAILED : SUCCESSED
145
+ dataset.where(id: @record[:id]).update(state: state, end_time: Time.now)
146
+ reload
147
+ end
148
+
149
+ def digest
150
+ Digest::MD5.hexdigest(@task.name + "\n" + serialize(@task.params))
151
+ end
152
+ end
153
+ end