abid 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +306 -4
- data/abid.gemspec +1 -0
- data/lib/Abidfile.rb +1 -80
- data/lib/abid.rb +3 -1
- data/lib/abid/application.rb +21 -17
- data/lib/abid/concurrent_extention.rb +6 -0
- data/lib/abid/concurrent_extention/ivar.rb +12 -0
- data/lib/abid/dsl_definition.rb +2 -2
- data/lib/abid/params_parser.rb +1 -1
- data/lib/abid/play.rb +19 -60
- data/lib/abid/rake_extensions/task.rb +82 -71
- data/lib/abid/state.rb +63 -8
- data/lib/abid/task.rb +101 -8
- data/lib/abid/task_manager.rb +13 -23
- data/lib/abid/tasks/core.rake +85 -0
- data/lib/abid/version.rb +1 -1
- data/lib/abid/worker.rb +26 -16
- metadata +19 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7cc7876b6fa9c07f29b893a57ffca6222569385c
|
4
|
+
data.tar.gz: fabd1296f57c4ad14b58564882250b71a917fe68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1fcf95c39e0b23ee3bb8116d5e227d27ff9439fc83bbcbd2cbff7bd86b0f8a4ec837d430af09b6dd0e71643328367a7daee483157bfdc53cc868dc2050876190
|
7
|
+
data.tar.gz: 57b0276deeaef0a06f3e35aa57674f2cc7d8eb15494e9b14b2fab717b9397e23bd74f460854779beba8afe48e397fc64a0104c414740cf5d6331a59f78340acd
|
data/README.md
CHANGED
@@ -41,8 +41,8 @@ play :fetch_source do
|
|
41
41
|
param :date, type: :date
|
42
42
|
|
43
43
|
def run
|
44
|
-
mkdir 'out'
|
45
44
|
open('http://example.com') do |f|
|
45
|
+
FileUtils.makedirs "out/#{date.strftime('%Y-%m-%d')}"
|
46
46
|
File.write("out/#{date.strftime('%Y-%m-%d')}/example.com", f.read)
|
47
47
|
end
|
48
48
|
end
|
@@ -51,7 +51,7 @@ end
|
|
51
51
|
play :count do
|
52
52
|
param :date, type: :date
|
53
53
|
|
54
|
-
|
54
|
+
setup do
|
55
55
|
needs 'fetch_source', date: date
|
56
56
|
end
|
57
57
|
|
@@ -67,11 +67,313 @@ Then you can invoke the task:
|
|
67
67
|
$ bundle exec abid count date=2016-01-01
|
68
68
|
```
|
69
69
|
|
70
|
-
This Abidfile has two tasks: `fetch_source` and `count`. They
|
70
|
+
This Abidfile has two tasks: `fetch_source` and `count`. They are kinds of rake tasks, but they have some additional features:
|
71
71
|
|
72
72
|
* A play can take parameters. They are declared with `param` keyword, and passed via environment variables.
|
73
73
|
* All play results are saved to the external database. If a play is invoked twice with same parameters, it will be ignored.
|
74
|
-
* Depending tasks can be declared in `setup`
|
74
|
+
* Depending tasks can be declared in `setup` block. If a depending task is a play task, parameters can be specified.
|
75
|
+
|
76
|
+
|
77
|
+
## Execution Model
|
78
|
+
|
79
|
+
When a play is invoked, its parameters and results are saved in a database by default.
|
80
|
+
If the play has been executed with same parameters and successed, it will not be executed any more.
|
81
|
+
|
82
|
+
```ruby
|
83
|
+
# Abidfile.rb
|
84
|
+
play :test do
|
85
|
+
params :name, type: :string
|
86
|
+
def run
|
87
|
+
puts name
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
$ abid test name=apple #=> "apple"
|
92
|
+
$ abid test name=apple # nothing happens
|
93
|
+
$ abid test name=orange #=> "orange"
|
94
|
+
```
|
95
|
+
|
96
|
+
Normal rake task results are not stored in DB.
|
97
|
+
They are always executed even if they have been executed and successed.
|
98
|
+
These tasks are called "volatile".
|
99
|
+
|
100
|
+
If prerequisites tasks have been failed, the subsequent task also fails.
|
101
|
+
When prerequisites failed, you have to manually fix a problem and re-execute them.
|
102
|
+
|
103
|
+
```ruby
|
104
|
+
# Abidfile.rb
|
105
|
+
play :query do
|
106
|
+
def run
|
107
|
+
result = `mysql -e 'SELECT COUNT(*) FROM users'`
|
108
|
+
File.write(result, 'result.txt')
|
109
|
+
end
|
110
|
+
end
|
111
|
+
play :report do
|
112
|
+
setup { needs :query }
|
113
|
+
def run
|
114
|
+
`cat result.txt | sendmail all@example.com`
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
$ abid query #=> Failed because of MySQL server down
|
119
|
+
$ abid report #=> Fails because prerequisites failed
|
120
|
+
$ ... # restart MySQL server
|
121
|
+
$ abid query #=> ok
|
122
|
+
$ abid report #=> ok
|
123
|
+
|
124
|
+
```
|
125
|
+
|
126
|
+
### Volatile plays
|
127
|
+
|
128
|
+
Abid plays can also be volatile.
|
129
|
+
|
130
|
+
```
|
131
|
+
play :voaltile_play do
|
132
|
+
set :volatile, true
|
133
|
+
def run
|
134
|
+
...
|
135
|
+
end
|
136
|
+
end
|
137
|
+
```
|
138
|
+
|
139
|
+
These plays are not stored in DB and will be always executed.
|
140
|
+
|
141
|
+
### Repair mode
|
142
|
+
|
143
|
+
When abid is executed with `--repair` flag, failed prerequisites are re-executed and successed tasks are executed only when their prerequisites are executed.
|
144
|
+
|
145
|
+
```
|
146
|
+
$ abid report #=> Failed because of MySQL server down
|
147
|
+
$ ... # restart MySQL server
|
148
|
+
$ abid --repair report #=> :query and :report tasks are executed
|
149
|
+
```
|
150
|
+
|
151
|
+
### Parallel execution
|
152
|
+
|
153
|
+
All tasks are executed in a thread pool.
|
154
|
+
|
155
|
+
By default, the thread pool size is 1, i.e. all tasks are executed in single thread. When `-m` option is given, the thread pool size is decided from CPU size. You can specify the thread pool size by `-j` option.
|
156
|
+
|
157
|
+
Abid supports multiple thread pools.
|
158
|
+
Each tasks can be executed in different thread pools.
|
159
|
+
|
160
|
+
```ruby
|
161
|
+
define_worker :copy, 2
|
162
|
+
define_worker :hive, 4
|
163
|
+
|
164
|
+
play :copy_source_1 do
|
165
|
+
set :worker, :copy
|
166
|
+
def run
|
167
|
+
...
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
play :hive_query_1 do
|
172
|
+
set :worker, :hive
|
173
|
+
def run
|
174
|
+
...
|
175
|
+
end
|
176
|
+
end
|
177
|
+
```
|
178
|
+
|
179
|
+
Two thread pools `copy` and `hive` are defined in above example, each thread pool sizes are 2 and 4.
|
180
|
+
`:copy_source_1` is executed in `copy` thread pool, and `:hive_query_1` is executed in `hive` thread pool.
|
181
|
+
|
182
|
+
## Plays detail
|
183
|
+
|
184
|
+
### Params
|
185
|
+
|
186
|
+
```ruby
|
187
|
+
play :sample do
|
188
|
+
param :name, type: :string
|
189
|
+
param :date, type: :date, default: Date.today - 1
|
190
|
+
|
191
|
+
def run
|
192
|
+
date #=> #<Date: ????-??-?? ((0000000j,0s,0n),+0s,0000000j)>
|
193
|
+
end
|
194
|
+
end
|
195
|
+
```
|
196
|
+
|
197
|
+
Each parameters are initialized by corresponding environment variables. If no environment variable with same name is found, default will be used.
|
198
|
+
|
199
|
+
Abid supports following types:
|
200
|
+
|
201
|
+
- `boolean`
|
202
|
+
- `int`
|
203
|
+
- `float`
|
204
|
+
- `string`
|
205
|
+
- `date`
|
206
|
+
- `datetime`
|
207
|
+
- `time`
|
208
|
+
|
209
|
+
### Settings
|
210
|
+
|
211
|
+
```ruby
|
212
|
+
play :count do
|
213
|
+
set(:file_path, './sql/count.sql')
|
214
|
+
set(:query) { File.read(file_path) }
|
215
|
+
|
216
|
+
def run
|
217
|
+
query #=> the contents of './sql/count.sql'
|
218
|
+
end
|
219
|
+
end
|
220
|
+
```
|
221
|
+
|
222
|
+
Plays settings can be declared by `set` and referenced in `run` method.
|
223
|
+
If block given, it is evaluated in the same context as `run` method.
|
224
|
+
The block is called only once and its result is cached.
|
225
|
+
|
226
|
+
Following settings are used in abid core:
|
227
|
+
- `worker`
|
228
|
+
- `volatile`
|
229
|
+
|
230
|
+
### Dependencies
|
231
|
+
|
232
|
+
```ruby
|
233
|
+
play :sample do
|
234
|
+
param :name, type: :string
|
235
|
+
setup do
|
236
|
+
needs "parent_task:#{name}"
|
237
|
+
end
|
238
|
+
|
239
|
+
def run
|
240
|
+
# executed after `parent_task`
|
241
|
+
end
|
242
|
+
end
|
243
|
+
```
|
244
|
+
|
245
|
+
All prerequisites must be declared in `setup` block.
|
246
|
+
You can refer parameters and settings in `setup` block.
|
247
|
+
|
248
|
+
### Callbacks
|
249
|
+
|
250
|
+
```ruby
|
251
|
+
play :sample do
|
252
|
+
def run
|
253
|
+
...
|
254
|
+
end
|
255
|
+
|
256
|
+
before do
|
257
|
+
# executed before running
|
258
|
+
end
|
259
|
+
|
260
|
+
after do
|
261
|
+
# executed when the task successed
|
262
|
+
end
|
263
|
+
|
264
|
+
around do |body|
|
265
|
+
body.call # `run` method is called
|
266
|
+
ensure
|
267
|
+
... # executed even if the task failed
|
268
|
+
end
|
269
|
+
end
|
270
|
+
```
|
271
|
+
|
272
|
+
### Extending plays
|
273
|
+
|
274
|
+
You can extend plays in object-oriented style.
|
275
|
+
All parameters, settings and methods are inherited.
|
276
|
+
|
277
|
+
```ruby
|
278
|
+
play :abstract_count do
|
279
|
+
def run
|
280
|
+
`hive -f #{file_path}`
|
281
|
+
end
|
282
|
+
end
|
283
|
+
|
284
|
+
play :count, extends: :abstract_count do
|
285
|
+
set :file_path, 'sql/count.sql'
|
286
|
+
end
|
287
|
+
|
288
|
+
---
|
289
|
+
|
290
|
+
$ abid count #=> hive -f sql/count.sql
|
291
|
+
```
|
292
|
+
|
293
|
+
Common base play can be defined by `play_base` keyword:
|
294
|
+
|
295
|
+
```
|
296
|
+
play_base do
|
297
|
+
param :date, type: :date
|
298
|
+
end
|
299
|
+
```
|
300
|
+
|
301
|
+
All plays inherit the `play_base` definition.
|
302
|
+
|
303
|
+
### Plays internal
|
304
|
+
|
305
|
+
The play implementation can be illustrated as below:
|
306
|
+
|
307
|
+
```ruby
|
308
|
+
play :sample do
|
309
|
+
param :date, type: :date
|
310
|
+
set(:file_path, 'sql/count.sql')
|
311
|
+
set(:query) { File.read(file_path) }
|
312
|
+
def run
|
313
|
+
...
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
# <=>
|
318
|
+
|
319
|
+
class Sample < Abid::Play
|
320
|
+
attr_reader :date
|
321
|
+
def initialize(date)
|
322
|
+
@date = Date.parse(date)
|
323
|
+
end
|
324
|
+
|
325
|
+
def file_path
|
326
|
+
'sql/count.sql'
|
327
|
+
end
|
328
|
+
def query
|
329
|
+
@query ||= File.read(file_path)
|
330
|
+
end
|
331
|
+
|
332
|
+
def run
|
333
|
+
...
|
334
|
+
end
|
335
|
+
end
|
336
|
+
|
337
|
+
task :sample do
|
338
|
+
Sample.new(ENV['date']).run
|
339
|
+
end
|
340
|
+
```
|
341
|
+
|
342
|
+
When play is defined, new subclass of Avid::Play is created and play body is evaluated in that new class context. So, any class goodies can be put in play's body, i.e. including modules, `attr_reader` / `attr_writer`, method definitions, etc..
|
343
|
+
|
344
|
+
## Built-in tasks
|
345
|
+
|
346
|
+
### `state:list`
|
347
|
+
|
348
|
+
```
|
349
|
+
$ abid state:list started_after="2000-01-01 00:00:00" started_before="2000=01-02 00:00:00"
|
350
|
+
```
|
351
|
+
|
352
|
+
Display plays current states.
|
353
|
+
|
354
|
+
### `state:revoke`
|
355
|
+
|
356
|
+
```
|
357
|
+
$ abid state:revoke[id]
|
358
|
+
```
|
359
|
+
|
360
|
+
Remove the play recored from DB.
|
361
|
+
|
362
|
+
### `state:assume`
|
363
|
+
|
364
|
+
```
|
365
|
+
$ abid state:assume[task_name] date=2000-01-01
|
366
|
+
```
|
367
|
+
|
368
|
+
Insert a record that the play successed into DB.
|
369
|
+
|
370
|
+
### `db:migrate`
|
371
|
+
|
372
|
+
```
|
373
|
+
$ abid db:migrate
|
374
|
+
```
|
375
|
+
|
376
|
+
Initialize or update DB.
|
75
377
|
|
76
378
|
## Development
|
77
379
|
|
data/abid.gemspec
CHANGED
@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
|
|
21
21
|
|
22
22
|
spec.add_development_dependency "bundler", "~> 1.10"
|
23
23
|
spec.add_development_dependency "minitest"
|
24
|
+
spec.add_development_dependency "pry-byebug"
|
24
25
|
|
25
26
|
spec.add_dependency "rake", "~> 10.0"
|
26
27
|
spec.add_dependency "concurrent-ruby-ext"
|
data/lib/Abidfile.rb
CHANGED
@@ -1,80 +1 @@
|
|
1
|
-
|
2
|
-
task default: :list
|
3
|
-
|
4
|
-
desc 'Show play histories'
|
5
|
-
play :list do
|
6
|
-
set :volatile, true
|
7
|
-
|
8
|
-
param :started_after, type: :time, default: nil
|
9
|
-
param :started_before, type: :time, default: nil
|
10
|
-
|
11
|
-
def run
|
12
|
-
states = Abid::State.list(
|
13
|
-
started_after: started_after,
|
14
|
-
started_before: started_before
|
15
|
-
)
|
16
|
-
|
17
|
-
table = states.map do |state|
|
18
|
-
params = state[:params].map do |k, v|
|
19
|
-
v.to_s =~ /\s/ ? "#{k}='#{v}'" : "#{k}=#{v}"
|
20
|
-
end.join(' ')
|
21
|
-
[
|
22
|
-
state[:id].to_s,
|
23
|
-
state[:state].to_s,
|
24
|
-
state[:name],
|
25
|
-
params,
|
26
|
-
state[:start_time].to_s,
|
27
|
-
state[:end_time].to_s
|
28
|
-
]
|
29
|
-
end
|
30
|
-
|
31
|
-
header = %w(id state name params start_time end_time)
|
32
|
-
|
33
|
-
tab_width = header.each_with_index.map do |c, i|
|
34
|
-
[c.length, table.map { |row| row[i].length }.max || 0].max
|
35
|
-
end
|
36
|
-
|
37
|
-
header.each_with_index do |c, i|
|
38
|
-
print c.ljust(tab_width[i] + 2)
|
39
|
-
end
|
40
|
-
puts
|
41
|
-
|
42
|
-
header.each_with_index do |_, i|
|
43
|
-
print '-' * (tab_width[i] + 2)
|
44
|
-
end
|
45
|
-
puts
|
46
|
-
|
47
|
-
table.map do |row|
|
48
|
-
row.each_with_index do |v, i|
|
49
|
-
print v.ljust(tab_width[i] + 2)
|
50
|
-
end
|
51
|
-
puts
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
namespace :db do
|
58
|
-
desc 'Delete play history'
|
59
|
-
task :revoke, [:task] do |_t, args|
|
60
|
-
task = Rake.application[args[:task]]
|
61
|
-
state = Abid::State.find(task)
|
62
|
-
state.revoke
|
63
|
-
end
|
64
|
-
|
65
|
-
desc 'Run migrations'
|
66
|
-
task :migrate, [:version] do |_t, args|
|
67
|
-
migrations_path = File.expand_path('../../migrations', __FILE__)
|
68
|
-
|
69
|
-
require 'sequel'
|
70
|
-
Sequel.extension :migration
|
71
|
-
db = Rake.application.database
|
72
|
-
if args[:version]
|
73
|
-
puts "Migrating to version #{args[:version]}"
|
74
|
-
Sequel::Migrator.run(db, migrations_path, target: args[:version].to_i)
|
75
|
-
else
|
76
|
-
puts 'Migrating to latest'
|
77
|
-
Sequel::Migrator.run(db, migrations_path)
|
78
|
-
end
|
79
|
-
end
|
80
|
-
end
|
1
|
+
# allows the built-in tasks to load without a abidfile
|
data/lib/abid.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
require 'rake'
|
2
2
|
require 'date'
|
3
3
|
require 'digest/md5'
|
4
|
-
require '
|
4
|
+
require 'English'
|
5
5
|
require 'forwardable'
|
6
|
+
require 'monitor'
|
6
7
|
require 'time'
|
7
8
|
require 'yaml'
|
8
9
|
require 'concurrent'
|
@@ -12,6 +13,7 @@ require 'sqlite3'
|
|
12
13
|
require 'sequel'
|
13
14
|
|
14
15
|
require 'abid/rake_extensions'
|
16
|
+
require 'abid/concurrent_extention'
|
15
17
|
require 'abid/version'
|
16
18
|
require 'abid/waiter'
|
17
19
|
require 'abid/worker'
|