bricolage 5.29.2 → 6.0.0beta4

Sign up to get free protection for your applications and to get access to all the features.
@@ -11,7 +11,7 @@ module Bricolage
11
11
  alias s sql_string_literal
12
12
 
13
13
  def escape_sql_string(s)
14
- s.gsub(/'/, "''")
14
+ s.gsub(/'/, "''").gsub(/\\/, '\\\\')
15
15
  end
16
16
 
17
17
  def sql_timestamp_literal(time)
@@ -24,6 +24,48 @@ module Bricolage
24
24
  time.strftime('%Y-%m-%d %H:%M:%S')
25
25
  end
26
26
 
27
+ def compile_set_expr(values_hash)
28
+ columns = values_hash.keys.map(&:to_s).join(', ')
29
+ values = values_hash.values.map{|v| convert_value(v) }.join(', ')
30
+ return columns, values
31
+ end
32
+
33
+ def convert_value(value)
34
+ if value == :now
35
+ 'now()'
36
+ elsif value.nil?
37
+ "null"
38
+ elsif value == true or value == false
39
+ "#{value.to_s}"
40
+ elsif value.instance_of?(Integer) or value.instance_of?(Float)
41
+ "#{value.to_s}"
42
+ elsif value.instance_of?(String) or value.instance_of?(Pathname)
43
+ "#{s(value.to_s)}"
44
+ else
45
+ raise "invalid type for 'value' argument in JobExecution#convert_value: #{value} is #{value.class}"
46
+ end
47
+ end
48
+
49
+ def compile_where_expr(conds_hash)
50
+ conds_hash.map{|k,v| convert_cond(k,v) }.join(' and ')
51
+ end
52
+
53
+ def convert_cond(column, cond)
54
+ if cond.nil?
55
+ "#{column} is null"
56
+ elsif cond.instance_of?(Array) # not support subquery
57
+ in_clause = cond.map{|c| convert_cond(column, c)}.join(' or ')
58
+ "(#{in_clause})"
59
+ elsif cond == true or cond == false
60
+ "#{column} is #{cond.to_s}"
61
+ elsif cond.instance_of?(Integer) or cond.instance_of?(Float)
62
+ "#{column} = #{cond}"
63
+ elsif cond.instance_of?(String) or cond.instance_of?(Pathname)
64
+ "#{column} = #{s(cond.to_s)}"
65
+ else
66
+ raise "invalid type for 'cond' argument in JobExecution#convert_cond: #{cond} is #{cond.class}"
67
+ end
68
+ end
27
69
  end
28
70
 
29
71
  end
@@ -1,11 +1,17 @@
1
1
  require 'bricolage/jobnet'
2
+ require 'bricolage/sqlutils'
2
3
  require 'bricolage/exception'
4
+ require 'bricolage/dao/job'
5
+ require 'bricolage/dao/jobnet'
6
+ require 'bricolage/dao/jobexecution'
3
7
  require 'fileutils'
4
8
  require 'pathname'
9
+ require 'pg'
5
10
 
6
11
  module Bricolage
7
12
 
8
- class TaskQueue
13
+ class MemoryTaskQueue
14
+
9
15
  def initialize
10
16
  @queue = []
11
17
  end
@@ -18,77 +24,90 @@ module Bricolage
18
24
  @queue.size
19
25
  end
20
26
 
21
- def queued?
22
- not empty?
23
- end
24
-
25
27
  def each(&block)
26
28
  @queue.each(&block)
27
29
  end
28
30
 
29
31
  def consume_each
30
- lock
31
- save
32
- while task = self.next
33
- yield task
34
- deq
32
+ while job = @queue.first
33
+ result = yield job
34
+ break unless result.success?
35
+ @queue.shift
35
36
  end
36
- ensure
37
- unlock
38
37
  end
39
38
 
40
- def enq(task)
41
- @queue.push task
39
+ def restore_jobnet(jobnet)
42
40
  end
43
41
 
44
- def next
45
- @queue.first
42
+ def enqueue_jobnet(jobnet)
43
+ jobnet.sequential_jobs.each do |job|
44
+ @queue.push job
45
+ end
46
46
  end
47
47
 
48
- def deq
49
- task = @queue.shift
50
- save
51
- task
48
+ def locked?(jobnet)
49
+ false
52
50
  end
53
51
 
54
- def save
52
+ def unlock_help(jobnet)
53
+ raise "[BUG] this message must not be shown"
55
54
  end
56
55
 
57
- def restore
56
+ def cancel_jobnet(jobnet, message)
57
+ @queue.clear
58
58
  end
59
59
 
60
- def locked?
61
- false
60
+ end
61
+
62
+
63
+ class FileTaskQueue
64
+
65
+ def initialize(path:)
66
+ @path = path
67
+ @queue = []
62
68
  end
63
69
 
64
- def lock
70
+ def empty?
71
+ @queue.empty?
65
72
  end
66
73
 
67
- def unlock
74
+ def size
75
+ @queue.size
68
76
  end
69
77
 
70
- def unlock_help
71
- "[MUST NOT HAPPEN] this message must not be shown"
78
+ def restore_jobnet(jobnet)
79
+ return unless File.exist?(@path)
80
+ File.foreach(@path) do |line|
81
+ @queue.push Task.deserialize(line)
82
+ end
72
83
  end
73
- end
74
84
 
75
- class FileTaskQueue < TaskQueue
76
- def FileTaskQueue.restore_if_exist(path)
77
- q = new(path)
78
- q.restore if q.queued?
79
- q
85
+ def enqueue_jobnet(jobnet)
86
+ jobnet.sequential_jobs.each do |ref|
87
+ @queue.push Task.new(ref)
88
+ end
80
89
  end
81
90
 
82
- def initialize(path)
83
- super()
84
- @path = path
91
+ def each
92
+ @queue.each do |task|
93
+ yield task.job
94
+ end
85
95
  end
86
96
 
87
- def queued?
88
- @path.exist?
97
+ def consume_each
98
+ lock
99
+ save
100
+ while task = @queue.first
101
+ task_result = yield task.job
102
+ break unless task_result.success?
103
+ @queue.shift
104
+ save
105
+ end
106
+ ensure
107
+ unlock
89
108
  end
90
109
 
91
- def save
110
+ private def save
92
111
  if empty?
93
112
  @path.unlink if @path.exist?
94
113
  return
@@ -97,7 +116,7 @@ module Bricolage
97
116
  tmpname = "#{@path}.tmp.#{Process.pid}"
98
117
  begin
99
118
  File.open(tmpname, 'w') {|f|
100
- each do |task|
119
+ @queue.each do |task|
101
120
  f.puts task.serialize
102
121
  end
103
122
  }
@@ -107,48 +126,187 @@ module Bricolage
107
126
  end
108
127
  end
109
128
 
110
- def restore
111
- File.foreach(@path) do |line|
112
- enq JobTask.deserialize(line)
113
- end
114
- end
115
-
116
- def locked?
129
+ def locked?(jobnet)
117
130
  lock_file_path.exist?
118
131
  end
119
132
 
120
- def lock
121
- FileUtils.touch lock_file_path
133
+ private def lock
134
+ FileUtils.touch(lock_file_path)
122
135
  end
123
136
 
124
- def unlock
125
- FileUtils.rm_f lock_file_path
137
+ private def unlock
138
+ FileUtils.rm_f(lock_file_path)
126
139
  end
127
140
 
128
- def lock_file_path
141
+ private def lock_file_path
129
142
  Pathname.new("#{@path}.LOCK")
130
143
  end
131
144
 
132
- def unlock_help
145
+ def unlock_help(jobnet)
133
146
  "remove the file: #{lock_file_path}"
134
147
  end
148
+
149
+ def cancel_jobnet(jobnet, message)
150
+ unlock
151
+ FileUtils.rm_f(@path)
152
+ @queue.clear
153
+ end
154
+
155
+ class Task
156
+ def initialize(job)
157
+ @job = job
158
+ end
159
+
160
+ attr_reader :job
161
+
162
+ def serialize
163
+ [@job].join("\t")
164
+ end
165
+
166
+ def Task.deserialize(str)
167
+ job, * = str.strip.split("\t")
168
+ new(JobNet::Ref.parse(job))
169
+ end
170
+ end
171
+
135
172
  end
136
173
 
137
- class JobTask
138
- def initialize(job)
139
- @job = job
174
+
175
+ class DatabaseTaskQueue
176
+
177
+ def initialize(datasource:, executor_id:, enable_lock: false)
178
+ @ds = datasource
179
+ @executor_id = executor_id
180
+ @enable_lock = enable_lock
181
+
182
+ @queue = []
183
+ @jobnet_dao = DAO::JobNet.new(@ds)
184
+ @job_dao = DAO::Job.new(@ds)
185
+ @jobexecution_dao = DAO::JobExecution.new(@ds)
186
+ @jobnet = nil
187
+ end
188
+
189
+ def empty?
190
+ @queue.empty?
191
+ end
192
+
193
+ def size
194
+ @queue.size
195
+ end
196
+
197
+ private def find_or_create_jobnet(ref)
198
+ @jobnet_rec ||= @jobnet_dao.find_or_create(ref)
199
+ end
200
+
201
+ def restore_jobnet(jobnet)
202
+ raise "jobnet is already bound to queue" if @jobnet
203
+
204
+ job_executions = @jobexecution_dao.enqueued_jobs(jobnet.ref)
205
+ unless job_executions.empty?
206
+ job_executions.each do |job_execution|
207
+ @queue.push Task.for_job_execution(job_execution)
208
+ end
209
+ @jobnet = jobnet
210
+ end
211
+ end
212
+
213
+ def enqueue_jobnet(jobnet)
214
+ raise "jobnet is already bound to queue" if @jobnet
215
+
216
+ jobnet_rec = find_or_create_jobnet(jobnet.ref)
217
+ jobnet.sequential_jobs.each_with_index do |job_ref, index|
218
+ job = @job_dao.find_or_create(jobnet_rec.id, job_ref)
219
+ job_execution = @jobexecution_dao.enqueue_job(job, index + 1)
220
+ @queue.push Task.for_job_execution(job_execution)
221
+ end
222
+ @jobnet = jobnet
223
+ end
224
+
225
+ def each
226
+ @queue.each do |task|
227
+ yield task.job
228
+ end
140
229
  end
141
230
 
142
- attr_reader :job
231
+ def consume_each
232
+ raise "jobnet is not bound to queue" unless @jobnet
233
+
234
+ jobnet_rec = find_or_create_jobnet(@jobnet.ref)
235
+ @jobnet_dao.lock(jobnet_rec.id, @executor_id) if @enable_lock
236
+ while task = @queue.first
237
+ @job_dao.lock(task.job_id, @executor_id) if @enable_lock
238
+ begin
239
+ @jobexecution_dao.transition_to_running(task.job_execution_id)
240
+
241
+ # Note: fork(2) breaks current connections,
242
+ # we must close current connections before fork.
243
+ # (psql datasource forks process)
244
+ @ds.clear_connection_pool
245
+
246
+ job_completed = false
247
+ begin
248
+ task_result = yield task.job
143
249
 
144
- def serialize
145
- [@job].join("\t")
250
+ if task_result.success?
251
+ @jobexecution_dao.transition_to_succeeded(task.job_execution_id)
252
+ job_completed = true
253
+ @queue.shift
254
+ else
255
+ @jobexecution_dao.transition_to_failed(task.job_execution_id, task_result.message)
256
+ job_completed = true
257
+ break
258
+ end
259
+ ensure
260
+ unless job_completed
261
+ begin
262
+ @jobexecution_dao.transition_to_failed(task.job_execution_id, 'unexpected error')
263
+ rescue => ex
264
+ $stderr.puts "warning: could not write job state: #{ex.class}: #{ex.message} (this error is ignored)"
265
+ end
266
+ end
267
+ end
268
+ ensure
269
+ @job_dao.unlock(task.job_id, @executor_id) if @enable_lock
270
+ end
271
+ end
272
+ ensure
273
+ @jobnet_dao.unlock(jobnet_rec.id, @executor_id) if @enable_lock
146
274
  end
147
275
 
148
- def JobTask.deserialize(str)
149
- job, * = str.strip.split("\t")
150
- new(JobNet::Ref.parse(job))
276
+ def locked?(jobnet)
277
+ @jobnet_dao.locked?(jobnet.ref)
151
278
  end
279
+
280
+ def unlock_help(jobnet)
281
+ jobnet_rec = find_or_create_jobnet(jobnet.ref)
282
+ locked_jobs = @job_dao.locked_jobs(jobnet_rec.id)
283
+ "clear executor_id of the jobnet (id: #{jobnet_rec.id}) and/or the jobs (id: #{locked_jobs.map(&:id).join(', ')})"
284
+ end
285
+
286
+ def cancel_jobnet(jobnet, message)
287
+ @jobexecution_dao.cancel_jobnet(jobnet.ref, message)
288
+ jobnet_rec = find_or_create_jobnet(jobnet.ref)
289
+ @jobnet_dao.clear_lock(jobnet_rec.id)
290
+ @job_dao.clear_lock_all(jobnet_rec.id)
291
+ end
292
+
293
+ class Task
294
+ def Task.for_job_execution(exec)
295
+ job_ref = JobNet::JobRef.new(exec.subsystem, exec.job_name, JobNet::Location.dummy)
296
+ new(job_ref, exec)
297
+ end
298
+
299
+ def initialize(job_ref, job_execution)
300
+ @job = job_ref
301
+ @job_id = job_execution.job_id
302
+ @job_execution_id = job_execution.job_execution_id
303
+ end
304
+
305
+ attr_reader :job
306
+ attr_reader :job_id
307
+ attr_reader :job_execution_id
308
+ end
309
+
152
310
  end
153
311
 
154
312
  end
@@ -1,4 +1,4 @@
1
1
  module Bricolage
2
2
  APPLICATION_NAME = 'Bricolage'
3
- VERSION = '5.29.2'
3
+ VERSION = '6.0.0beta4'
4
4
  end
@@ -0,0 +1,13 @@
1
+ FROM ruby:2.6.5-stretch
2
+
3
+ RUN apt-get update -qq && apt-get install -y build-essential libpq-dev libpq5 postgresql-client
4
+
5
+ COPY ./Gemfile /tmp/Gemfile
6
+ COPY ./Gemfile.lock /tmp/Gemfile.lock
7
+ RUN cd /tmp && bundle install -j4 --deployment --without 'development test'
8
+
9
+ WORKDIR /app
10
+ COPY . /app
11
+ RUN cp -a /tmp/vendor /app/
12
+
13
+ CMD ["bundle", "exec", "ridgepole", "-f", "Schemafile", "-c", "database.yml", "--merge", "--dry-run"]
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'ridgepole'
4
+ gem 'pg'