hq-engine 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,136 @@
1
+ require "hq/engine/errors"
2
+
3
+ module HQ
4
+ module Engine
5
+ class SubProcessRuleProvider
6
+
7
+ class Session
8
+
9
+ def initialize client, session_id
10
+ @client = client
11
+ @session_id = session_id
12
+ end
13
+
14
+ def set_library_module module_name, module_text
15
+
16
+ request = {
17
+ "name" => "set library module",
18
+ "arguments" => {
19
+ "session id" => @session_id,
20
+ "module name" => module_name,
21
+ "module text" => module_text,
22
+ }
23
+ }
24
+
25
+ reply = @client.perform request
26
+
27
+ case reply["name"]
28
+
29
+ when "ok"
30
+ # do nothing
31
+
32
+ else
33
+ raise "Unknown response: #{reply["name"]}"
34
+ end
35
+ end
36
+
37
+ def compile_xquery xquery_text, xquery_filename
38
+
39
+ request = {
40
+ "name" => "compile xquery",
41
+ "arguments" => {
42
+ "session id" => @session_id,
43
+ "xquery text" => xquery_text,
44
+ "xquery filename" => xquery_filename,
45
+ }
46
+ }
47
+
48
+ reply = @client.perform request
49
+
50
+ case reply["name"]
51
+
52
+ when "ok"
53
+
54
+ return reply["arguments"]["result text"]
55
+
56
+ when "error"
57
+
58
+ arguments = reply["arguments"]
59
+
60
+ exception = XQueryError.new
61
+ exception.file = arguments["file"]
62
+ exception.line = arguments["line"]
63
+ exception.column = arguments["column"]
64
+ exception.message = arguments["error"]
65
+
66
+ raise exception
67
+
68
+ else
69
+
70
+ raise "Unknown response: #{reply["name"]}"
71
+
72
+ end
73
+
74
+ end
75
+
76
+ def run_xquery input_text, &callback
77
+
78
+ request = {
79
+ "name" => "run xquery",
80
+ "arguments" => {
81
+ "session id" => @session_id,
82
+ "input text" => input_text,
83
+ }
84
+ }
85
+
86
+ # make call and process functions
87
+
88
+ reply = nil
89
+
90
+ loop do
91
+
92
+ reply = @client.perform request
93
+
94
+ break unless reply["name"] == "function call"
95
+
96
+ function_return_values =
97
+ callback.call \
98
+ reply["arguments"]["name"],
99
+ reply["arguments"]["arguments"]
100
+
101
+ request = {
102
+ "name" => "function return",
103
+ "arguments" => {
104
+ "values" => function_return_values,
105
+ },
106
+ }
107
+
108
+ end
109
+
110
+ # process response
111
+
112
+ case reply["name"]
113
+
114
+ when "ok"
115
+ return reply["arguments"]["result text"]
116
+
117
+ when "error"
118
+ arguments = reply["arguments"]
119
+ file = arguments["file"]
120
+ file = "file" if file.empty?
121
+ line = arguments["line"]
122
+ column = arguments["column"]
123
+ error = arguments["error"]
124
+ raise "#{file}:#{line}:#{column} #{error}"
125
+
126
+ else
127
+ raise "Unknown response: #{reply["name"]}"
128
+ end
129
+
130
+ end
131
+
132
+ end
133
+
134
+ end
135
+ end
136
+ end
@@ -0,0 +1,71 @@
1
+ module HQ
2
+ module Engine
3
+ class SubProcessRuleProvider
4
+
5
+ def self.start path_to_server
6
+
7
+ ctl_rd, ctl_wr = IO.pipe
8
+ req_rd, req_wr = IO.pipe
9
+ resp_rd, resp_wr = IO.pipe
10
+
11
+ File.executable? path_to_server \
12
+ or raise "Not found: #{path_to_server}"
13
+
14
+ # TODO why do we fork twice? i don't think we need to...
15
+
16
+ outer_pid = fork do
17
+
18
+ at_exit { exit! }
19
+
20
+ ctl_rd.close
21
+
22
+ inner_pid = fork do
23
+
24
+ $stdin.reopen req_rd
25
+ $stdout.reopen resp_wr
26
+ #$stderr.reopen "/dev/null", "w"
27
+
28
+ req_rd.close
29
+ resp_wr.close
30
+
31
+ exec path_to_server
32
+
33
+ end
34
+
35
+ ctl_wr.puts inner_pid
36
+
37
+ exit!
38
+
39
+ end
40
+
41
+ ctl_wr.close
42
+ req_rd.close
43
+ resp_wr.close
44
+
45
+ inner_pid = ctl_rd.gets.strip.to_i
46
+ ctl_rd.close
47
+
48
+ Process.wait outer_pid
49
+
50
+ at_exit do
51
+ begin
52
+ Process.kill "TERM", inner_pid
53
+ rescue Errno::ESRCH
54
+ # do nothing
55
+ end
56
+ end
57
+
58
+ require "hq/engine/subprocess-rule-provider/rule-provider"
59
+
60
+ client =
61
+ Client.new \
62
+ req_wr,
63
+ resp_rd
64
+
65
+ return client
66
+
67
+ end
68
+
69
+ end
70
+ end
71
+ end
@@ -0,0 +1,460 @@
1
+ require "hq/engine/libxmlruby-mixin"
2
+
3
+ module HQ
4
+ module Engine
5
+ class Transformer
6
+
7
+ include LibXmlRubyMixin
8
+
9
+ attr_accessor :parent
10
+
11
+ def logger() parent.logger end
12
+ def rule_provider() parent.rule_provider end
13
+
14
+ attr_accessor :schema_file
15
+ attr_accessor :rules_dir
16
+ attr_accessor :include_dir
17
+ attr_accessor :input_dir
18
+ attr_accessor :output_dir
19
+
20
+ attr_reader :data
21
+
22
+ def load_schema
23
+
24
+ @schemas =
25
+ load_schema_file schema_file
26
+
27
+ end
28
+
29
+ def load_rules
30
+
31
+ logger.debug "loading transformation rules"
32
+
33
+ @rules = {}
34
+
35
+ Dir.glob("#{rules_dir}/**/*").each do
36
+ |filename|
37
+
38
+ next unless filename =~ /^
39
+ #{Regexp.quote "#{rules_dir}/"}
40
+ (
41
+ (.+)
42
+ \. (xquery)
43
+ )
44
+ $/x
45
+
46
+ rule = {}
47
+ rule[:name] = $2
48
+ rule[:type] = $3
49
+ rule[:filename] = "#{$2}.#{$3}"
50
+ rule[:path] = filename
51
+ rule[:source] = File.read rule[:path]
52
+ rule[:in] = []
53
+ rule[:out] = []
54
+ rule[:source].scan(
55
+ /\(: (in|out) ([a-z0-9]+(?:-[a-z0-9]+)*) :\)$/
56
+ ).each do |type, name|
57
+ rule[type.to_sym] << name
58
+ end
59
+
60
+ @rules[rule[:name]] = rule
61
+
62
+ end
63
+
64
+ @rules = Hash[@rules.sort]
65
+
66
+ end
67
+
68
+ def init_rule_provider_session
69
+
70
+ @rule_provider_session =
71
+ rule_provider.session
72
+
73
+ # add hq module
74
+ # TODO move this somewhere
75
+
76
+ @rule_provider_session.set_library_module \
77
+ "hq",
78
+ "
79
+ module namespace hq = \"hq\";
80
+
81
+ declare function hq:get (
82
+ $id as xs:string
83
+ ) as element () ?
84
+ external;
85
+
86
+ declare function hq:get (
87
+ $type as xs:string,
88
+ $id-parts as xs:string *
89
+ ) as element () ?
90
+ external;
91
+
92
+ declare function hq:find (
93
+ $type as xs:string
94
+ ) as element () *
95
+ external;
96
+ "
97
+
98
+ end
99
+
100
+ def rebuild
101
+
102
+ logger.notice "performing transformation"
103
+
104
+ logger.time "performing transformation" do
105
+
106
+ remove_output
107
+
108
+ init_rule_provider_session
109
+
110
+ @data = {}
111
+
112
+ load_schema
113
+ load_rules
114
+ load_input
115
+ load_includes
116
+
117
+ @remaining_rules =
118
+ @rules.clone
119
+
120
+ pass_number = 0
121
+
122
+ loop do
123
+
124
+ num_processed =
125
+ rebuild_pass pass_number
126
+
127
+ break if num_processed == 0
128
+
129
+ pass_number += 1
130
+
131
+ end
132
+
133
+ end
134
+
135
+ return {
136
+ :success => @remaining_rules.empty?,
137
+ :remaining_rules => @remaining_rules.keys,
138
+ :missing_types =>
139
+ (
140
+ @remaining_rules
141
+ .values
142
+ .map { |rule| rule[:in] }
143
+ .flatten
144
+ .uniq
145
+ .sort
146
+ ) - (
147
+ @schema_types
148
+ .to_a
149
+ .select { |type| type =~ /^schema\// }
150
+ .map { |type| type.gsub /^schema\//, "" }
151
+ )
152
+ }
153
+
154
+ end
155
+
156
+ def load_input
157
+
158
+ logger.debug "reading input from disk"
159
+
160
+ logger.time "reading input from disk" do
161
+
162
+ Dir["#{input_dir}/*.xml"].each do
163
+ |filename|
164
+
165
+ input_data =
166
+ load_data_file filename
167
+
168
+ input_data.each do
169
+ |item_dom|
170
+
171
+ store_data item_dom
172
+
173
+ end
174
+
175
+ end
176
+
177
+ end
178
+
179
+ end
180
+
181
+ def load_includes
182
+
183
+ Dir["#{include_dir}/*.xquery"].each do
184
+ |path|
185
+
186
+ path =~ /^ #{Regexp.quote include_dir} \/ (.+) $/x
187
+ name = $1
188
+
189
+ @rule_provider_session.set_library_module \
190
+ name,
191
+ File.read(path)
192
+
193
+ end
194
+
195
+ end
196
+
197
+ def remove_output
198
+
199
+ if File.directory? output_dir
200
+ FileUtils.remove_entry_secure output_dir
201
+ end
202
+
203
+ FileUtils.mkdir output_dir
204
+
205
+ end
206
+
207
+ def rebuild_pass pass_number
208
+
209
+ logger.debug "beginning pass #{pass_number}"
210
+
211
+ @incomplete_types =
212
+ Set.new(
213
+ @remaining_rules.map {
214
+ |rule_name, rule|
215
+ rule[:out]
216
+ }.flatten.uniq.sort
217
+ )
218
+
219
+ @schema_types =
220
+ Set.new(
221
+ @schemas.keys
222
+ )
223
+
224
+ rules_for_pass =
225
+ Hash[
226
+ @remaining_rules.select do
227
+ |rule_name, rule|
228
+
229
+ missing_input_types =
230
+ rule[:in].select {
231
+ |in_type|
232
+ @incomplete_types.include? in_type
233
+ }
234
+
235
+ missing_input_schemas =
236
+ rule[:in].select {
237
+ |in_type|
238
+ ! @schema_types.include? "schema/#{in_type}"
239
+ }
240
+
241
+ missing_output_schemas =
242
+ rule[:out].select {
243
+ |out_type|
244
+ ! @schema_types.include? "schema/#{out_type}"
245
+ }
246
+
247
+ result = [
248
+ missing_input_types,
249
+ missing_input_schemas,
250
+ missing_output_schemas,
251
+ ].flatten.empty?
252
+
253
+ messages = []
254
+
255
+ messages << "incomplete inputs: %s" % [
256
+ missing_input_types.join(", "),
257
+ ] unless missing_input_types.empty?
258
+
259
+ messages << "missing input schemas: %s" % [
260
+ missing_input_schemas.join(", "),
261
+ ] unless missing_input_schemas.empty?
262
+
263
+ messages << "missing output schemas: %s" % [
264
+ missing_output_schemas.join(", "),
265
+ ] unless missing_output_schemas.empty?
266
+
267
+ unless messages.empty?
268
+ logger.debug "rule %s: %s" % [
269
+ rule_name,
270
+ messages.join("; "),
271
+ ]
272
+ end
273
+
274
+ result
275
+
276
+ end
277
+ ]
278
+
279
+ num_processed = 0
280
+
281
+ rules_for_pass.each do
282
+ |rule_name, rule|
283
+
284
+ used_types =
285
+ rebuild_one rule
286
+
287
+ missing_types =
288
+ used_types.select {
289
+ |type|
290
+ @incomplete_types.include? type
291
+ }
292
+
293
+ raise "Error" unless missing_types.empty?
294
+
295
+ if missing_types.empty?
296
+ @remaining_rules.delete rule_name
297
+ num_processed += 1
298
+ end
299
+
300
+ end
301
+
302
+ return num_processed
303
+
304
+ end
305
+
306
+ def rebuild_one rule
307
+
308
+ rule_name = rule[:name]
309
+ rule_type = rule[:type]
310
+
311
+ logger.debug "rebuilding rule #{rule_name}"
312
+ logger.time "rebuilding rule #{rule_name}" do
313
+
314
+ # perform query
315
+
316
+ used_types = Set.new
317
+ result_str = nil
318
+
319
+ begin
320
+
321
+ @rule_provider_session.compile_xquery \
322
+ rule[:source],
323
+ rule[:filename]
324
+
325
+ result_str =
326
+ @rule_provider_session.run_xquery \
327
+ "<xml/>" \
328
+ do
329
+ |name, args|
330
+
331
+ case name
332
+
333
+ when "get record by id"
334
+ args["id"] =~ /^([^\/]+)\//
335
+ used_types << $1
336
+ record = @data[args["id"]]
337
+ record ? [ record ] : []
338
+
339
+ when "get record by id parts"
340
+ used_types << args["type"]
341
+ id = [ args["type"], *args["id parts"] ].join "/"
342
+ record = @data[id]
343
+ record ? [ record ] : []
344
+
345
+ when "search records"
346
+ used_types << args["type"]
347
+ regex = /^#{Regexp.escape args["type"]}\//
348
+ @data \
349
+ .select { |id, record| id =~ regex }
350
+ .sort
351
+ .map { |id, record| record }
352
+
353
+ else
354
+ raise "No such function #{name}"
355
+
356
+ end
357
+
358
+ end
359
+
360
+ #puts "USED: #{used_types.to_a.join " "}"
361
+ #puts "INCOMPLETE: #{@incomplete_types.to_a.join " "}"
362
+ missing_types = used_types & @incomplete_types
363
+ puts "MISSING: #{missing_types.to_a.join " "}" unless missing_types.empty?
364
+ return missing_types unless missing_types.empty?
365
+
366
+ rescue RuleError => exception
367
+
368
+ logger.die "%s:%s:%s %s" % [
369
+ exception.file,
370
+ exception.line,
371
+ exception.column,
372
+ exception.message
373
+ ]
374
+
375
+ rescue => exception
376
+ logger.error "%s: %s" % [
377
+ exception.class,
378
+ exception.to_s,
379
+ ]
380
+ logger.detail exception.backtrace.join("\n")
381
+ FileUtils.touch "#{work_dir}/error-flag"
382
+ raise "error compiling #{rule[:path]}"
383
+ end
384
+
385
+ # process output
386
+
387
+ result_doms =
388
+ load_data_string result_str
389
+
390
+ result_doms.each do
391
+ |item_dom|
392
+
393
+ begin
394
+
395
+ item_id =
396
+ get_record_id_long \
397
+ @schemas,
398
+ item_dom
399
+
400
+ rescue => e
401
+
402
+ logger.die "record id error for %s created by %s" % [
403
+ item_dom.name,
404
+ rule_name,
405
+ ]
406
+
407
+ end
408
+
409
+ store_data item_dom
410
+
411
+ end
412
+
413
+ return []
414
+
415
+ end
416
+
417
+ end
418
+
419
+ def store_data item_dom
420
+
421
+ # determine id
422
+
423
+ item_id =
424
+ get_record_id_long \
425
+ @schemas,
426
+ item_dom
427
+
428
+ if @data[item_id]
429
+ raise "duplicate record id #{item_id}"
430
+ end
431
+
432
+ # store in memory
433
+
434
+ item_xml =
435
+ to_xml_string item_dom
436
+
437
+ @data[item_id] =
438
+ item_xml
439
+
440
+ # store in filesystem
441
+
442
+ item_path =
443
+ "#{output_dir}/data/#{item_id}.xml"
444
+
445
+ item_dir =
446
+ File.dirname item_path
447
+
448
+ FileUtils.mkdir_p \
449
+ item_dir
450
+
451
+ File.open item_path, "w" do
452
+ |file_io|
453
+ file_io.puts item_xml
454
+ end
455
+
456
+ end
457
+
458
+ end
459
+ end
460
+ end
@@ -0,0 +1,60 @@
1
+ module HQ
2
+ module Engine
3
+ class UnlockCommand
4
+
5
+ attr_accessor :hq
6
+
7
+ def couch() hq.couch end
8
+ def logger() hq.logger end
9
+
10
+ def go command_name
11
+
12
+ locks =
13
+ couch.get "mandar-locks"
14
+
15
+ if locks["deploy"]
16
+
17
+ if locks["deploy"]["role"] == $deploy_role
18
+
19
+ logger.warning "unlocking deployment for role " +
20
+ "#{locks["deploy"]["role"]}"
21
+
22
+ locks["deploy"] = nil
23
+
24
+ else
25
+
26
+ logger.error "not unlocking deployment for role " +
27
+ "#{locks["deploy"]["role"]}"
28
+
29
+ end
30
+
31
+ end
32
+
33
+ locks["changes"].each do
34
+ |role, change|
35
+
36
+ next if change["state"] == "stage"
37
+
38
+ if role == $deploy_role
39
+
40
+ logger.warning "unlocking changes in state " +
41
+ "#{change["state"]} for role #{role}"
42
+
43
+ change["state"] = "stage"
44
+
45
+ else
46
+
47
+ logger.warning "not unlocking changes in state " +
48
+ "#{change["state"]} for role #{role}"
49
+
50
+ end
51
+
52
+ end
53
+
54
+ couch.update locks
55
+
56
+ end
57
+
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,9 @@
1
+ module HQ
2
+ module Engine
3
+
4
+ describe Transformer do
5
+
6
+ end
7
+
8
+ end
9
+ end