hq-engine 0.0.1 → 0.0.2

data/lib/hq/engine/subprocess-rule-provider/session.rb

@@ -0,0 +1,136 @@
+require "hq/engine/errors"
+
+module HQ
+module Engine
+class SubProcessRuleProvider
+
+class Session
+
+	def initialize client, session_id
+		@client = client
+		@session_id = session_id
+	end
+
+	def set_library_module module_name, module_text
+
+		request = {
+			"name" => "set library module",
+			"arguments" => {
+				"session id" => @session_id,
+				"module name" => module_name,
+				"module text" => module_text,
+			}
+		}
+
+		reply = @client.perform request
+
+		case reply["name"]
+
+			when "ok"
+				# do nothing
+
+			else
+				raise "Unknown response: #{reply["name"]}"
+		end
+	end
+
+	def compile_xquery xquery_text, xquery_filename
+
+		request = {
+			"name" => "compile xquery",
+			"arguments" => {
+				"session id" => @session_id,
+				"xquery text" => xquery_text,
+				"xquery filename" => xquery_filename,
+			}
+		}
+
+		reply = @client.perform request
+
+		case reply["name"]
+
+			when "ok"
+
+				return reply["arguments"]["result text"]
+
+			when "error"
+
+				arguments = reply["arguments"]
+
+				exception = XQueryError.new
+				exception.file = arguments["file"]
+				exception.line = arguments["line"]
+				exception.column = arguments["column"]
+				exception.message = arguments["error"]
+
+				raise exception
+
+			else
+
+				raise "Unknown response: #{reply["name"]}"
+
+		end
+
+	end
+
+	def run_xquery input_text, &callback
+
+		request = {
+			"name" => "run xquery",
+			"arguments" => {
+				"session id" => @session_id,
+				"input text" => input_text,
+			}
+		}
+
+		# make call and process functions
+
+		reply = nil
+
+		loop do
+
+			reply = @client.perform request
+
+			break unless reply["name"] == "function call"
+
+			function_return_values =
+				callback.call \
+					reply["arguments"]["name"],
+					reply["arguments"]["arguments"]
+
+			request = {
+				"name" => "function return",
+				"arguments" => {
+					"values" => function_return_values,
+				},
+			}
+
+		end
+
+		# process response
+
+		case reply["name"]
+
+			when "ok"
+				return reply["arguments"]["result text"]
+
+			when "error"
+				arguments = reply["arguments"]
+				file = arguments["file"]
+				file = "file" if file.empty?
+				line = arguments["line"]
+				column = arguments["column"]
+				error = arguments["error"]
+				raise "#{file}:#{line}:#{column} #{error}"
+
+			else
+				raise "Unknown response: #{reply["name"]}"
+		end
+
+	end
+
+end
+
+end
+end
+end

data/lib/hq/engine/subprocess-rule-provider/start.rb

@@ -0,0 +1,71 @@
+module HQ
+module Engine
+class SubProcessRuleProvider
+
+	def self.start path_to_server
+
+		ctl_rd, ctl_wr = IO.pipe
+		req_rd, req_wr = IO.pipe
+		resp_rd, resp_wr = IO.pipe
+
+		File.executable? path_to_server \
+			or raise "Not found: #{path_to_server}"
+
+		# TODO why do we fork twice? i don't think we need to...
+
+		outer_pid = fork do
+
+			at_exit { exit! }
+
+			ctl_rd.close
+
+			inner_pid = fork do
+
+				$stdin.reopen req_rd
+				$stdout.reopen resp_wr
+				#$stderr.reopen "/dev/null", "w"
+
+				req_rd.close
+				resp_wr.close
+
+				exec path_to_server
+
+			end
+
+			ctl_wr.puts inner_pid
+
+			exit!
+
+		end
+
+		ctl_wr.close
+		req_rd.close
+		resp_wr.close
+
+		inner_pid = ctl_rd.gets.strip.to_i
+		ctl_rd.close
+
+		Process.wait outer_pid
+
+		at_exit do
+			begin
+				Process.kill "TERM", inner_pid
+			rescue Errno::ESRCH
+				# do nothing
+			end
+		end
+
+		require "hq/engine/subprocess-rule-provider/rule-provider"
+
+		client =
+			Client.new \
+				req_wr,
+				resp_rd
+
+		return client
+
+	end
+
+end
+end
+end

data/lib/hq/engine/transformer.rb

@@ -0,0 +1,460 @@
+require "hq/engine/libxmlruby-mixin"
+
+module HQ
+module Engine
+class Transformer
+
+	include LibXmlRubyMixin
+
+	attr_accessor :parent
+
+	def logger() parent.logger end
+	def rule_provider() parent.rule_provider end
+
+	attr_accessor :schema_file
+	attr_accessor :rules_dir
+	attr_accessor :include_dir
+	attr_accessor :input_dir
+	attr_accessor :output_dir
+
+	attr_reader :data
+
+	def load_schema
+
+		@schemas =
+			load_schema_file schema_file
+
+	end
+
+	def load_rules
+
+		logger.debug "loading transformation rules"
+
+		@rules = {}
+
+		Dir.glob("#{rules_dir}/**/*").each do
+			|filename|
+
+			next unless filename =~ /^
+				#{Regexp.quote "#{rules_dir}/"}
+				(
+					(.+)
+					\. (xquery)
+				)
+			$/x
+
+			rule = {}
+			rule[:name] = $2
+			rule[:type] = $3
+			rule[:filename] = "#{$2}.#{$3}"
+			rule[:path] = filename
+			rule[:source] = File.read rule[:path]
+			rule[:in] = []
+			rule[:out] = []
+			rule[:source].scan(
+				/\(: (in|out) ([a-z0-9]+(?:-[a-z0-9]+)*) :\)$/
+			).each do |type, name|
+				rule[type.to_sym] << name
+			end
+
+			@rules[rule[:name]] = rule
+
+		end
+
+		@rules = Hash[@rules.sort]
+
+	end
+
+	def init_rule_provider_session
+
+		@rule_provider_session =
+			rule_provider.session
+
+		# add hq module
+		# TODO move this somewhere
+
+		@rule_provider_session.set_library_module \
+			"hq",
+			"
+				module namespace hq = \"hq\";
+
+				declare function hq:get (
+					$id as xs:string
+				) as element () ?
+				external;
+
+				declare function hq:get (
+					$type as xs:string,
+					$id-parts as xs:string *
+				) as element () ?
+				external;
+
+				declare function hq:find (
+					$type as xs:string
+				) as element () *
+				external;
+			"
+
+	end
+
+	def rebuild
+
+		logger.notice "performing transformation"
+
+		logger.time "performing transformation" do
+
+			remove_output
+
+			init_rule_provider_session
+
+			@data = {}
+
+			load_schema
+			load_rules
+			load_input
+			load_includes
+
+			@remaining_rules =
+				@rules.clone
+
+			pass_number = 0
+
+			loop do
+
+				num_processed =
+					rebuild_pass pass_number
+
+				break if num_processed == 0
+
+				pass_number += 1
+
+			end
+
+		end
+
+		return {
+			:success => @remaining_rules.empty?,
+			:remaining_rules => @remaining_rules.keys,
+			:missing_types =>
+				(
+					@remaining_rules
+						.values
+						.map { |rule| rule[:in] }
+						.flatten
+						.uniq
+						.sort
+				) - (
+					@schema_types
+						.to_a
+						.select { |type| type =~ /^schema\// }
+						.map { |type| type.gsub /^schema\//, "" }
+				)
+		}
+
+	end
+
+	def load_input
+
+		logger.debug "reading input from disk"
+
+		logger.time "reading input from disk" do
+
+			Dir["#{input_dir}/*.xml"].each do
+				|filename|
+
+				input_data =
+					load_data_file filename
+
+				input_data.each do
+					|item_dom|
+
+					store_data item_dom
+
+				end
+
+			end
+
+		end
+
+	end
+
+	def load_includes
+
+		Dir["#{include_dir}/*.xquery"].each do
+			|path|
+
+			path =~ /^ #{Regexp.quote include_dir} \/ (.+) $/x
+			name = $1
+
+			@rule_provider_session.set_library_module \
+				name,
+				File.read(path)
+
+		end
+
+	end
+
+	def remove_output
+
+		if File.directory? output_dir
+			FileUtils.remove_entry_secure output_dir
+		end
+
+		FileUtils.mkdir output_dir
+
+	end
+
+	def rebuild_pass pass_number
+
+		logger.debug "beginning pass #{pass_number}"
+
+		@incomplete_types =
+			Set.new(
+				@remaining_rules.map {
+					|rule_name, rule|
+					rule[:out]
+				}.flatten.uniq.sort
+			)
+
+		@schema_types =
+			Set.new(
+				@schemas.keys
+			)
+
+		rules_for_pass =
+			Hash[
+				@remaining_rules.select do
+					|rule_name, rule|
+
+					missing_input_types =
+						rule[:in].select {
+							|in_type|
+							@incomplete_types.include? in_type
+						}
+
+					missing_input_schemas =
+						rule[:in].select {
+							|in_type|
+							! @schema_types.include? "schema/#{in_type}"
+						}
+
+					missing_output_schemas =
+						rule[:out].select {
+							|out_type|
+							! @schema_types.include? "schema/#{out_type}"
+						}
+
+					result = [
+						missing_input_types,
+						missing_input_schemas,
+						missing_output_schemas,
+					].flatten.empty?
+
+					messages = []
+
+					messages << "incomplete inputs: %s" % [
+						missing_input_types.join(", "),
+					] unless missing_input_types.empty?
+
+					messages << "missing input schemas: %s" % [
+						missing_input_schemas.join(", "),
+					] unless missing_input_schemas.empty?
+
+					messages << "missing output schemas: %s" % [
+						missing_output_schemas.join(", "),
+					] unless missing_output_schemas.empty?
+
+					unless messages.empty?
+						logger.debug "rule %s: %s" % [
+							rule_name,
+							messages.join("; "),
+						]
+					end
+
+					result
+
+				end
+			]
+
+		num_processed = 0
+
+		rules_for_pass.each do
+			|rule_name, rule|
+
+			used_types =
+				rebuild_one rule
+
+			missing_types =
+				used_types.select {
+					|type|
+					@incomplete_types.include? type
+				}
+
+			raise "Error" unless missing_types.empty?
+
+			if missing_types.empty?
+				@remaining_rules.delete rule_name
+				num_processed += 1
+			end
+
+		end
+
+		return num_processed
+
+	end
+
+	def rebuild_one rule
+
+		rule_name = rule[:name]
+		rule_type = rule[:type]
+
+		logger.debug "rebuilding rule #{rule_name}"
+		logger.time "rebuilding rule #{rule_name}" do
+
+			# perform query
+
+			used_types = Set.new
+			result_str = nil
+
+			begin
+
+				@rule_provider_session.compile_xquery \
+					rule[:source],
+					rule[:filename]
+
+				result_str =
+					@rule_provider_session.run_xquery \
+						"<xml/>" \
+				do
+					|name, args|
+
+					case name
+
+					when "get record by id"
+						args["id"] =~ /^([^\/]+)\//
+						used_types << $1
+						record = @data[args["id"]]
+						record ? [ record ] : []
+
+					when "get record by id parts"
+						used_types << args["type"]
+						id = [ args["type"], *args["id parts"] ].join "/"
+						record = @data[id]
+						record ? [ record ] : []
+
+					when "search records"
+						used_types << args["type"]
+						regex = /^#{Regexp.escape args["type"]}\//
+						@data \
+							.select { |id, record| id =~ regex }
+							.sort
+							.map { |id, record| record }
+
+					else
+						raise "No such function #{name}"
+
+					end
+
+				end
+
+#puts "USED: #{used_types.to_a.join " "}"
+#puts "INCOMPLETE: #{@incomplete_types.to_a.join " "}"
+				missing_types = used_types & @incomplete_types
+puts "MISSING: #{missing_types.to_a.join " "}" unless missing_types.empty?
+				return missing_types unless missing_types.empty?
+
+			rescue RuleError => exception
+
+				logger.die "%s:%s:%s %s" % [
+					exception.file,
+					exception.line,
+					exception.column,
+					exception.message
+				]
+
+			rescue => exception
+				logger.error "%s: %s" % [
+					exception.class,
+					exception.to_s,
+				]
+				logger.detail exception.backtrace.join("\n")
+				FileUtils.touch "#{work_dir}/error-flag"
+				raise "error compiling #{rule[:path]}"
+			end
+
+			# process output
+
+			result_doms =
+				load_data_string result_str
+
+			result_doms.each do
+				|item_dom|
+
+				begin
+
+					item_id =
+						get_record_id_long \
+							@schemas,
+							item_dom
+
+				rescue => e
+
+					logger.die "record id error for %s created by %s" % [
+						item_dom.name,
+						rule_name,
+					]
+
+				end
+
+				store_data item_dom
+
+			end
+
+			return []
+
+		end
+
+	end
+
+	def store_data item_dom
+
+		# determine id
+
+		item_id =
+			get_record_id_long \
+				@schemas,
+				item_dom
+
+		if @data[item_id]
+			raise "duplicate record id #{item_id}"
+		end
+
+		# store in memory
+
+		item_xml =
+			to_xml_string item_dom
+
+		@data[item_id] =
+			item_xml
+
+		# store in filesystem
+
+		item_path =
+			"#{output_dir}/data/#{item_id}.xml"
+
+		item_dir =
+			File.dirname item_path
+
+		FileUtils.mkdir_p \
+			item_dir
+
+		File.open item_path, "w" do
+			|file_io|
+			file_io.puts item_xml
+		end
+
+	end
+
+end
+end
+end

data/lib/hq/engine/unlock-command.rb

@@ -0,0 +1,60 @@
+module HQ
+module Engine
+class UnlockCommand
+
+	attr_accessor :hq
+
+	def couch() hq.couch end
+	def logger() hq.logger end
+
+	def go command_name
+
+		locks =
+			couch.get "mandar-locks"
+
+		if locks["deploy"]
+
+			if locks["deploy"]["role"] == $deploy_role
+
+				logger.warning "unlocking deployment for role " +
+					"#{locks["deploy"]["role"]}"
+
+				locks["deploy"] = nil
+
+			else
+
+				logger.error "not unlocking deployment for role " +
+					"#{locks["deploy"]["role"]}"
+
+			end
+
+		end
+
+		locks["changes"].each do
+			|role, change|
+
+			next if change["state"] == "stage"
+
+			if role == $deploy_role
+
+				logger.warning "unlocking changes in state " +
+					"#{change["state"]} for role #{role}"
+
+				change["state"] = "stage"
+
+			else
+
+				logger.warning "not unlocking changes in state " +
+					"#{change["state"]} for role #{role}"
+
+			end
+
+		end
+
+		couch.update locks
+
+	end
+
+end
+end
+end

data/spec/hq/engine/transformer-spec.rb

@@ -0,0 +1,9 @@
+module HQ
+module Engine
+
+describe Transformer do
+
+end
+
+end
+end