nuri 0.5.3 → 0.5.4

data/examples/v2/tarpackage.sfp

@@ -0,0 +1,5 @@
+include "package.sfp"
+
+schema TarPackage extends Package {
+	final home = "/opt/local"
+}

data/lib/nuri.rb

@@ -1,16 +1,11 @@
-# external dependencies
-require 'rubygems'
-require 'json'
-require 'sfplanner'
-require 'colorize'
-require "coderay"
-require 'logger'
-require 'yaml'
-
 # define main module
 module Nuri
 	def self.windows?
-		RUBY_PLATFORM.downcase.include?("mswin")
+		platform.include?("mswin")
+	end
+
+	def self.platform
+		RUBY_PLATFORM.downcase
 	end
 
 	def self.home
@@ -52,6 +47,15 @@ module Nuri
 	end
 end
 
+# external dependencies
+require 'rubygems'
+require 'json'
+require 'sfplanner'
+require 'colorize'
+require "coderay"
+require 'logger'
+require 'yaml'
+
 # internal dependencies
 libdir = File.dirname(__FILE__) << '/nuri'
 ['constraint_helper.rb', 'helper.rb', 'orchestrator.rb',

data/lib/nuri/choreographer.rb

@@ -35,10 +35,10 @@ module Nuri::Choreographer
 			# generate local BSig model
 			local_bsigs = {}
 			self.get_agents.each do |name,model|
-				ref = model.ref
+				namespace = model.ref + '.'
 				local = {'id' => bsig['id'], 'operators' => [], 'goal' => {}}
-				bsig['operators'].each { |op| local['operators'] << op if op['name'][0, ref.length] == ref }
-				bsig['goal_operator'].each { |var,op_name| local['goal'][var] = bsig['goal'][var] if op_name[0, ref.length] == ref }
+				bsig['operators'].each { |op| local['operators'] << op if op['name'][0, namespace.length] == namespace }
+				bsig['goal_operator'].each { |var,op_name| local['goal'][var] = bsig['goal'][var] if op_name[0, namespace.length] == namespace }
 				local_bsigs[name] = local
 			end
 		end

data/lib/nuri/helper.rb

@@ -42,27 +42,30 @@ module Sfp::Helper
 end
 
 module Nuri::Helper
-	def post_data(address, port, path, data, open_timeout=5, read_timeout=1800)
+	DefaultHTTPOpenTimeout = 2
+	DefaultHTTPReadTimeout = 1800
+
+	def post_data(address, port, path, data, open_timeout=DefaultHTTPOpenTimeout, read_timeout=DefaultHTTPReadTimeout)
 		uri = create_uri(address, port, path)
 		req = Net::HTTP::Post.new(uri.path)
 		req.set_form_data(data)
 		http_request(uri, req, open_timeout, read_timeout)
 	end
 
-	def put_data(address, port, path, data, open_timeout=5, read_timeout=1800)
+	def put_data(address, port, path, data, open_timeout=DefaultHTTPOpenTimeout, read_timeout=DefaultHTTPReadTimeout)
 		uri = create_uri(address, port, path)
 		req = Net::HTTP::Put.new(uri.path)
 		req.set_form_data(data)
 		http_request(uri, req, open_timeout, read_timeout)
 	end
 
-	def get_data(address, port, path, open_timeout=5, read_timeout=1800)
+	def get_data(address, port, path, open_timeout=DefaultHTTPOpenTimeout, read_timeout=DefaultHTTPReadTimeout)
 		uri = create_uri(address, port, path)
 		req = Net::HTTP::Get.new(uri.path)
 		http_request(uri, req, open_timeout, read_timeout)
 	end
 
-	def delete_data(address, port, path, open_timeout=5, read_timeout=1800)
+	def delete_data(address, port, path, open_timeout=DefaultHTTPOpenTimeout, read_timeout=DefaultHTTPReadTimeout)
 		uri = create_uri(address, port, path)
 		req = Net::HTTP::Delete.new(uri.path)
 		http_request(uri, req, open_timeout, read_timeout)
@@ -80,14 +83,21 @@ module Nuri::Helper
 	end
 
 	def use_http_proxy?(uri)
-		ENV['no_proxy'].to_s.split(',').each { |pattern|
-			pattern.chop! if pattern[-1] == '*'
-			return false if uri.host[0,pattern.length] == pattern
-		}
-		true
+		parts = uri.host.split('.')
+		if parts[0] == '10' or
+			(parts[0] == '172' and parts[1] == '16') or
+			(parts[0] == '192' and parts[1] == '168')
+			false
+		else
+			ENV['no_proxy'].to_s.split(',').each { |pattern|
+				pattern.chop! if pattern[-1] == '*'
+				return false if uri.host[0,pattern.length] == pattern
+			}
+			true
+		end
 	end
 
-	def http_request(uri, request, open_timeout=5, read_timeout=1800)
+	def http_request(uri, request, open_timeout=DefaultHTTPOpenTimeout, read_timeout=DefaultHTTPReadTimeout)
 		if ENV['http_proxy'].to_s.strip != '' and use_http_proxy?(uri)
 			proxy = URI.parse(ENV['http_proxy'])
 			http = Net::HTTP::Proxy(proxy.host, proxy.port).new(uri.host, uri.port)

data/lib/nuri/master.rb

@@ -94,6 +94,7 @@ class Nuri::Master
 				node_name = name
 				node_state = get_node_state(node_name, !!p[:push_modules])
 				mutex.synchronize { state[node_name] = node_state }
+				state[node_name] = node_state
 			}
 		end
 		total = agents.keys.length - vms.keys.length
@@ -110,6 +111,7 @@ class Nuri::Master
 				node_name = name
 				node_state = get_node_state(node_name, !!p[:push_modules])
 				mutex.synchronize { state[node_name] = node_state }
+				state[node_name] = node_state
 			}
 		}
 
@@ -124,28 +126,35 @@ class Nuri::Master
 		# update <vm>.in_cloud value
 		update_cloud_vm_relations(state, vms)
 
+		agents.merge!(exist_vms)
+		push_agents_list(agents, {:reset => true})
+
 		state
 	end
 
 	protected
 	def format_benchmark(benchmark)
-		"cpu-time: user=#{benchmark.cutime.round(2)} sys=#{benchmark.cstime.round(2)} total=#{benchmark.total.round(2)}"
+		user = (benchmark.utime + benchmark.cutime).round(3)
+		system = (benchmark.stime + benchmark.cstime).round(3)
+		real = benchmark.real.round(3)
+		"benchmark: user=#{user} sys=#{system} real=#{real}"
 	end
 
-	def create_plan_task(p={})
+	def create_plan_task(opts={})
 		task = get_schemata
 
 		print "Getting current state "
-		puts (p[:color] ? "[Wait]".yellow : "[Wait]")
+		puts (opts[:color] ? "[Wait]".yellow : "[Wait]")
 
-		b = Benchmark.measure do
-			task['initial'] = to_state('initial', get_state(p))
+		benchmark = Benchmark.measure do
+			task['initial'] = to_state('initial', get_state(opts))
 		end
+		#puts YAML.dump(task['initial'])
 
-		print "Getting current state "
-		print (p[:color] ? "[OK]".green : "[OK]")
-		puts " " + format_benchmark(b)
+		puts "Getting current state " + (opts[:color] ? "[OK] ".green : "[OK] ") +
+		     format_benchmark(benchmark)
 
+		task['initial'].accept(FinalAttributeRemover)
 		task['initial'].accept(Sfp::Visitor::SfpGenerator.new(task))
 		f1 = Sfp::Helper::SfpFlatten.new
 		task['initial'].accept(f1)
@@ -162,26 +171,26 @@ class Nuri::Master
 		task['goal'] = goalgen.results
 
 		# find dead-node, remove from the task, print WARNING to the console
-		dead_nodes = task['initial'].select { |k,v| v.is_a?(Sfp::Unknown) }
-		dead_nodes.each_key { |name|
-			task['initial'].delete(name)
-			task['goal'].keep_if { |k,v| !(k =~ /(\$\.#{name}\.|\$\.#{name}$)/) }
-			print (p[:color] ? "[Warn]".red : "[Warn]")
-			puts " Removing node #{name} from the task."
-		}
+		init = task['initial']
+		init.keys.each do |name|
+			next if not init[name].is_a?(Sfp::Unknown)
+			init.delete(name)
+			task['goal'].keep_if { |var,val| !(var =~ /(\$\.#{name}\.|\$\.#{name}$)/) }
+			puts (opts[:color] ? "[Warn]".red : "[Warn]") + " Remove node #{name} from the planning task."
+		end
 
 		# print the status of goal state
 		puts "Goal state:"
-		goalgen.results.each { |k,v|
-			next if k[0,1] == '_'
+		goalgen.results.each { |var,val|
+			next if var[0] == '_'
 
-			print "  #{k}: "
-			value = Sfp::Helper::Sfp2Ruby.val(v['_value']).to_s
-			print (p[:color] ? value.green : value) + " "
+			print "  #{var}: "
+			value = Sfp::Helper::Sfp2Ruby.val(val['_value']).to_s
+			print (opts[:color] ? value.green : value)
 
-			if f1.results.has_key?(k) and f1.results[k] != v['_value']
-				value = Sfp::Helper::Sfp2Ruby.val(f1.results[k]).to_s
-				print (p[:color] ? value.red : value)
+			if f1.results.has_key?(var) and f1.results[var] != val['_value']
+				value = Sfp::Helper::Sfp2Ruby.val(f1.results[var]).to_s
+				print " ( " + (opts[:color] ? value.red : value) + " )"
 			end
 
 			puts ""
@@ -232,12 +241,13 @@ class Nuri::Master
 	end
 
 	def get_not_exist_vm_state(model)
-		s = {'state' => Sfp::Helper.deep_clone(model)}
+		state = Sfp::Helper.deep_clone(model)
+		s = {'state' => state}
 		s.accept(VisitorNotExistNodeState)
 		s.accept(ParentEliminator)
-		s['state']['created'] = false
-		s['state']['in_cloud'] = {'_context' => 'null', '_value' => CloudSchema}
-		s['state']
+		state['created'] = false
+		state['in_cloud'] = {'_context' => 'null', '_value' => CloudSchema}
+		state
 	end
 
 	def update_cloud_vm_relations(state, vms)
@@ -250,6 +260,7 @@ class Nuri::Master
 				next if not vms.has_key?(name)
 				if state[name].is_a?(Hash)
 					state[name]['in_cloud'] = cloud
+					state[name]['created'] = true
 				elsif state[name].is_a?(Sfp::Unknown)
 					state[name] = get_dead_vm_state(vms[name], cloud)
 				end
@@ -310,26 +321,33 @@ class Nuri::Master
 		true
 	end
 
-	def push_agents_list
+	def push_agents_list(agents=nil, options={})
 		begin
-			agents = {}
+			agents ||= get_agents
+			data = {}
 			# generate agents list
 			get_agents.each do |name, model|
 				next if not model['sfpAddress'].is_a?(String)
 				address = model['sfpAddress'].to_s.strip
 				port = model['sfpPort'].to_s.strip.to_i
 				next if address == '' or port <= 0
-				agents[name] = {:sfpAddress => address, :sfpPort => port}
+				data[name] = {:sfpAddress => address, :sfpPort => port}
 			end
-			data = {'agents' => JSON.generate(agents)}
+			json = {'agents' => JSON.generate(data)}
 
 			# send the list to all agents
-			agents.each do |name, agent|
-				code, _ = put_data(agent[:sfpAddress], agent[:sfpPort], '/agents', data, 5, 20)
+			data.each do |name, agent|
+				if options[:reset]
+					delete_data(agent[:sfpAddress], agent[:sfpPort], '/agents')
+				end
+
+				# update current agents list
+				code, _ = put_data(agent[:sfpAddress], agent[:sfpPort], '/agents', json, 5, 20)
 				raise Exception, "Push agents list to #{agent[:sfpAddress]}:#{agent[:sfpPort]} [Failed]" if code.to_i != 200
 			end
 			return true
 		rescue Exception => exp
+			#$stderr.puts "#{exp}\n#{exp.backtrace.join("\n")}"
 		end
 		false
 	end
@@ -359,9 +377,11 @@ class Nuri::Master
 
 		begin
 			# get modules list
-			code, body = get_data(address, port, '/modules')
+			code, body = get_data(address, port, '/modules', DefaultHTTPOpenTimeout, 5)
 			raise Exception, "Unable to get modules list from #{name}" if code.to_i != 200
 
+			#puts "#{name}'s modules: #{body}"
+
 			modules = JSON[body]
 			tobe_installed_modules = []
 			schemata.each do |name|
@@ -376,7 +396,9 @@ class Nuri::Master
 
 			### install new modules and replace old ones
 			list = tobe_installed_modules.join(" ")
-			output = JSON.parse(`cd #{@modules_dir}; #{InstallModule} #{address}:#{port} #{list}`)
+			cmd = "#{InstallModule} #{address}:#{port} #{list}"
+			#puts cmd
+			output = JSON.parse(`cd #{@modules_dir}; #{cmd}`)
 			if output['installed_modules'].length > 0
 				puts ("Push modules: " + output['installed_modules'].join(" ") + " to agent #{name} [OK]").green
 			end
@@ -435,7 +457,7 @@ class Nuri::Master
 			model = Sfp::Helper.deep_clone(model)
 			model.accept(ParentEliminator)
 			data = {'model' => JSON.generate(model)}
-			code, _ = put_data(address, port, '/model', data)
+			code, _ = put_data(address, port, '/model', data, DefaultHTTPOpenTimeout, 5)
 			return (code.to_i == 200)
 		end
 		false
@@ -448,7 +470,7 @@ class Nuri::Master
 		address = model[name]['sfpAddress'].to_s.strip
 		port = model[name]['sfpPort'].to_s.strip
 		if address != '' and port != ''
-			code, body = get_data(address, port, '/sfpstate')
+			code, body = get_data(address, port, '/sfpstate', DefaultHTTPOpenTimeout, 20)
 			if code.to_i == 200 and body.length >= 2
 				state = JSON[body]
 				return state['state'] if state.is_a?(Hash)
@@ -464,12 +486,9 @@ class Nuri::Master
 		}
 	end
 
-	def get_agents
-		Nuri::Master.agents(@model)
-	end
-
-	def self.agents(sfp)
-		sfp.select { |k,v| !(k[0] == '_' or not v.is_a?(Hash) or
+	def get_agents(model=nil)
+		model ||= @model
+		model.select { |k,v| !(k[0] == '_' or not v.is_a?(Hash) or
 			v['_context'] != 'object' or v['_classes'].index(AgentSchema).nil?)
 		}
 	end
@@ -497,8 +516,18 @@ class Nuri::Master
 
 	VisitorNotExistNodeState = Object.new
 	def VisitorNotExistNodeState.visit(name, value, parent)
+		parent.delete(name) if name == '_parent'
 		return false if name[0,1] == '_'
-		if not value.is_a?(Hash)
+		if value.is_a?(Hash)
+			case value['_context']
+			when 'null', 'any_value'
+				parent[name] = Sfp::Undefined.create(value['_isa'])
+			when 'object', 'procedure'
+				# do nothing
+			else
+				parent.delete(name)
+			end
+		else
 			if value.is_a?(String)
 				if value.isref
 					ref_value = parent.at?(value)
@@ -508,7 +537,7 @@ class Nuri::Master
 					elsif ref_value.is_a?(Sfp::Undefined) or ref_value.is_a?(Sfp::Unknown)
 						parent[name] = ref_value
 					else
-						puts "[WARN] Sfp::Undefined => #{parent.ref.push(name)}: #{ref_value.class.name}"
+						#puts "[WARN] Sfp::Undefined => #{parent.ref.push(name)}: #{ref_value.class.name}"
 						parent[name] = SfpUndefined
 					end
 				else
@@ -519,13 +548,9 @@ class Nuri::Master
 			elsif value.is_a?(TrueClass) or value.is_a?(FalseClass)
 				parent[name] = SfpUndefinedBoolean
 			else
-				puts "[WARN] Sfp::Undefined => " + parent.ref.push(name) + ": " + value.class.name
+				#puts "[WARN] Sfp::Undefined => " + parent.ref.push(name) + ": " + value.class.name
 				parent[name] = SfpUndefined
 			end
-		elsif value['_context'] == 'null' or value['_context'] == 'any_value'
-			parent[name] = Sfp::Undefined.create(value['_isa'])
-		elsif value['_context'] != 'object'
-			parent.delete(name)
 		end
 		true
 	end
@@ -543,7 +568,7 @@ class Nuri::Master
 					elsif ref_value.is_a?(Sfp::Unknown) or ref_value.is_a?(Sfp::Unknown)
 						parent[name] = ref_value
 					else
-						puts "[WARN] Sfp::Unknown => #{parent.ref.push(name)}: #{ref_value.class.name}"
+						#puts "[WARN] Sfp::Unknown => #{parent.ref.push(name)}: #{ref_value.class.name}"
 						parent[name] = SfpUnknown
 					end
 				else
@@ -554,7 +579,7 @@ class Nuri::Master
 			elsif value.is_a?(TrueClass) or value.is_a?(FalseClass)
 				parent[name] = Sfp::Unknown.create('$.Boolean')
 			else
-				puts "[WARN] Sfp::Unknown => " + parent.ref.push(name) + ": " + value.class.name
+				#puts "[WARN] Sfp::Unknown => " + parent.ref.push(name) + ": " + value.class.name
 				parent[name] = SfpUnknown
 			end
 		elsif value['_context'] == 'null' or value['_context'] == 'any_value'
@@ -628,8 +653,11 @@ class Nuri::Master
 				next if var.nil? # the variable is not found
 
 				if v.is_a?(Hash)
-					val = parser.types[v['_value']][0] if v['_context'] == 'null'
-					raise Exception, "Not implemented yet."      # this may arise on Set values
+					if v['_context'] == 'null'
+						val = parser.types[v['_value']][0]
+					else
+						raise Exception, "Not implemented yet."      # this may arise on Set values
+					end
 				else
 					val = v
 				end

data/lib/nuri/orchestrator.rb

@@ -164,7 +164,7 @@ module Nuri::Orchestrator
 							success = execute_action(action, options)
 						rescue Exception => exp
 							puts "Executing(#{i}) #{action[:string]} - thread ##{tid} " + (options[:color] ? "[Failed]".red : "[Failed]")
-							puts "#{exp}\n#{exp.backtrace.join("\n")}"
+							puts "#{exp}" #\n#{exp.backtrace.join("\n")}"
 						end
 						break if success
 					end

data/modules/.gitignore

@@ -1,4 +0,0 @@
-*.tgz
-cell
-autonomy*
-idol*

data/modules/README.md

@@ -0,0 +1,11 @@
+This directory contains the modules that control your resources. Each directory contains a module. You can add your own module by creating a new directory and required files.
+
+There are two types of module:
+  1. Native module - a module that is implemented in Ruby; it has two main files:
+    - `<module-name>.sfp` - contains the model of the resource in SFP language,
+    - `<module-name>.rb` - the implementation in Ruby.
+  2. Non-native module - a module that is implemented in other language (e.g. `pyfile`); it has two main files:
+    - `<module-name>.sfp` - contains the model of the resource in SFP language,
+    - `main` - the implementation in any programming language (must be executable through console).
+    
+You can put any necessary file to the directory. All files of the module's directory will be automatically pushed to the target agent by Nuri during deployment. Nuri will intelligently check any changes, and push the latest version.

data/modules/apache/apache.sfp

@@ -15,7 +15,8 @@ schema Apache extends Service {
 	// load balancer config
 	is_load_balancer : Bool
 	lb_members isset Node
-	lb_method : String  // byrequests, bytraffic, bybusyness
+	// byrequests, bytraffic, bybusyness
+	lb_method : String
 
 	sub set_port(target isref Integer) {
 		condition {