nuri 0.5.1 → 0.5.2

data/modules/hadoop1/hadoop1.sfp

@@ -1,4 +1,5 @@
 include "../service/service.sfp"
+include "../aptpackage/aptpackage.sfp"
 
 /**
  * Module for managing Hadoop version 1
@@ -17,24 +18,28 @@ schema Hadoop1 {
 	final password = "!"
 	final source = "http://www.mirrorservice.org/sites/ftp.apache.org/hadoop/common"
 	final version = "1.2.1"
-	final java_home = "" //"/usr/local/java/jre1.6.0_34"
-	final data_dir = "/tmp/hadoop"
+	final java_home = ""
+	final data_dir = "/opt/hadoop/data"
 	final replication = 2
 
+	java isa AptPackage {
+		package_name = "default-jre"
+	}
+
 	sub install {
 		condition {
 			this.installed != true
+			this.java.installed = true
 		}
 		effect {
 			this.installed = true
-			this.running = false
 		}
 	}
 	
 	sub uninstall {
 		condition {
-			this.running = false
 			this.installed = true
+			this.running = false
 		}
 		effect {
 			this.installed = false
@@ -43,8 +48,8 @@ schema Hadoop1 {
 
 	sub start {
 		condition {
-			this.running != true
 			this.installed = true
+			this.running != true
 		}
 		effect {
 			this.running = true
@@ -53,6 +58,7 @@ schema Hadoop1 {
 
 	sub stop {
 		condition {
+			this.installed = true
 			this.running = true
 		}
 		effect {
@@ -61,20 +67,25 @@ schema Hadoop1 {
 	}
 }
 
-schema Hadoop1Master extends Hadoop1 { }
-
+schema Hadoop1Master extends Hadoop1 {
+	home = "/opt/hadoopmaster"
+	data_dir = "/opt/hadoopmaster/data"
+}
 
 schema Hadoop1Slave extends Hadoop1 {
 	master isref Hadoop1Master
 
+	home = "/opt/hadoopslave"
+	data_dir = "/opt/hadoopslave/data"
+
 	sub install (master : Hadoop1Master) {
 		condition {
 			this.installed != true
 			master.parent.created = true
+			this.java.installed = true
 		}
 		effect {
 			this.installed = true
-			this.running = false
 			this.master = master
 		}
 	}

data/modules/hadoop2/hadoop2.rb

@@ -139,7 +139,7 @@ module Sfp::Module::Hadoop2Common
 			'dfs_datanode_data_dir' => @model['data_dir'] + "/datanode_data",
 			'mapreduce_framework_name' => 'yarn',
 			'mapreduce_map_memory_mb' => 1536,
-			'mapreduce_map_java_opts' => '-Xmx1024M',
+			'mapreduce_map_java_opts' => '-Xmx2048M',
 			'mapreduce_reduce_memory_mb' => 3072,
 			'mapreduce_reduce_java_opts' => '-Xmx2560M',
 			'mapreduce_task_io_sort_mb' => 512,
@@ -150,7 +150,7 @@ module Sfp::Module::Hadoop2Common
 			'yarn_log_aggregation_enable' => false,
 			'yarn_resourcemanager_scheduler_class' => 'org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler',
 			'yarn_scheduler_minimum_allocation_mb' => 256,
-			'yarn_scheduler_maximum_allocation_mb' => 1024,
+			'yarn_scheduler_maximum_allocation_mb' => 2048,
 			'yarn_nodemanager_resource_memory_mb' => 512,
 			'yarn_nodemanager_vmem_pmem_ratio' => 2,
 			'yarn_nodemanager_log_retain_seconds' => 10800,
@@ -159,10 +159,16 @@ module Sfp::Module::Hadoop2Common
 			'yarn_log_aggregation_retain_check_interval_seconds' => -1,
 			'yarn_nodemanager_local_dirs' => @model['data_dir'] + "/yarn_local_dir",
 			'yarn_nodemanager_log_dirs' => @model['data_dir'] + "/yarn_log_dir",
+			'yarn_web_proxy_address' => local_address,
 			'io_file_buffer_size' => 131072,
 		}
 	end
 
+	def local_address
+		domain = `dnsdomainname`.to_s.strip
+		`hostname`.to_s.strip + (domain.length > 0 ? '.' + domain : '')
+	end
+
 	# TODO -- user "useradd" and "groupadd"
 	def add_user_group
 		user = @model['user']
@@ -308,7 +314,7 @@ class Sfp::Module::Hadoop2Master < Sfp::Module::TarPackage
 
 	def map
 		map = common_map
-		map['master'] = `hostname`.to_s.strip
+		map['master'] = local_address
 		map
 	end
 
@@ -392,6 +398,7 @@ class Sfp::Module::Hadoop2Slave < Sfp::Module::TarPackage
 	def map
 		map = common_map
 		map['master'] = resolve(@model['master'] + '.parent.sfpAddress')
+		map['yarn_web_proxy_address'] = resolve(@model['master'] + '.parent.sfpAddress')
 		map
 	end
 

data/modules/hadoop2/hadoop2.sfp

@@ -14,7 +14,7 @@ schema Hadoop2 extends TarPackage {
 	final group = "hadoop"
 	final password = "!"
 	final java_home = ""
-	final data_dir = "/tmp/hadoop"
+	final data_dir = "/opt/hadoop/data"
 
 	sub install {
 		condition {

data/modules/hadoop2/yarn-site.xml

@@ -18,70 +18,43 @@
 
 <property>
         <name>yarn.acl.enable</name>
-        <value>false</value>
+        <value><%= yarn_acl_enable %></value>
 </property> 
 
 <property>
         <name>yarn.admin.acl</name>
-        <value>*</value>
+        <value><%= yarn_admin_acl %></value>
 </property> 
 
 <property>
         <name>yarn.log-aggregation-enable</name>
-        <value>false</value>
+        <value><%= yarn_log_aggregation_enable %></value>
 </property> 
 
 <property>
-        <name>yarn.resourcemanager.address</name>
-        <value><%= master %>:54311</value>
-</property> 
-<property>
-        <name>yarn.resourcemanager.scheduler.address</name>
-        <value><%= master %>:54312</value>
-</property> 
-<property>
-        <name>yarn.resourcemanager.admin.address</name>
-        <value><%= master %>:54313</value>
-</property> 
-<property>
-        <name>yarn.resourcemanager.webapp.address</name>
-        <value><%= master %>:54314</value>
-</property> 
+        <name>yarn.resourcemanager.hostname</name>
+        <value><%= master %></value>
+</property>
+
 <property>
         <name>yarn.resourcemanager.scheduler.class</name>
-        <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
+        <value><%= yarn_resourcemanager_scheduler_class %></value>
 </property>
 <property>
         <name>yarn.web-proxy.address</name>
-        <value><%= master %>:54315</value>
+        <value><%= yarn_web_proxy_address %>:54315</value>
 </property>
-<property>
-        <name>yarn.scheduler.minimum-allocation-mb</name>
-        <value>256</value>
-</property> 
-<property>
-        <name>yarn.scheduler.maximum-allocation-mb</name>
-        <value>1024</value>
-</property> 
-<property>
-        <name>yarn.nodemanager.resource.memory-mb</name>
-        <value>512</value>
-</property> 
-<property>
-        <name>yarn.nodemanager.vmem-pmem-ratio</name>
-        <value>2</value>
-</property> 
 <property>
         <name>yarn.nodemanager.local-dirs</name>
-        <value><%= yarn_local_dir %></value>
+        <value><%= yarn_nodemanager_local_dirs %></value>
 </property> 
 <property>
         <name>yarn.nodemanager.log-dirs</name>
-        <value><%= yarn_log_dir %></value>
+        <value><%= yarn_nodemanager_log_dirs %></value>
 </property> 
 <property>
         <name>yarn.nodemanager.log.retain-seconds</name>
-        <value>10800</value>
+        <value><%= yarn_nodemanager_log_retain_seconds %></value>
 </property> 
 <property>
         <name>yarn.nodemanager.remote-app-log-dir</name>
@@ -93,15 +66,7 @@
 </property> 
 <property>
         <name>yarn.nodemanager.aux-services</name>
-        <value>mapreduce_shuffle</value>
-</property> 
-<property>
-        <name>yarn.log-aggregation.retain-seconds</name>
-        <value>-1</value>
-</property> 
-<property>
-        <name>yarn.log-aggregation.retain-check-interval-seconds</name>
-        <value>-1</value>
+        <value><%= yarn_nodemanager_aux_services %></value>
 </property> 
 
 </configuration>

data/modules/install_module

@@ -1,65 +1,54 @@
-#!/bin/bash
-
-DefaultPort=1314
-
-# verify the arguments
-if [[ "$1" == "" ]] || [[ "$2" == "" ]]; then
-	echo "Usage: install_module <address> [port] <module-name> ..."
-	exit 1
-fi
-
-# set the agent's address
-address=$1
-shift
-
-# set the agent's port number
-re='^[0-9]+$'
-if [[ $1 =~ $re ]]; then
-	port=$1
-	shift
-else
-	port=$DefaultPort
-fi
-
-# set a template command for sending the modules
-cmd="curl -s -i -X PUT $address:$port/modules"
-
-# setup directory for temporary files
-dir="/tmp/"$(date +%s%N)"$RANDOM"
-mkdir -p $dir
-
-# for every module in the arguments:
-# - archive the module's files to a temporary file
-# - update the sending command by adding the module
-# if the module is not exist then the program will
-# set missing_module flag and then break from the loop
-missing_module=0
-for module in "$@" ; do
-	if [[ -d "$module" ]]; then
-		tar cvzhf $dir/$module.tgz $module 1>/dev/null
-		cmd="$cmd -F $module=@$dir/$module.tgz"
-	else
-		echo "Module $module is not exist!"
-		missing_module=`expr $missing_module + 1`
-	fi
-done
-
-if [[ $missing_module == 0 ]]; then
-	# execute the sending command there is no missing module
-	result=`$cmd`
-	re='.*HTTP/1\.1 200 OK.*'
-	if ! [[ $result =~ $re ]]; then
-		missing_module=`expr $missing_module + 1`
-	fi
-fi
-
-# delete directory that holds temporary files
-rm -rf $dir
-
-if [[ $missing_module == 0 ]]; then
-	echo "status: ok"
-else
-	echo "status: failed"
-fi
-
-exit $missing_module
+#!/usr/bin/env ruby
+
+require 'json'
+require 'uri'
+require 'net/http'
+require File.dirname(__FILE__) + '/../lib/nuri/targz'
+
+module Nuri
+	module Util
+		TarGzip = Object.new.extend(Util::Tar)
+	
+		def self.install_modules(address, port, modules, protocol="http")
+			data = {}
+			modules.each do |module_name|
+				raise Exception, "Module #{module_name} is not exist!" if not ::File.directory?(module_name)
+				data[module_name] = TarGzip.targzip(module_name, module_name).read
+			end
+
+			url = "#{protocol}://#{address}:#{port}/modules"
+			uri = URI.parse(url)
+			http = Net::HTTP.new(uri.host, uri.port)
+			request = Net::HTTP::Put.new(uri.request_uri)
+			request.set_form_data(data)
+			response = http.request(request)
+
+			(response.code == '200')
+		end
+	end
+end
+
+if $0 == __FILE__
+	if ARGV.length < 2
+		puts "Usage: install_module <address> [port] <module-name> ..."
+		exit(1)
+	end
+
+	DefaultPort = 1314
+
+	address = ARGV.shift
+	port = (ARGV.length >= 2 ? ARGV.shift : DefaultPort)
+	modules = ARGV
+	missing = modules.select { |mod| not ::File.directory?(mod) }
+	modules = modules - missing
+	
+	success = Nuri::Util.install_modules(address, port, modules),
+	output = {
+		:status => success,
+		:installed_modules => modules,
+		:missing_modules => missing
+	}
+	puts JSON.generate(output)
+
+	exit (success ? 0 : 1)
+end

data/modules/package/package.sfp

@@ -1,5 +1,5 @@
 schema Package {
-	installed : Bool
+	installed = true
 
 	final package_name = ""
 	final version : String

data/modules/tarpackage/tarpackage.rb

@@ -1,3 +1,6 @@
+require 'uri'
+require 'net/http'
+
 class Sfp::Module::TarPackage
 	include Sfp::Resource
 
@@ -18,40 +21,42 @@ class Sfp::Module::TarPackage
 	##############################
 
 	def install(p={})
-		src = url
-		dest = home
-		return false if dest.length <= 0
-
-		# create destination directory if not exist
-		system "mkdir -p #{dest}" if !::File.exist?(dest)
+		return false if home.length <= 0
 
-		# install axel if it's not available
-		system "apt-get install axel" if `which axel`.strip.length <= 0
+		# create home directory if not exist
+		shell "mkdir -p #{home}" if !::File.exist?(home)
 
-		# download the file
-		system "cd #{dest} && axel -q #{src}"
+		file = url.split('/').last.to_s
+		dest = "#{home}/#{file}"
+		download(url, dest)
 
-		file = src.split('/').last.to_s
 		# if downloaded file is not exist, then return false
-		if !::File.exist?("#{dest}/#{file}")
+		if !::File.exist?(dest)
 			log.error "Failed to download file from #{url}"
 			return false
 		end
 
 		# extract tar file, and then delete it
-		system "cd #{dest} && tar xvzf #{file}"
-		system "rm -f #{dest}/#{file}"
+		shell "cd #{home} && tar xvzf #{file} && rm -f #{file}"
+
+		basename = case ::File.extname(file)
+		when '.gz'
+			::File.basename(file, '.tar.gz')
+		when '.tgz'
+			::File.basename(file, '.tgz')
+		else
+			file
+		end
 
-		basename = (::File.extname(file) == '.gz' ? ::File.basename(file, '.tar.gz') : ::File.basename(file, ::File.extname(file)))
-		system "bash -c 'cd #{dest}/#{basename} && shopt -s dotglob && mv -f * .. && cd .. && rm -rf #{basename}'"
+		shell "bash -c 'cd #{home}/#{basename} && shopt -s dotglob && mv -f * .. && cd .. && rm -rf #{basename}'"
 
-		File.open("#{dest}/#{Signature}", 'w') { |f| f.write(Time.now.to_s) }
+		File.open("#{home}/#{Signature}", 'w') { |f| f.write(Time.now.to_s) }
 
 		true
 	end
 
 	def uninstall(p={})
-		!!system("rm -rf #{home}") if File.exist?(home)
+		!!shell("rm -rf #{home}") if File.exist?(home)
 		true
 	end
 
@@ -62,6 +67,37 @@ class Sfp::Module::TarPackage
 	##############################
 
 	protected
+	def download(source, destination)
+		def use_http_proxy?(uri)
+			ENV['no_proxy'].to_s.split(',').each { |pattern|
+				pattern.chop! if pattern[-1] == '*'
+				return false if uri.hist[0, pattern.length] == pattern
+			}
+			true
+		end
+
+		file = nil
+		begin
+			uri = URI.parse(source)
+			http = nil
+			if use_http_proxy?(uri)
+				proxy = URI.parse(ENV['http_proxy'])
+				http = Net::HTTP::Proxy(proxy.host, proxy.port).new(uri.host, uri.port)
+			else
+				http = Net::HTTP.new(uri.host, uri.port)
+			end
+			http.request_get(uri.path) do |response|
+				file = ::File.open(destination, 'wb')
+				response.read_body do |segment|
+					file.write segment
+				end
+				file.flush
+			end
+		ensure
+			file.close if not file.nil?
+		end
+	end
+
 	def installed?
 		::File.exist?("#{home}/#{Signature}")
 	end