nuri 0.5.3 → 0.5.4

data/modules/file/file.rb

@@ -13,14 +13,14 @@ class Sfp::Module::File
 
 	def update_state
 		path = @model['path'].to_s.strip
-		if @model['exists']
-			create(path)
-		else
-			delete(path)
+
+		if not exist? or type? != @model['type']
+			create if del
 		end
 
 		@state['path'] = path
-		@state['exists'] = ::File.exist?(path)
+		@state['exists'] = exist?
+		@state['type'] = type?
 		@state['content'] = content?
 		@state['user'], @state['group'] = user_group?
 		@state['permission'] = permission?
@@ -34,28 +34,58 @@ class Sfp::Module::File
 
 	protected
 
-	def delete(file)
-		::File.delete(file) if ::File.exist?(file)
+	def del(p={})
+		return true if not exist?
+
+		file = @model['path']
+		if ::File.file?(file)
+			::File.delete(file)
+		elsif ::File.directory?(file) and file != '/'
+			shell "rm -rf #{file}"
+		end
+		not exist?
 	end
 
-	def create(file)
-		log.warn "Failed to create/update file #{file}!" if
+	def create(p={})
+		return true if exist?
+		file = @model['path']
+		if @model['type'] == 'file'
 			not set_content(file) or
 			not set_owner(file) or
 			not set_permission(file)
+		elsif @model['type'] == 'directory'
+			Dir.mkdir(file)
+		end
+		exist?
+	end
+
+	def exist?
+		::File.exist?(@model['path'])
+	end
+
+	def type?
+		if exist?
+			(::File.file?(@model['path']) ? 'file' : 'directory')
+		else
+			''
+		end
 	end
 
 	def set_content(file)
-		return true if not @model['content'].is_a?(String)
-		begin
-			current = (::File.exist?(file) ? content? : nil)
-			desired = Digest::SHA1.hexdigest(@model['content'])
-			File.open(file, 'w') { |f| f.write(@model['content']) } if current != desired
-			return true
-		rescue Exception => e
-			log.error "#{e}\n#{e.backtrace.join("\n")}"
+		if @model['type'] == 'directory'
+			shell "mkdir #{@model['path']}"
+		else
+			return true if not @model['content'].is_a?(String)
+			begin
+				current = (::File.exist?(file) ? content? : nil)
+				desired = Digest::SHA1.hexdigest(@model['content'])
+				File.open(file, 'w') { |f| f.write(@model['content']) } if current != desired
+				return true
+			rescue Exception => e
+				log.error "#{e}\n#{e.backtrace.join("\n")}"
+			end
+			false
 		end
-		false
 	end
 
 	def set_owner(file)
@@ -72,7 +102,7 @@ class Sfp::Module::File
 	end
 
 	def content?
-		(::File.exist?(@model['path']) ? Digest::SHA1.hexdigest(::File.read(@model['path'])) : '')
+		(::File.file?(@model['path']) ? Digest::SHA1.hexdigest(::File.read(@model['path'])) : '')
 	end
 
 	def user_group?

data/modules/hadoop1/hadoop1.rb

@@ -24,7 +24,7 @@ class Sfp::Module::Hadoop1Common
 		model = OpenStruct.new(@model)
 
 		# add group hadoop
-		if `grep '^#{model.group}' /etc/group`.length <= 0
+		if %x[grep '^#{model.group}' /etc/group].length <= 0
 			log.info "adding group #{model.group}"
 			shell "echo '#{model.group}:x:8000:' >> /etc/group"
 		else
@@ -32,7 +32,7 @@ class Sfp::Module::Hadoop1Common
 		end
 
 		# add user hadoop
-		if `grep '^#{model.user}' /etc/passwd`.length <= 0
+		if %x[grep '^#{model.user}' /etc/passwd].length <= 0
 			log.info "adding user #{model.user}"
 			shell "echo '#{model.user}:x:8000:8000::#{model.home}:/bin/bash' >> /etc/passwd &&
 			        echo '#{model.user}:#{model.password}:15958:0:99999:7:::' >> /etc/shadow"
@@ -46,11 +46,13 @@ class Sfp::Module::Hadoop1Common
 		
 		# download and extract hadoop binaries
 		log.info "download and install hadoop binaries"
-		source = (model.source[-7,7] == '.tar.gz' or model.source[-4,4] == '.tgz' ? model.source : "#{model.source}/hadoop-#{model.version}/hadoop-#{model.version}.tar.gz")
+		#source = (model.source[-7,7] == '.tar.gz' or model.source[-4,4] == '.tgz' ? model.source : "#{model.source}/hadoop-#{model.version}/hadoop-#{model.version}.tar.gz")
+		source = (model.source[-7,7] == '.tar.gz' or model.source[-4,4] == '.tgz' ? model.source : "#{model.source}/hadoop-#{model.version}.tar.gz")
 
 		file = source.split('/').last.to_s
 		basename = (::File.extname(file) == '.gz' ? ::File.basename(file, '.tar.gz') : ::File.basename(file, ::File.extname(file)))
 		destination = "#{model.home}/#{file}"
+log.info "download #{source} to #{destination}"
 		download source, destination
 		return false if not ::File.exist?(destination)
 		shell "cd #{model.home} &&
@@ -92,8 +94,10 @@ class Sfp::Module::Hadoop1Common
 		services.each { |name|
 			pid = pids[name]
 			if pid <= 0
-				cmd = "#{@model['home']}/bin/hadoop-daemon.sh start #{name}"
-				log.info `su -c '#{cmd} && sleep 3' #{@model['user']}`
+				cmd = "su -c '#{@model['home']}/bin/hadoop-daemon.sh start #{name} && sleep 3' #{@model['user']}"
+				pid = spawn(cmd)
+				Process.wait pid
+				#log.info %x[sudo su -c '#{cmd} && sleep 3' #{@model['user']}]
 			end
 		}
 
@@ -105,8 +109,10 @@ class Sfp::Module::Hadoop1Common
 		services.reverse.each { |name|
 			pid = pids[name]
 			if pid > 0
-				cmd = "#{@model['home']}/bin/hadoop-daemon.sh stop #{name}"
-				log.info `su -c '#{cmd}' #{@model['user']}`
+				cmd = "su -c '#{@model['home']}/bin/hadoop-daemon.sh stop #{name}' #{@model['user']}"
+				pid = spawn(cmd)
+				Process.wait pid
+				#log.info %x[sudo su -c '#{cmd}' #{@model['user']}]
 			end
 		}
 
@@ -141,7 +147,7 @@ class Sfp::Module::Hadoop1Common
 
 	def java_home
 		return @model['java_home'] if @model['java_home'].to_s.strip.length > 0
-		java = resolve_link(`which java`.strip)
+		java = resolve_link(%x[which java].strip)
 		return '' if java.length <= 0
 		::File.expand_path(java + '/../../')
 	end
@@ -158,7 +164,7 @@ class Sfp::Module::Hadoop1Common
 	def pids
 		data = {}
 		services.each { |name|
-			data[name] = `ps axf | grep java | grep -v grep | grep hadoop | grep Dproc_#{name}`.to_s.strip.split(' ', 2)[0].to_i
+			data[name] = %x[ps axf | grep java | grep -v grep | grep hadoop | grep Dproc_#{name}].to_s.strip.split(' ', 2)[0].to_i
 		}
 		data
 	end
@@ -185,7 +191,7 @@ class Sfp::Module::Hadoop1Master < Sfp::Module::Hadoop1Common
 	def uninstall(p={})
 		super
 
-		shell "rm -rf #{model.data_dir}"
+		shell "rm -rf #{@model['data_dir']}"
 
 		not installed?
 	end
@@ -206,7 +212,7 @@ class Sfp::Module::Hadoop1Master < Sfp::Module::Hadoop1Common
 	def map
 		{
 			'user' => @model['user'],
-			'master' => `hostname`.strip,
+			'master' => resolve("$." + Sfp::Agent.whoami? + ".sfpAddress"), #'master' => %x[hostname].strip,
 			'java_home' => java_home,
 			'tmp_dir' => @model['data_dir'],
 			'replication' => @model['replication'],
@@ -240,6 +246,7 @@ class Sfp::Module::Hadoop1Slave < Sfp::Module::Hadoop1Common
 			'user' => @model['user'],
 			'master' => resolve(@model['master'] + '.parent.sfpAddress'),
 			'java_home' => java_home,
+			'tmp_dir' => @model['data_dir'],
 			'replication' => resolve(@model['master'] + '.replication')
 		}
 	end

data/modules/hadoop2/hadoop2.rb

@@ -1,9 +1,7 @@
-require ::File.dirname(__FILE__) + '/../tarpackage/tarpackage.rb'
+require ::File.dirname(__FILE__) + '/../package2/package2.rb'
 
 module Sfp::Module::Hadoop2Common
 	def update_state
-		to_model
-
 		super
 
 		@state['running'] = running?
@@ -11,12 +9,12 @@ module Sfp::Module::Hadoop2Common
 		@state['pids'] = pids
 		@state['java_home'] = java_home
 
-		start if @state['running']
+		#start if @state['running']
 	end
 
 	##############################
 	#
-	# Action methods (see Hadoop.sfp)
+	# Action methods (see hadoop2.sfp)
 	#
 	##############################
 
@@ -157,6 +155,7 @@ module Sfp::Module::Hadoop2Common
 			'yarn_nodemanager_aux_services' => 'mapreduce_shuffle',
 			'yarn_log_aggregation_retain_seconds' => -1,
 			'yarn_log_aggregation_retain_check_interval_seconds' => -1,
+			'yarn_nodemanager_hostname' => local_address,
 			'yarn_nodemanager_local_dirs' => @model['data_dir'] + "/yarn_local_dir",
 			'yarn_nodemanager_log_dirs' => @model['data_dir'] + "/yarn_log_dir",
 			'yarn_web_proxy_address' => local_address,
@@ -165,8 +164,9 @@ module Sfp::Module::Hadoop2Common
 	end
 
 	def local_address
-		domain = `dnsdomainname`.to_s.strip
-		`hostname`.to_s.strip + (domain.length > 0 ? '.' + domain : '')
+		resolve("$.#{Sfp::Agent.whoami?}.sfpAddress")
+		#domain = `dnsdomainname`.to_s.strip
+		#`hostname`.to_s.strip + (domain.length > 0 ? '.' + domain : '')
 	end
 
 	# TODO -- user "useradd" and "groupadd"
@@ -216,14 +216,14 @@ module Sfp::Module::Hadoop2Common
 	end
 end
 
-class Sfp::Module::Hadoop2Master < Sfp::Module::TarPackage
+class Sfp::Module::Hadoop2Master < Sfp::Module::Package2
 	include ::Sfp::Module::Hadoop2Common
 
 	Services = ['namenode', 'resourcemanager', 'historyserver', 'proxyserver']
 
 	##############################
 	#
-	# Action methods (see TarPackage.sfp)
+	# Action methods (see hadoop2.sfp)
 	#
 	##############################
 
@@ -330,14 +330,14 @@ class Sfp::Module::Hadoop2Master < Sfp::Module::TarPackage
 	end
 end
 
-class Sfp::Module::Hadoop2Slave < Sfp::Module::TarPackage
+class Sfp::Module::Hadoop2Slave < Sfp::Module::Package2
 	include ::Sfp::Module::Hadoop2Common
 
 	Services = ['datanode', 'nodemanager']
 
 	##############################
 	#
-	# Action methods (see TarPackage.sfp)
+	# Action methods (see hadoop2.sfp)
 	#
 	##############################
 

data/modules/hadoop2/hadoop2.sfp

@@ -1,20 +1,21 @@
-include "../tarpackage/tarpackage.sfp"
+include "../package2/package2.sfp"
 
-schema Hadoop2 extends TarPackage {
+schema Hadoop2 extends Package2 {
 	installed = true
 	running = true
 	configured = true
 	
-	final package_name = "hadoop"
+	final name = "hadoop"
+	final provider = "tar"
 	final version = "2.2.0"
-	final source = "http://www.mirrorservice.org/sites/ftp.apache.org/hadoop/common"
-	final home = "/opt/hadoop"
+	final source = "http://master.herry.ext9.sup.hpl.hp.com/hadoop"
+	final home = "/opt/hadoop2"
 
 	final user = "hadoop"
 	final group = "hadoop"
 	final password = "!"
 	final java_home = ""
-	final data_dir = "/opt/hadoop/data"
+	final data_dir = "/opt/hadoop2/data"
 
 	sub install {
 		condition {

data/modules/hadoop2/yarn-site.xml

@@ -44,6 +44,11 @@
         <name>yarn.web-proxy.address</name>
         <value><%= yarn_web_proxy_address %>:54315</value>
 </property>
+
+<property>
+        <name>yarn.nodemanager.hostname</name>
+        <value><%= yarn_nodemanager_hostname %></value>
+</property>
 <property>
         <name>yarn.nodemanager.local-dirs</name>
         <value><%= yarn_nodemanager_local_dirs %></value>

data/modules/machine/machine.rb

@@ -4,7 +4,9 @@ class Sfp::Module::Machine
 	def update_state
 		to_model
 
-		load_kernel_modules(['acpiphp'])
+		if platform.include?('linux')
+			load_kernel_modules(['acpiphp'])
+		end
 
 		@state['sfpAddress'] = @model['sfpAddress']
 		@state['sfpPort'] = @model['sfpPort']
@@ -20,7 +22,11 @@ class Sfp::Module::Machine
 		@state["cpus"] = (File.exist?('/proc/cpuinfo') ? `cat /proc/cpuinfo | grep processor | wc -l`.strip.to_i : -1)
 		@state['memory'] = (`which free`.strip != '' ? `free`.split("\n")[1].split(" ")[1] : -1)
 
-		@state['disk'] = get_disk_state
+		if platform.include?('linux')
+			@state['disks'] = get_disks_state
+		else
+			@state['disks'] = {}
+		end
 	end
 
 	##############################
@@ -29,6 +35,10 @@ class Sfp::Module::Machine
 	#
 	##############################
 
+	def platform
+		RUBY_PLATFORM.downcase
+	end
+
 	protected
 
 	def load_kernel_modules(modules=[])
@@ -48,12 +58,12 @@ class Sfp::Module::Machine
 
 	# generate the disks' state, try to automatically mount the disk to target directory
 	#
-	def get_disk_state
+	def get_disks_state
 		def generate_state
-			disk = {}
+			disks = {}
 			# get disks UUID
 			uuids = {}
-			`blkid`.each_line do |line|
+			`/sbin/blkid`.each_line do |line|
 				line.strip!
 				next if line.length <= 0
 				device, info = line.split(':', 2)
@@ -69,32 +79,32 @@ class Sfp::Module::Machine
 				if data[0][0..4] == '/dev/'
 					name = 'root'
 					if data[5] != '/'
-						model = (@model['disk'].is_a?(Hash) ? @model['disk'].select { |k,v| v['mount'] == data[5] if k[0] != '_' } : {})
+						model = (@model['disks'].is_a?(Hash) ? @model['disks'].select { |k,v| v['mount'] == data[5] if k[0] != '_' } : {})
 						name = (model.length > 0 ? model.keys.first : "uuid_#{uuids[data[0]]}")
 					end
-					disk[name] = {
+					disks[name] = {
 						'size' => (data[1].to_f / 1000.0).to_i,
 						'mount' => data[5],
 						'uuid' => uuids[data[0]]
 					}
 				end
 			end
-			disk
+			disks
 		end
 
-		disk = generate_state
+		disks = generate_state
 
-		if @model['disk'].is_a?(Hash)
-			names = @model['disk'].keys.sort { |x,y| x <=> y }
+		if @model['disks'].is_a?(Hash)
+			names = @model['disks'].keys.sort { |x,y| x <=> y }
 			device = "/dev/vdb"
 			# format unformatted disks, mount unmount disks
 			names.each { |name|
-				next if name[0] == '_' or disk.has_key?(name)
-				spec = @model['disk'][name]
+				next if name[0] == '_' or disks.has_key?(name)
+				spec = @model['disks'][name]
 				status = `file -s #{device}`
 				if not (status =~ /ERROR/)
 					target = spec['mount'].to_s.strip
-					# format the disk if not yet formatted
+					# format the disks if not yet formatted
 					system "mkfs.ext4 #{device}" if not (status =~ /.+ filesystem data/)
 					# create target directory if not exist
 					system "mkdir -p #{target}" if !File.exist? target

data/modules/openstack/README.md

@@ -0,0 +1,24 @@
+OpenStack module
+----------------
+
+Required Ruby Gems:
+- fog
+- json
+
+To use:
+- Assume that your key-name for SSH is `default`.
+- Copy your VM's SSH private key to filename `default.pem` into this directory.
+- Edit file `config.yml` by filling in fields:
+	- `username` - the username of your account
+	- `password` - the password of your account
+
+  Those information can be found in your OpenStack account under menu `Access`.
+- Set `key_name` value.
+- Replace any necessary default values of HPCloud's attributes such as:
+	- `auth_uri` - the URL of OpenStack's end point (append `/tokens`)
+	- `vm_image` - the UUID of the VM's image
+	- `vm_flavour` - the ID of the flavour
+	- `vm_ssh_user` - the user that will be used by the module for SSH (`root`, `ubuntu`, or other)
+	- `vm_ssh_key_name` - the SSH key name
+	- `vm_security_group` - the security group that will be applied to your VM
+	- `vm_network` - the network that will be connected to your VM

data/modules/openstack/config.yml

@@ -0,0 +1,5 @@
+access_key: 
+secret_key: 
+tenant_id: 
+username: 
+password: 

data/modules/openstack/example.sfp

@@ -0,0 +1,9 @@
+include "modules/vm/vm.sfp"
+include "modules/openstack/openstack.sfp"
+
+proxy isa Node {
+	sfpAddress is "localhost"
+	openstack isa OpenStack {
+		auth_uri is "http://16.25.166.21:5000/v2.0/tokens"
+	}
+}