CloudyScripts 1.6.1 → 1.7.27

data/lib/audit/checks/VAR_LIST_HOME_DIRECTORIES.check

@@ -0,0 +1,5 @@
+ID: VAR_LIST_HOME_DIRECTORIES
+Depends: [USERS_INIT_1, HAS_CAT, HAS_TR, HAS_CUT, HAS_SED]
+Description: Export colon-separated home directory list from /etc/passwd.
+Type: [export]
+Script: "export HOME_DIRS_LIST=$(${CAT} /etc/passwd | ${CUT} -d: -f6 | ${TR} '\n' ':' | ${SED} -e 's/:$//')"

data/lib/audit/checks/benchmark.group

@@ -0,0 +1,6 @@
+ID: BENCHMARK
+Children: 
+   - APACHE2
+   - SSH
+   - LYNIS_AUTH
+   - VARIOUS

data/lib/audit/checks/footer.template

@@ -0,0 +1,12 @@
+   EXIT_CODE=$?
+   %%SCRIPT_ID%%_EXITCODE=${EXIT_CODE}
+   if [ ${EXIT_CODE} -eq 0 ]
+   then 
+      script_return "pass"
+   else 
+		script_return "fail"
+   fi
+else
+   %%SCRIPT_ID%%_EXITCODE=2
+   script_return "notchecked"
+fi

data/lib/audit/checks/header.template

@@ -0,0 +1,10 @@
+
+
+#################################
+### %%SCRIPT_ID%%
+#################################
+
+MY_SCRIPT_ID="%%SCRIPT_ID%%"
+IFS=$( printf ' \t\n+' ); IFS=${IFS%+}
+if [ 0 -eq 0 %%DEPENDS_CONDITION%% ]
+then

data/lib/audit/checks/helpers/head.sh

@@ -0,0 +1,59 @@
+#!/bin/sh
+
+if [ "${1##--}" = "version" ]
+then
+	echo "internal helper"
+	exit 0
+fi
+
+
+NB_LINES=${1##-}
+
+#test if number of lines parameter is really numerical
+NB_LINES_FILTERED=${NB_LINES##[0-9]}
+NB_LINES_FILTERED=${NB_LINES_FILTERED##[0-9]}
+NB_LINES_FILTERED=${NB_LINES_FILTERED##[0-9]}
+if [ -z "${NB_LINES_FILTERED}" ]
+then 
+	shift
+else
+	NB_LINES=10
+fi
+
+#if 0 lines of head, simply return
+if [ "${NB_LINES}" -le 0 ]
+then
+	return 0
+fi
+
+# test if second parameter is given
+if [ -z "$1" ]
+then
+	INPUT_FILE=/dev/stdin
+else
+	TEMPFILE=/tmp/$$
+	mkfifo ${TEMPFILE}
+	cat "$1" > ${TEMPFILE} &
+	INPUT_FILE=${TEMPFILE}
+fi
+
+#read each line and count down the line counter
+while read LINE < ${INPUT_FILE}
+do
+	echo ${LINE}
+	NB_LINES=$((${NB_LINES}-1))
+	#if line counter reaches zero, it's finished
+	if [ "${NB_LINES}" -le 0 ]
+	then
+		if [ ! -z "${TEMPFILE}" ]
+		then
+			rm ${TEMPFILE}
+		fi
+		return 0
+	fi
+done
+
+if [ ! -z "${TEMPFILE}" ]
+then
+	rm ${TEMPFILE}
+fi

data/lib/audit/checks/script_header.template

@@ -0,0 +1,69 @@
+
+###################################################
+# Auditor scanning script #########################
+###################################################
+# (C) 2010 SecludIT, Jonas Zaddach ################
+###################################################
+
+script_raw_message() {
+	msg="%% ${MY_SCRIPT_ID}"
+	for val in "$@"
+	do
+		if [ ! val = "" ]
+		then 
+			msg="${msg} %% ${val}"
+		fi
+	done
+	echo "${msg}"
+}
+
+script_message() {
+	script_raw_message "$1" "MESSAGE" "$2"
+}
+
+script_info_message() {
+	script_message "INFO" "$1"
+}
+
+script_warn_message() {
+	script_message "WARN" "$1"
+}
+
+script_error_message() {
+	script_message "ERROR" "$1"
+}
+
+script_return() {
+	script_raw_message "INFO" "CHECK_FINISHED" "$1"
+}
+
+script_program_name() {
+	script_raw_message "INFO" "PROGRAM_NAME" "$1" "$2"
+}
+
+script_not_found() {
+	script_raw_message "ERROR" "NOT_FOUND" "$1"
+}
+
+script_data() {
+	script_raw_message "INFO" "DATA" "$1" "$2"
+}
+
+script_debug() {
+	echo "$1" 1>&2
+}
+
+script_attach_file() {
+	script_raw_message "INFO" "ATTACH_FILE" "$1" "$2"
+}
+
+
+script_set_exit_code() {
+	/bin/sh -c "exit $1"
+}
+
+# create directory that can be used to store audit files
+AUDIT_DIRECTORY="/tmp/audit"
+rm -Rf ${AUDIT_DIRECTORY}
+mkdir -p ${AUDIT_DIRECTORY}
+

data/lib/audit/create_benchmark.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+
+BENCHMARK_FILE=$1
+ZIP_FILE=$2
+MISC_FILES="script_header.template header.template footer.template"
+REPOSIT="checks"
+
+if [ "${BENCHMARK_FILE}" = "" ] || [ "${ZIP_FILE}" = "" ]
+then
+	echo "USAGE: "$0" <benchmark group file> <target zip file>"
+	exit 1
+fi
+
+#check that identifiers are unique
+DUPLICATE_IDENTIFIERS=$( cd $REPOSIT && ls -1 *.check *.group | sed -e 's/\.[^.]*$//' | sort | uniq -d )
+
+if [ ! "${DUPLICATE_IDENTIFIERS}" = "" ]
+then
+	for f in $( echo ${DUPLICATE_IDENTIFIERS} )
+	do
+		echo "WARNING: There exist several files with the same identifier: $f"
+	done
+fi
+
+#check for identifiers that do not correspond to file names
+WRONG_IDENTIFIERS=""
+
+cd $REPOSIT
+for f in *.check *.group
+do
+	FILENAME_ID=$( echo $f | sed -e 's/.[^.]*$//' )
+	INTERNAL_ID=$( grep "ID:" $f | sed -e 's/^ID:\s*//' )
+
+	if [ ! "${FILENAME_ID}" = "${INTERNAL_ID}" ]
+	then 
+		echo "WARNING: ID in file $f is different from file name"
+	fi
+done
+
+#if zip file exists already, delete it
+rm -f ../${ZIP_FILE}
+
+#build list of tests required by benchmark
+CHECKS_FIFO="${BENCHMARK_FILE%.group}"
+DONE_CHECKS=""
+
+while [ ! "${CHECKS_FIFO}" = "" ]
+do
+	CURRENT_CHECK=$( echo ${CHECKS_FIFO} | cut -d" " -f1 )
+	CHECKS_FIFO=$( echo ${CHECKS_FIFO} | sed -e "s/${CURRENT_CHECK}//" )
+	DONE_CHECKS="${DONE_CHECKS} ${CURRENT_CHECK}"
+
+	echo "adding check: ${CURRENT_CHECK}"
+
+	if [ -f "${CURRENT_CHECK}.group" ]
+	then
+		CHILDREN=$( ruby -r "yaml" -e "File.open('${CURRENT_CHECK}.group') {|f| x = YAML::load(f)['Children']; print x ? x.join(' ') : '' }" )
+		#add child if it is not already in benchmark
+		for f in $( echo ${CHILDREN} )
+		do
+			if ( ! echo ${CHECKS_FIFO} | grep $f 2>/dev/null 1>/dev/null ) && ( ! echo ${DONE_CHECKS} | grep $f 2>/dev/null 1>/dev/null )
+			then
+				CHECKS_FIFO="${CHECKS_FIFO} $f"
+			fi
+		done
+
+		zip -9 ../${ZIP_FILE} "${CURRENT_CHECK}.group" 2>/dev/null 1>/dev/null
+	elif [ -f "${CURRENT_CHECK}.check" ]
+	then
+		# get list of dependencies
+		DEPENDENCIES=$( ruby -r "yaml" -e "File.open('${CURRENT_CHECK}.check') {|f| x = YAML::load(f)['Depends']; print x ? x.join(' ') : '' }" )
+
+		for f in $( echo ${DEPENDENCIES} )
+		do
+			if ( ! echo ${CHECKS_FIFO} | grep $f 2>/dev/null 1>/dev/null ) && ( ! echo ${DONE_CHECKS} | grep $f 2>/dev/null 1>/dev/null )
+			then
+				CHECKS_FIFO="${CHECKS_FIFO} $f"
+			fi
+		done
+
+		zip -9 ../${ZIP_FILE} "${CURRENT_CHECK}.check" 2>/dev/null 1>/dev/null
+	else
+		echo "WARNING: Unsatisfied dependency: ${CURRENT_CHECK}"
+	fi
+done
+
+for f in $( echo ${MISC_FILES} )
+do
+	echo "adding file: $f"
+	zip -9 ../${ZIP_FILE} $f 2>/dev/null 1>/dev/null
+done
+
+cd -

data/lib/audit/lib/audit.rb

@@ -0,0 +1,136 @@
+require 'logger'
+	
+require 'connection/connection_factory'
+require 'benchmark/benchmark_factory'
+require 'linear_script_generator'
+require 'parser/script_output_parser'
+require 'util/random_string'
+require 'benchmark/benchmark_result'
+require 'lazy'
+
+class Audit
+	attr_reader :benchmark
+	attr_reader :connection
+	attr_reader :start_time
+	attr_reader :end_time
+	attr_reader :results
+	attr_reader :exceptions
+
+	# Create a new audit.
+	# The audit will be initialized, but not started.
+	# * <em>benchmark</em> is a path string that points to the benchmark file
+	#   that should be used for the audit.
+	# * <em>attachment_dir</em> is a path string that points to the directory where incoming
+	#   files will be saved (that directory needs to be writable). If a null value
+	#   is passed, ATTACH_FILE requests will be ignored and no files will be saved.
+	# * <em>connection_type</em> is a symbol for the connection type that will be used.
+	#   Anything that can be given to the ConnectionFactory (connection/ConnectionFactory::create)
+	#   is valid (:ssh, ...)
+	# * <em>connection_params</em> is a dictionary of connection parameters that are
+	#   specific to the connection type chosen with <em>connection_type</em>. See the
+	#   ConnectionFactory class for more datails.
+	# * <em>logger</em> is an optional logger that the debug output is logged to
+	def initialize(options)
+		raise "Option :benchmark is required" unless options[:benchmark]
+#		raise "Option :attachment_dir is required" unless options[:attachment_dir]
+		raise "Option :connection_type is required" unless options[:connection_type]
+		raise "Option :connection_params is required" unless options[:connection_params]
+
+		if options[:logger] then
+			@logger = options[:logger]
+		else
+			@logger = Logger.new(STDOUT)
+		end
+		
+		@benchmark = BenchmarkFactory.new(:logger => @logger).load(:benchmark => options[:benchmark])
+		@connection = ConnectionFactory.new(:logger => @logger).create(:connection_type => options[:connection_type], 
+		                                                               :connection_params => options[:connection_params])
+		@results = {}
+		@exceptions = []
+		@attachment_dir = options[:attachment_dir]
+	end
+	
+	def start(parallel = true)
+		@start_time = Time.now.utc
+	
+		launch_audit = Proc.new do
+			remote_script_path = "/tmp/" + RandomString::generate() + ".sh"
+			script = LinearScriptGenerator.generate(@benchmark)
+
+			@connection.open() do|conn|
+				conn.write_to_remote_file(script, remote_script_path)
+				@response_parser = ScriptOutputParser.new(:benchmark => @benchmark, 
+				                                          :connection => conn, 
+				                                          :attachment_dir => @attachment_dir, 
+				                                          :logger => @logger)
+				@response_parser.on_check_completed() do|rule_result|
+					@results[rule_result.rule_idref] = rule_result
+					@check_completed_handler.call(rule_result) unless @check_completed_handler.nil?
+				end
+				@response_parser.on_finished() do|benchmark, rule_results|
+					@end_time = Time.now.utc
+					@finished_handler.call(benchmark, rule_results) unless @finished_handler.nil?
+				end
+					
+				conn.exec("/bin/sh " + remote_script_path)
+			end
+		end
+	
+		if (parallel) then
+			begin
+				Thread.new {launch_audit.call}
+			rescue Exception => ex
+				exceptions << ex
+				@logger.error {"Exception type: #{ex.class.name}"}
+				@logger.error {"=== stack trace of exception #{ex.message}"}
+				ex.backtrace.each do|line|
+					@logger.error {line}
+				end
+				@logger.error {"=== end stack trace"}
+			end
+		else
+			launch_audit.call
+		end
+		return self
+	end
+
+	def on_check_completed(&block)
+		@check_completed_handler = block
+	end
+
+	def on_finished(&block)
+		@finished_handler = block
+	end
+
+	def progress()
+		return ((@response_parser.progress() unless @response_parser.nil?) or 0.0)
+	end
+	
+	def abort()
+		@connection.abort() if @connection
+	end
+	
+	def finished?()
+		return !end_time.nil?
+	end
+
+	def name()
+		return (@benchmark.name || @benchmark.id)  + "#" + @connection.to_s() + (@start_time ? "#" + @start_time.to_s() : "")
+	end
+	
+	def remaining_time()
+		return @benchmark.duration() if @response_parser.nil?
+		return @response_parser.remaining_time()
+	end
+
+	def to_hash()
+		return {
+			:type => :AUDIT,
+			:start_time => @start_time,
+			:end_time => @end_time,
+			:connection => @connection.to_hash(),
+			:benchmark => @benchmark.to_hash(),
+			:results => Lazy.new(@results.values(), :map) {|x| Lazy.new(x, :to_hash)}
+		}
+	end
+end

data/lib/audit/lib/audit_facade.rb

@@ -0,0 +1,5 @@
+class AuditFacade
+  def start_audit(parameters, &block)
+    return Audit.new(parameters, block).start
+  end
+end

data/lib/audit/lib/benchmark/audit_benchmark.rb

@@ -0,0 +1,165 @@
+require 'benchmark/check'
+require 'benchmark/item_exception'
+require 'lazy'
+
+class AuditBenchmark
+	attr_reader :item_repository
+	
+	def execution_order()
+		# resolve dependencies between checks based on the depends-tag of the checks.
+		# In a first pass, all dependencies are discovered iteratively popping checks
+		# from a queue of checks with unresolved dependencies, pushing them onto a resolved
+		# queue and pushing the check's dependencies onto the unresolved queue if they
+		# are not yet in the resolved or unresolved queue. Also, the reversed dependencies
+		# (which check is needed by which) are stored for the second pass.
+		#
+		# In a second pass, starting from checks which do not depend on any other checks,
+		# all checks are labelled with the dependency level they're in. Checks without dependencies
+		# have dependency level 0, checks which rely on checks from level 0 have level 1,
+		# and so on. This is not the optimal solution for the problem ... but I have forgot why,
+		# so figure this out yourself.
+		#
+		# You might wonder why I don't do dependency tracking with the Imports and Exports
+		# declarations: If there are two scripts which provide a variable (alternatives),
+		# it is very easy to write a mediating script, which chooses one of the provider scripts,
+		# and then can be depended on, but it is very hard to do the resolution based on 
+		# imports and exports. So imagine this like the linker, which uses library/object
+		# names to include dependencies, but still checks that all symbols are resolved
+		# correctly (TODO: Add a check that all Imports are satisfied by Exports)
+		
+		#find all dependencies
+		items = @children.dup
+		
+		unresolved = []
+		while not items.empty?
+			item = items.shift
+			if item.class == Group
+				item.children.each do|x| 
+					items << x unless items.include? x or unresolved.include? x
+				end
+			elsif item.class == Check
+				unresolved << item
+			end
+		end
+
+		resolved = []
+		dependency_root = []
+		reversed_dependencies = {}
+		iterations = 0
+		
+		while !unresolved.empty? and iterations < @item_repository.length
+			cur = unresolved.shift
+			
+			dependency_root.push(cur) if cur.dependencies.empty? and not dependency_root.include? cur
+			
+			cur.dependencies.each do|dep|
+				unresolved.push dep unless unresolved.include? dep or resolved.include? dep
+				reversed_dependencies[dep] = [] if reversed_dependencies[dep].nil?
+				reversed_dependencies[dep] << cur
+			end
+		end
+		
+		untagged = []
+		tagged = []
+		
+		dependency_root.each {|x| untagged.push(x)}
+		untagged.push(:NEXT_LEVEL)
+		
+		level = 0
+		while untagged.length > 1
+			cur = untagged.shift
+			
+			if cur == :NEXT_LEVEL then
+				level = level + 1
+				untagged.push(:NEXT_LEVEL)
+				next
+			end
+			
+			tag = tagged.select {|x| x[:check] == cur}
+			if tag.empty? then
+				tagged << {:level => level, :check => cur}  
+			else 
+				raise "multiple tags for check #{cur.id} found" if tag.length != 1
+				
+				tag[0][:level] = level
+			end
+			
+			unless reversed_dependencies[cur].nil? then
+				reversed_dependencies[cur].each {|x| untagged.push(x)}
+			end
+		end
+		
+		retval = []
+		(0 .. level).each {|i| retval.push(tagged.select {|x| x[:level] == i}.map {|x| x[:check]})}
+		
+		return retval
+	end
+	
+	def element(name)
+		@elements = {} unless @elements
+		@elements[name] = element_impl(name) unless @elements[name]
+		
+		return @elements[name]
+	end
+
+  def rules()
+    untraversed = @children.dup
+    rules = []
+
+    while untraversed.length > 0
+      item = untraversed.shift
+
+      if item.kind_of? Group then
+        untraversed = untraversed + item.children
+      elsif item.kind_of? Check then
+        rules << item
+      end
+    end
+
+    return rules.uniq
+  end
+
+  def dependencies()
+    return (rules().map {|x| x.dependencies }.flatten()).uniq
+  end
+
+  def automatic_dependencies()
+    return dependencies() - rules()
+  end
+
+#  def checks()
+#    checks = []
+#    not_traversed = @children.dup
+#
+#    while !not_traversed.empty? do
+#      item = not_traversed.shift
+#
+#      if (item.class == Group) then
+#        item.children.each do |child|
+#          not_traversed << child unless not_traversed.include? child
+#        end
+#      elsif item.class == Check then
+#        checks << item
+#        item.dependencies.each do |dep|
+#          not_traversed << dep unless not_traversed.include? dep
+#        end
+#      end
+#    end
+#
+#    return checks
+#  end
+
+  def duration()
+    execution_order().flatten().each().inject(0) {|result, element| result + element.duration}
+  end
+
+  def to_hash()
+    return {
+      :type => :BENCHMARK,
+      :id => @id,
+      :name => @name,
+      :description => @description,
+      :children => Lazy.new(Lazy.new(@children, :reject) {|x| !x.in_report?}, :map) {|child| Lazy.new(child, :to_hash)}
+    }
+  end
+end