wcc 2.2.0 → 2.3.0

data/assets/conf.yml

@@ -6,6 +6,7 @@ conf:
 #  tag: wcc
 #  filterd: ./filter.d
 #  templated: ./template.d
+#  stats: yes
 #  # default config assuming local mail server
 #  email:
 #    smtp:

data/lib/wcc.rb

@@ -100,6 +100,7 @@ module WCC
 				:simulate => false,
 				:clean => false,
 				:nomails => false,
+				:stats => false,
 				# when you want to use ./tmp it must be writeable
 				:cache_dir => '/var/tmp/wcc',
 				:tag => 'wcc',
@@ -175,6 +176,8 @@ module WCC
 						@options[:filter_dir] ||= val
 					when 'templated'
 						@options[:template_dir] ||= val
+					when 'stats'
+						@options[:stats] ||= val
 					else
 						if not Notificators.mappings.include?(key)
 							plugin_name = "wcc-#{key}-notificator"
@@ -341,166 +344,23 @@ module WCC
 	end
 	
 	class Prog
-		def self.checkForUpdate(site)
-			WCC.logger.info "Requesting '#{site.uri.to_s}'"
-			begin
-				res = site.fetch
-			rescue Timeout::Error => ex
-				# don't claim on this
-				return false
-			rescue => ex
-				WCC.logger.error "Cannot connect to #{site.uri.to_s} : #{ex.to_s}"
-				return false
-			end
-			if res.kind_of?(Net::HTTPOK)
-				# be happy!
-			elsif res.kind_of?(Net::HTTPMovedPermanently)
-				loc = res['Location']
-				if loc.nil?
-					WCC.logger.error "Site #{site.uri.to_s} moved permanently, skippong it - no new location given."
-				else
-					WCC.logger.error "Site #{site.uri.to_s} moved permanently to '#{loc}', skipping it - please update your conf.yml adequately!"
-				end
-				return false
-			elsif res.kind_of?(Net::HTTPSeeOther) or res.kind_of?(Net::HTTPTemporaryRedirect)
-				loc = URI.parse(res['Location'])
-				WCC.logger.warn "Redirect: requesting '#{loc.to_s}'"
-				res = site.fetch_redirect(loc)
-				if not res.kind_of?(Net::HTTPOK)
-					WCC.logger.error "Redirected site #{loc.to_s} returned #{res.code} code, skipping it."
-					WCC.logger.error "Headers: #{res.to_hash.inspect}"
-					return false
-				end
-			elsif res.kind_of?(Net::HTTPUnauthorized)
-				WCC.logger.error "Site #{site.uri.to_s} demands authentication for '#{res['www-authenticate']}', skipping it - consider using 'auth:' option in your conf.yml."
-				return false
-			elsif res.kind_of?(Net::HTTPNotFound)
-				WCC.logger.error "Site #{site.uri.to_s} not found, skipping it."
-				return false
-			elsif res.kind_of?(Net::HTTPForbidden)
-				WCC.logger.error "Site #{site.uri.to_s} forbids access, skipping it."
-				return false
-			elsif res.kind_of?(Net::HTTPInternalServerError)
-				WCC.logger.error "Site #{site.uri.to_s} has internal errors, skipping it."
-				return false
-			elsif res.kind_of?(Net::HTTPServiceUnavailable)
-				#retry_after = res['Retry-After']
-				WCC.logger.warn "Site #{site.uri.to_s} currently not available, skipping it."
-				return false
-			else
-				WCC.logger.error "Site #{site.uri.to_s} returned #{res.code} code, skipping it."
-				WCC.logger.error "Headers: #{res.to_hash.inspect}"
-				return false
-			end
-			
-			new_content = res.body
-			
-			# detect encoding from http header, meta element, default utf-8
-			# do not use utf-8 regex because it will fail on non utf-8 pages
-			encoding = (res['content-type'].to_s.match(/;\s*charset=([A-Za-z0-9-]*)/i).to_a[1] || 
-						new_content.match(/<meta.*charset=([a-zA-Z0-9-]*).*/i).to_a[1]).to_s.downcase || 'utf-8'
-			
-			WCC.logger.info "Encoding is '#{encoding}'"
-			
-			# convert to utf-8
-			begin
-				new_content = Iconv.conv('utf-8', encoding, new_content)
-			rescue => ex
-				WCC.logger.error "Cannot convert site from '#{encoding}': #{ex.to_s}"
-				return false
-			end
-			
-			# strip html
-			new_content = new_content.strip_html if site.strip_html?
-			new_hash = Digest::MD5.hexdigest(new_content)
-			
-			WCC.logger.debug "Compare hashes\n  old: #{site.hash.to_s}\n  new: #{new_hash.to_s}"
-			return false if new_hash == site.hash
-			
-			# do not try diff or anything if site was never checked before
-			if site.new?
-				site.hash, site.content = new_hash, new_content
-				
-				# signal that no diff was posible
-				diff = nil
-			else
-				# save old site to tmp file
-				old_site_file = Tempfile.new("wcc-#{site.id}-")
-				old_site_file.write(site.content)
-				old_site_file.close
-				
-				# calculate labels before updating
-				old_label = "OLD (%s)" % File.mtime(Conf.file(site.id + ".md5")).strftime(DIFF_TIME_FMT)
-				new_label = "NEW (%s)" % Time.now.strftime(DIFF_TIME_FMT)
-			
-				site.hash, site.content = new_hash, new_content
-				
-				# diff between OLD and NEW
-				diff = %x[diff -U 1 --label "#{old_label}" --label "#{new_label}" #{old_site_file.path} #{Conf.file(site.id + '.site')}]
-			end
-			
-			# construct the data made available to filters and templates
-			data = OpenStruct.new
-			data.site = site
-			data.diff = diff.nil? ? nil : WCC::Differ.new(diff)
-			data.tag = Conf[:tag]
-			
-			# HACK: there *was* an update but no notification is required
-			return false if not Filters.accept(data, site.filters)
-			
-			site.notify.each do |name|
-				rec = Conf.recipients[name]
-				if rec.nil?
-					WCC.logger.error "Could not notify recipient #{name} - not found!"
-				else
-					rec.each { |way| way.notify!(data) }
-				end
-			end
-			
-			true
-		end
-
-		# main
 		def self.run!
-			# first use of Conf initializes it
-			WCC.logger = Logger.new(STDOUT)
-			
 			# make sure logger is correctly configured
+			WCC.logger = Logger.new(STDOUT)
+			# first use of Conf initializes it
 			Conf.instance
 			
-			# create cache dir for hash and diff files
-			Dir.mkdir(Conf[:cache_dir]) unless File.directory?(Conf[:cache_dir])
-			
-			if(Conf[:clean])
-				WCC.logger.warn "Removing hash and diff files..."
-				Dir.foreach(Conf[:cache_dir]) do |f|
-					File.delete(Conf.file(f)) if f =~ /^.*\.(md5|site)$/
-				end
-				cache_file = Conf.file('cache.yml')
-				WCC.logger.warn "Removing timestamp cache..."
-				File.delete(cache_file) if File.exists?(cache_file)
-				Prog.exit 1
-			end
-			
-			# read filter.d
-			Dir[File.join(Conf[:filter_dir], '*.rb')].each { |file| require file }
+			create_cache_dir
+			clean_cache_dir if Conf[:clean]
+			load_filters
+			load_timestamps
 			
-			# timestamps
-			cache_file = Conf.file('cache.yml')
-			if File.exists?(cache_file)
-				WCC.logger.debug "Load timestamps from '#{cache_file}'"
-
-				# may be *false* if file is empty
-				yaml = YAML.load_file(cache_file)
-
-				if not yaml
-					WCC.logger.info "No timestamps loaded"
-				else
-					@@timestamps = yaml['timestamps']
-				end
-			else
-				@@timestamps = {}
-			end
+			# stats
+			@@stats = {
+				'nruns' => 1,
+				'nsites' => 0, 'nnotifications' => 0, 'nerrors' => 0,
+				'nlines' => 0, 'nhunks' => 0
+			}
 			
 			Conf.sites.each do |site|
 				ts_old = get_timestamp(site)
@@ -510,24 +370,22 @@ module WCC
 					WCC.logger.info "Skipping check for #{site.uri.host.to_s} due to check #{ts_diff} minute#{ts_diff == 1 ? '' : 's'} ago."
 					next
 				end
-				if checkForUpdate(site)
+				case checkForUpdate(site)
+				when :update
 					WCC.logger.warn "#{site.uri.host.to_s} has an update!"
-				else
+				when :noupdate
 					WCC.logger.info "#{site.uri.host.to_s} is unchanged"
+				when :error
+					@@stats['nerrors'] += 1
 				end
 				update_timestamp(site, ts_new)
 			end
 			
-			# save timestamps
-			File.open(cache_file, 'w+') do |f| YAML.dump({"timestamps" => @@timestamps}, f) end
-			
-			# shut down notificators
-			Notificators.mappings.each do |name,klass|
-				WCC.logger.debug "Shut down #{klass}"
-				klass.shut_down
-			end
+			save_timestamps
+			update_stats if Conf[:stats]
+			shut_down_notificators
 		end
-		
+
 		# Attempts to read the named template file from template.d
 		# and converts it into ERB.
 		#
@@ -543,7 +401,7 @@ module WCC
 			end
 			nil
 		end
-		
+
 		# Attempts to write the given raw content to the named template file
 		# in template.d. This should be used to create initial template files on demand
 		# and will work only when file does not already exist.
@@ -574,5 +432,194 @@ module WCC
 		def self.update_timestamp(site, t)
 			@@timestamps[site.uri.to_s] = t
 		end
+		
+		def self.create_cache_dir
+			Dir.mkdir(Conf[:cache_dir]) unless File.directory?(Conf[:cache_dir])
+		end
+
+		def self.clean_cache_dir
+			WCC.logger.warn "Removing hash and diff files..."
+			Dir.foreach(Conf[:cache_dir]) do |f|
+				File.delete(Conf.file(f)) if f =~ /^.*\.(md5|site)$/
+			end
+			# special files
+			cache_file = Conf.file('cache.yml')
+			WCC.logger.warn "Removing timestamp cache..."
+			File.delete(cache_file) if File.exists?(cache_file)
+			stats_file = Conf.file('stats.yml')
+			WCC.logger.warn "Removing stats file..."
+			File.delete(stats_file) if File.exists?(stats_file)
+			Prog.exit 1
+		end
+
+		def self.load_filters
+			Dir[File.join(Conf[:filter_dir], '*.rb')].each do |file|
+				require file
+			end
+		end
+
+		def self.load_timestamps
+			cache_file = Conf.file('cache.yml')
+			@@timestamps = {}
+			if File.exists?(cache_file)
+				WCC.logger.debug "Load timestamps from '#{cache_file}'"
+				# may be *false* if file is empty
+				yaml = YAML.load_file(cache_file)
+				if not yaml
+					WCC.logger.warn "No timestamps loaded"
+				else
+					@@timestamps = yaml['timestamps']
+				end
+			end
+		end
+
+		def self.save_timestamps
+			cache_file = Conf.file('cache.yml')
+			File.open(cache_file, 'w+') do |f| YAML.dump({"timestamps" => @@timestamps}, f) end
+		end
+
+		def self.update_stats
+			stats_file = Conf.file('stats.yml')
+			if File.exists?(stats_file)
+				WCC.logger.debug "Load stats from '#{stats_file}'"
+				yaml = YAML.load_file(stats_file)
+				if not yaml
+					WCC.logger.warn "No stats loaded"
+				else
+					# merge stats infos
+					@@stats.each do |k,v| @@stats[k] += yaml['stats'][k] end
+				end
+			end
+			File.open(stats_file, 'w+') do |f| YAML.dump({"stats" => @@stats}, f) end
+		end
+
+		def self.shut_down_notificators
+			Notificators.mappings.each do |name,klass|
+				WCC.logger.debug "Shut down #{klass}"
+				klass.shut_down
+			end
+		end
+		
+		def self.checkForUpdate(site)
+			WCC.logger.info "Requesting '#{site.uri.to_s}'"
+			begin
+				res = site.fetch
+			rescue Timeout::Error => ex
+				return :noupdate # don't claim on this
+			rescue => ex
+				WCC.logger.error "Cannot connect to #{site.uri.to_s} : #{ex.to_s}"
+				return :error
+			end
+			return :error if handle_http_errors(res, site)
+			
+			new_content = get_utf8_body(res, site)
+			return :error if new_content.nil?
+			
+			# strip html
+			new_content = new_content.strip_html if site.strip_html?
+
+			new_hash = Digest::MD5.hexdigest(new_content)
+			WCC.logger.debug "Compare hashes\n  old: #{site.hash.to_s}\n  new: #{new_hash.to_s}"
+			return :noupdate if new_hash == site.hash
+			
+			# do not try diff or anything if site was never checked before
+			if site.new?
+				site.hash, site.content = new_hash, new_content
+				diff = nil # no diff possible
+			else
+				# save old site to tmp file
+				old_site_file = Tempfile.open("wcc-#{site.id}-")
+				old_site_file.write(site.content)
+				old_site_file.close
+				old_label = "OLD (%s)" % File.mtime(Conf.file(site.id + '.md5')).strftime(DIFF_TIME_FMT)
+				new_label = "NEW (%s)" % Time.now.strftime(DIFF_TIME_FMT)
+				# save new site
+				site.hash, site.content = new_hash, new_content
+				# diff between OLD and NEW
+				diff = %x[diff -U 1 --label "#{old_label}" --label "#{new_label}" #{old_site_file.path} #{Conf.file(site.id + '.site')}]
+			end
+
+			# construct the data made available to filters and templates
+			data = OpenStruct.new(:site => site, :tag => Conf[:tag],
+				:diff => diff.nil? ? nil : WCC::Differ.new(diff))
+			
+			@@stats['nsites'] += 1
+			if not data.diff.nil?
+				@@stats['nlines'] += data.diff.nlinesc
+				@@stats['nhunks'] += data.diff.nhunks
+			end
+			
+			# HACK: there *was* an update but no notification is required
+			return :noupdate if not Filters.accept(data, site.filters)
+			
+			site.notify.each do |name|
+				rec = Conf.recipients[name]
+				if rec.nil?
+					WCC.logger.error "Could not notify recipient #{name} - not found!"
+				else
+					@@stats['nnotifications'] += 1
+					rec.each { |way| way.notify!(data) }
+				end
+			end
+			
+			:update
+		end
+
+		def self.handle_http_errors(res, site)
+			return false if res.kind_of?(Net::HTTPOK)
+			if res.kind_of?(Net::HTTPMovedPermanently)
+				loc = res['Location']
+				if loc.nil?
+					WCC.logger.error "Site #{site.uri.to_s} moved permanently, skipping it - no new location given."
+				else
+					WCC.logger.error "Site #{site.uri.to_s} moved permanently to '#{loc}', skipping it - please update your conf.yml adequately!"
+				end
+				return true
+			elsif res.kind_of?(Net::HTTPSeeOther) or res.kind_of?(Net::HTTPTemporaryRedirect)
+				loc = URI.parse(res['Location'])
+				WCC.logger.warn "Redirect: requesting '#{loc.to_s}'"
+				res = site.fetch_redirect(loc)
+				if not res.kind_of?(Net::HTTPOK)
+					WCC.logger.error "Redirected site #{loc.to_s} returned #{res.code} code, skipping it."
+					WCC.logger.error "Headers: #{res.to_hash.inspect}"
+					return true
+				end
+			elsif res.kind_of?(Net::HTTPUnauthorized)
+				WCC.logger.error "Site #{site.uri.to_s} demands authentication for '#{res['www-authenticate']}', skipping it - consider using 'auth:' option in your conf.yml."
+				return true
+			elsif res.kind_of?(Net::HTTPNotFound)
+				WCC.logger.error "Site #{site.uri.to_s} not found (404), skipping it."
+				return true
+			elsif res.kind_of?(Net::HTTPForbidden)
+				WCC.logger.error "Site #{site.uri.to_s} forbids access (403), skipping it."
+				return true
+			elsif res.kind_of?(Net::HTTPInternalServerError)
+				WCC.logger.error "Site #{site.uri.to_s} has internal errors (500), skipping it."
+				return true
+			elsif res.kind_of?(Net::HTTPServiceUnavailable)
+				#retry_after = res['Retry-After']
+				WCC.logger.warn "Site #{site.uri.to_s} currently not available (503), skipping it."
+				return true
+			else
+				WCC.logger.error "Site #{site.uri.to_s} returned #{res.code} code, skipping it."
+				WCC.logger.error "Headers: #{res.to_hash.inspect}"
+				return true
+			end
+		end
+
+		def self.get_utf8_body(res, site)
+			# detect encoding from http header, meta element, default utf-8
+			# do not use utf-8 regex because it will fail on non utf-8 pages
+			encoding = (res['content-type'].to_s.match(/;\s*charset=([A-Za-z0-9-]*)/i).to_a[1] || 
+						res.body.match(/<meta.*charset=([a-zA-Z0-9-]*).*/i).to_a[1]).to_s.downcase || 'utf-8'
+			WCC.logger.info "Encoding is '#{encoding}'"
+			# convert to utf-8
+			begin
+				return Iconv.conv('utf-8', encoding, res.body)
+			rescue => ex
+				WCC.logger.error "Cannot convert site #{site.uri.to_s} from '#{encoding}': #{ex.to_s}"
+			end
+			nil
+		end
 	end
 end

data/lib/wcc/version.rb

@@ -1,4 +1,4 @@
 
 module WCC
-	VERSION = "2.2.0"
+	VERSION = "2.3.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: wcc
 version: !ruby/object:Gem::Version 
-  hash: 7
+  hash: 3
   prerelease: 
   segments: 
   - 2
-  - 2
+  - 3
   - 0
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors: 
 - Christian Nicolai
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-24 00:00:00 Z
+date: 2012-06-30 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: htmlentities