feedupdater 0.1.0

data/lib/feed_updater.rb

@@ -0,0 +1,401 @@
+#--
+# Copyright (c) 2005 Robert Aman
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+require 'feed_updater/version'
+
+$:.unshift(File.dirname(__FILE__))
+$:.unshift(File.dirname(__FILE__) + "/feed_updater/vendor")
+
+# Make it so Rubygems can find the executable
+$:.unshift(File.expand_path(File.dirname(__FILE__) + "/../bin"))
+
+require 'rubygems'
+
+if !defined?(Daemons)
+  begin
+    require_gem('daemons', '>= 0.4.4')
+  rescue LoadError
+    require 'daemons'
+  end
+end
+
+if !defined?(FeedTools::FEED_TOOLS_VERSION)
+  begin
+    if File.exists?(File.expand_path(
+        File.dirname(__FILE__) + "/../../feedtools/lib"))
+      $:.unshift(File.expand_path(
+        File.dirname(__FILE__) + "/../../feedtools/lib"))
+      require("feed_tools")
+    else
+      require_gem('feedtools', '>= 0.2.23')
+    end
+  rescue LoadError
+    require 'feed_tools'
+  end
+end
+
+require 'benchmark'
+
+module FeedTools
+  # A simple daemon for scheduled updating of feeds.
+  class FeedUpdater
+    # Declares an on_update event.  The given block will be called after
+    # every feed update.  The block is passed the feed object that was loaded
+    # and the time it took in seconds to successfully load it.
+    def self.on_update(&block)
+      raise "No block supplied for on_update." if block.nil?
+      @@on_update = block
+    end
+
+    # Declares an on_error event.  The given block will be called after
+    # an error occurs during a feed update.  The block is passed the href of
+    # the feed that errored out, and the exception object that was raised.
+    def self.on_error(&block)
+      raise "No block supplied for on_error." if block.nil?
+      @@on_error = block
+    end
+
+    # Declares an on_complete event.  The given block will be called after
+    # all feeds have been updated.  The block is passed a list of feeds that
+    # FeedUpdater attempted to update.
+    def self.on_complete(&block)
+      raise "No block supplied for on_complete." if block.nil?
+      @@on_complete = block
+    end
+
+    # Returns the initial directory that the daemon was started from.
+    def initial_directory()
+      @initial_directory = nil if !defined?(@initial_directory)
+      return @initial_directory
+    end
+    
+    # Returns the directory where the pid files are stored.
+    def pid_file_dir()
+      @pid_file_dir = nil if !defined?(@pid_file_dir)
+      return @pid_file_dir
+    end
+    
+    # Sets the directory where the pid files are stored.
+    def pid_file_dir=(new_pid_file_dir)
+      @pid_file_dir = new_pid_file_dir
+    end
+    
+    # Returns the directory where the log files are stored.
+    def log_file_dir()
+      @log_file_dir = nil if !defined?(@log_file_dir)
+      return @log_file_dir
+    end
+    
+    # Sets the directory where the log files are stored.
+    def log_file_dir=(new_log_file_dir)
+      @log_file_dir = new_log_file_dir
+    end
+    
+    # Returns the path to the log file.
+    def log_file()
+      if !defined?(@log_file) || @log_file.nil?
+        if self.log_file_dir.nil?
+          @log_file = File.expand_path("./feed_updater.log")
+        else
+          @log_file = File.expand_path(
+            self.log_file_dir + "/feed_updater.log")
+        end
+      end
+      return @log_file
+    end
+    
+    # Returns the logger object.
+    def logger()
+      if !defined?(@logger) || @logger.nil?
+        @logger = Logger.new(self.log_file)
+      end
+      return @logger
+    end
+        
+    # Restarts the logger object.  This needs to be done after the program
+    # forks.
+    def restart_logger()
+      begin
+        self.logger.close()
+      rescue IOError
+      end
+      @logger = Logger.new(self.log_file)
+    end
+    
+    # Returns a list of feeds to be updated.
+    def feed_href_list()
+      if !defined?(@feed_href_list) || @feed_href_list.nil?
+        @feed_href_list_override = false
+        if self.updater_options.nil?
+          @feed_href_list = nil
+        elsif self.updater_options[:feed_href_list].kind_of?(Array)
+          @feed_href_list = self.updater_options[:feed_href_list]
+        else
+          @feed_href_list = nil
+        end
+      end
+      return @feed_href_list
+    end
+    
+    # Sets a list of feeds to be updated.
+    def feed_href_list=(new_feed_href_list)
+      @feed_href_list_override = true
+      @feed_href_list = new_feed_href_list
+    end
+    
+    # Returns either :running or :stopped depending on the daemon's current
+    # status.
+    def status()
+      @status = :stopped if @status.nil?
+      return @status
+    end
+    
+    # Returns a hash of the currently set updater options.
+    def updater_options()
+      if !defined?(@updater_options) || @updater_options.nil?
+        @updater_options = {
+          :start_delay => true
+        }
+      end
+      return @updater_options
+    end
+    
+    # Returns a hash of the currently set daemon options.
+    def daemon_options()
+      if !defined?(@daemon_options) || @daemon_options.nil?
+        @daemon_options = {
+          :app_name => "feed_updater_daemon",
+          :dir_mode => :normal,
+          :dir => self.pid_file_dir,
+          :backtrace => true,
+          :ontop => false
+        }
+      end
+      @daemon_options[:dir] = self.pid_file_dir
+      return @daemon_options
+    end
+    
+    # Returns a reference to the daemon application.
+    def application()
+      @application = nil if !defined?(@application)
+      return @application
+    end
+    
+    # Returns the process id of the daemon.  This should return nil if the
+    # daemon is not running.
+    def pid()
+      if File.exists?(File.expand_path(pid_file_dir + "/feed_updater.pid"))
+        begin
+          pid_file = File.open(
+            File.expand_path(pid_file_dir + "/feed_updater.pid"), "r")
+          return pid_file.read.to_s.strip.to_i
+        rescue Exception
+          return nil
+        end
+      else
+        return nil if self.application.nil?
+        begin
+          return self.application.pid.pid
+        rescue Exception
+          return nil
+        end
+      end
+    end
+    
+    def cloaker(&blk) #:nodoc:
+      (class << self; self; end).class_eval do
+        define_method(:cloaker_, &blk)
+        meth = instance_method(:cloaker_)
+        remove_method(:cloaker_)
+        meth
+      end
+    end
+    protected :cloaker
+    
+    # Starts the daemon.
+    def start()
+      if !defined?(@@on_update) || @@on_update.nil?
+        raise "No on_update handler block given."
+      end
+      if self.status == :running || self.pid != nil
+        puts "FeedUpdater is already running."
+        return self.pid
+      end
+
+      @initial_directory = File.expand_path(".")
+
+      if @application.nil?
+        self.logger()
+        feed_update_proc = lambda do
+          # Reestablish correct location
+          Dir.chdir(File.expand_path(self.initial_directory))
+
+          self.restart_logger()
+          
+          if FeedTools.configurations[:feed_cache].nil?
+            FeedTools.configurations[:feed_cache] =
+              "FeedTools::DatabaseFeedCache"
+          end
+          if !FeedTools.feed_cache.connected?
+            FeedTools.feed_cache.initialize_cache()
+          end
+          if !FeedTools.feed_cache.connected?
+            self.logger.fatal(
+              "Not connected to the feed cache. Please create database.yml.")
+            exit
+          end
+          
+          # A random start delay is introduced so that we don't have multiple
+          # feed updater daemons getting kicked off at the same time by
+          # multiple users.
+          if self.updater_options[:start_delay]
+            delay = (rand(45) + 15)
+            self.logger.info("Startup delay set for #{delay} minutes.")
+            sleep(delay.minutes)
+          end
+          
+          # The main feed update loop.
+          loop do
+            result = Benchmark.measure do
+              self.update_feeds()
+            end
+            self.logger.info(
+              "#{@feed_href_list.size} feeds updated " +
+              "in #{result.real.round} seconds.")
+            ObjectSpace.garbage_collect
+            sleepy_time = 1.hour - result.real.round
+            if sleepy_time > 0
+              self.logger.info(
+                "Sleeping for #{(sleepy_time / 60.0).round} minutes...")
+              sleep(sleepy_time)
+            else
+              self.logger.info(
+                "Update took more than 60 minutes, restarting immediately.")
+            end
+          end
+        end
+        options = self.daemon_options.merge({
+          :proc => feed_update_proc,
+          :mode => :proc
+        })
+        @application = Daemons::ApplicationGroup.new(
+          'feed_updater', options).new_application(options)
+        @application.start()
+      else
+        @application.start()
+      end
+      self.logger.info("-" * 79)
+      self.logger.info("Daemon started, " +
+        "FeedUpdater #{FeedTools::FEED_UPDATER_VERSION::STRING} / " +
+        "FeedTools #{FeedTools::FEED_TOOLS_VERSION::STRING}")
+      self.logger.info(" @ #{Time.now.utc.to_s}")
+      @status = :running
+      return self.pid
+    end
+    
+    # Stops the daemon.
+    def stop()
+      if self.pid.nil?
+        puts "FeedUpdater isn't running."
+      end
+      begin
+        # Die.
+        Process.kill('TERM', self.pid)
+      rescue Exception
+      end
+      begin
+        # No, really, I mean it.  You need to die.
+        system("kill #{self.pid} 2> /dev/null")
+        
+        # Perhaps I wasn't clear somehow?
+        system("kill -9 #{self.pid} 2> /dev/null")
+      rescue Exception
+      end
+      begin
+        # Clean the pid file up.
+        if File.exists?(
+            File.expand_path(self.pid_file_dir + "/feed_updater.pid"))
+          File.delete(
+            File.expand_path(self.pid_file_dir + "/feed_updater.pid"))
+        end
+      rescue Exception
+      end
+      self.logger.info("Daemon stopped.")
+      @status = :stopped
+      return nil if self.application.nil?
+      begin
+        self.application.stop()
+      rescue Exception
+      end
+      return nil
+    end
+    
+    # Restarts the daemon.
+    def restart()
+      self.stop()
+      self.start()
+    end
+    
+    # Updates all of the feeds.
+    def update_feeds()
+      if defined?(@feed_href_list_override) && @feed_href_list_override
+        self.feed_href_list()
+      else
+        @feed_href_list =
+            FeedTools.feed_cache.find(:all).collect do |cache_object|
+          cache_object.href
+        end
+      end
+      self.logger.info("Updating #{@feed_href_list.size} feeds...")
+      for href in @feed_href_list
+        begin
+          begin
+            feed = nil
+            feed_load_benchmark = Benchmark.measure do
+              feed = FeedTools::Feed.open(href)
+            end
+            unless @@on_update.nil?
+              self.cloaker(&(@@on_update)).bind(self).call(
+                feed, feed_load_benchmark.real)
+            end
+          rescue Exception => error
+            if @@on_error != nil
+              self.cloaker(&(@@on_error)).bind(self).call(href, error)
+            else
+              self.logger.info("Error updating '#{href}':")
+              self.logger.info(error)
+            end
+          end
+        rescue Exception => error
+          self.logger.info("Critical unhandled error.")
+          self.logger.info("Error updating '#{href}':")
+          self.logger.info(error)
+        end
+      end
+      unless @@on_complete.nil?
+        self.cloaker(&(@@on_complete)).bind(self).call(@feed_href_list)
+      end
+      self.logger.info("Finished updating.")
+    end
+  end
+end

data/rakefile

@@ -0,0 +1,121 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+require 'rake/gempackagetask'
+require 'rake/contrib/rubyforgepublisher'
+
+require 'date'
+require 'rbconfig'
+
+require File.join(File.dirname(__FILE__), 'lib/feed_updater', 'version')
+
+PKG_NAME      = 'feedupdater'
+PKG_VERSION   = FeedTools::FEED_UPDATER_VERSION::STRING
+PKG_FILE_NAME = "#{PKG_NAME}-#{PKG_VERSION}"
+
+RELEASE_NAME  = "REL #{PKG_VERSION}"
+
+RUBY_FORGE_PROJECT = "feedtools"
+RUBY_FORGE_USER    = "sporkmonger"
+
+PKG_FILES = FileList[
+  '[a-zA-Z]*',
+  'bin/**/*', 
+  'lib/**/*',
+  'config/**/*', 
+  'example/**/*'
+].exclude(/\bCVS\b|~$/).exclude(/database\.yml/).exclude(/\.svn$/)
+
+desc "Default Task"
+task :default => [ :package ]
+
+# Generate the RDoc documentation
+
+Rake::RDocTask.new { |rdoc|
+  rdoc.rdoc_dir = 'doc'
+  rdoc.title    = "FeedUpdater -- updater daemon for FeedTools"
+  rdoc.options << '--line-numbers' << '--inline-source' <<
+    '--accessor' << 'cattr_accessor=object'
+  rdoc.template = "#{ENV['template']}.rb" if ENV['template']
+  rdoc.rdoc_files.include('README', 'CHANGELOG')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+}
+
+# Create compressed packages
+
+spec = Gem::Specification.new do |s|
+  s.name = PKG_NAME
+  s.version = PKG_VERSION
+  s.summary = "Automatic feed updater daemon."
+  s.description = <<-DESC
+    Automatic feed updater daemon for use with FeedTools.
+  DESC
+  s.description.strip!
+
+  s.files = PKG_FILES.to_a
+  
+  s.add_dependency('feedtools', '>= 0.2.24')
+
+  s.require_path = 'lib'
+  s.autorequire = 'feed_updater'
+
+  s.rdoc_options << '--exclude' << '.'
+  s.has_rdoc = false
+  
+  s.bindir = "bin"
+  s.executables = ["feed_updater"]
+  s.default_executable = "feed_updater"
+  
+  s.author = "Bob Aman"
+  s.email = "bob@sporkmonger.com"
+  s.homepage = "http://sporkmonger.com/projects/feedupdater"
+  s.rubyforge_project = "feedtools"
+end
+  
+Rake::GemPackageTask.new(spec) do |p|
+  p.gem_spec = spec
+  p.need_tar = true
+  p.need_zip = true
+end
+
+task :lines do
+  lines, codelines, total_lines, total_codelines = 0, 0, 0, 0
+
+  for file_name in FileList["bin/*", "lib/**/*.rb"]
+    f = File.open(file_name)
+
+    while line = f.gets
+      lines += 1
+      next if line =~ /^\s*$/
+      next if line =~ /^\s*#/
+      codelines += 1
+    end
+    puts "L: #{sprintf("%4d", lines)}, LOC #{sprintf("%4d", codelines)} | #{file_name}"
+    
+    total_lines     += lines
+    total_codelines += codelines
+    
+    lines, codelines = 0, 0
+  end
+
+  puts "Total: Lines #{total_lines}, LOC #{total_codelines}"
+end
+
+# Publishing ------------------------------------------------------
+
+desc "Publish the API documentation"
+task :pdoc => [:rdoc] do 
+  Rake::SshDirPublisher.new(
+    "vacindak@sporkmonger.com",
+    "public_html/projects/feedupdater/api",
+    "doc").upload
+end
+
+desc "Publish the release files to RubyForge."
+task :release => [ :gem ] do
+  `rubyforge login`
+  release_command = "rubyforge add_release feedtools #{PKG_NAME} " +
+    "'REL #{PKG_VERSION}' pkg/#{PKG_NAME}-#{PKG_VERSION}.gem"
+  puts release_command
+  system(release_command)
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: feedupdater
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-04-13 00:00:00 -07:00
+summary: Automatic feed updater daemon.
+require_paths: 
+  - lib
+email: bob@sporkmonger.com
+homepage: http://sporkmonger.com/projects/feedupdater
+rubyforge_project: feedtools
+description: Automatic feed updater daemon for use with FeedTools.
+autorequire: feed_updater
+default_executable: feed_updater
+bindir: bin
+has_rdoc: false
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+  - Bob Aman
+files: 
+  - bin
+  - CHANGELOG
+  - config
+  - doc
+  - example
+  - lib
+  - log
+  - rakefile
+  - README
+  - bin/feed_updater
+  - lib/feed_updater
+  - lib/feed_updater.rb
+  - lib/feed_updater/vendor
+  - lib/feed_updater/version.rb
+  - lib/feed_updater/vendor/daemons
+  - lib/feed_updater/vendor/daemons.rb
+  - lib/feed_updater/vendor/daemons/application.rb
+  - lib/feed_updater/vendor/daemons/application_group.rb
+  - lib/feed_updater/vendor/daemons/cmdline.rb
+  - lib/feed_updater/vendor/daemons/controller.rb
+  - lib/feed_updater/vendor/daemons/daemonize.rb
+  - lib/feed_updater/vendor/daemons/exceptions.rb
+  - lib/feed_updater/vendor/daemons/monitor.rb
+  - lib/feed_updater/vendor/daemons/pid.rb
+  - lib/feed_updater/vendor/daemons/pidfile.rb
+  - lib/feed_updater/vendor/daemons/pidmem.rb
+  - config/feed_updater.pid
+  - config/feed_updater.yml
+  - example/custom_updater.rb
+test_files: []
+rdoc_options: 
+  - "--exclude"
+  - "."
+extra_rdoc_files: []
+executables: 
+  - feed_updater
+extensions: []
+requirements: []
+dependencies: 
+  - !ruby/object:Gem::Dependency 
+    name: feedtools
+    version_requirement: 
+    version_requirements: !ruby/object:Gem::Version::Requirement 
+      requirements: 
+        - 
+          - ">="
+          - !ruby/object:Gem::Version 
+            version: 0.2.24
+      version: