feedupdater 0.1.0

data/CHANGELOG

@@ -0,0 +1,2 @@
+== FeedUpdater 0.1.0
+ * initial version

data/README

@@ -0,0 +1,34 @@
+The FeedUpdater is a simple daemon for handling the automatic updating of
+FeedTools feed objects.
+
+ USAGE: feed_updater <subcommand>
+ 
+ Subcommands:
+   start
+   stop
+   restart
+   install
+
+The FeedUpdater daemon is very easy to use, especially if you're using it
+with a Rails application.  If you're going to be using it with a Rails app,
+simply install the feedupdater gem, navigate to your Rails app, and run
+the 'feed_updater install' command.  This will install FeedUpdater.  Then
+edit the feed_updater.yml config file as necessary.  You will probably need
+to define a custom updater class, which should probably go in your lib
+directory.  It's recommended that you use the example/custom_updater.rb file
+as a starting point.  You should using the updater events to copy any
+required fields to feed subscription tables or item tables as necessary so
+that a full feed parse isn't required every time the feed is displayed.
+
+The config file also has a start delay option.  This should almost always be
+set to true.  You should only turn it off when you are debugging a custom
+updater class, and it should probably never be turned off in production.
+Keeping it on prevents FeedUpdater from hitting the CPU too hard immediately
+after a server reboot, and also prevents hundreds of update processes from
+kicking off simultaneously on a shared server.
+
+If you chose to use FeedUpdater outside a Rails application, you should try
+to use a similar directory structure to what would be found in a Rails
+setup.  FeedUpdater will usually look in the current directory if it cannot
+find a file its looking for though, so if FeedUpdater can't find a file it 
+needs, changing directories will probably fix the problem.

data/bin/feed_updater

@@ -0,0 +1,309 @@
+#!/usr/local/bin/ruby
+
+#--
+# Copyright (c) 2005 Robert Aman
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+if RUBY_PLATFORM =~ /mswin/
+  puts "FeedUpdater is not supported on Windows due to a lack of fork()."
+  exit
+end
+
+# :stopdoc:
+# ROFLCOPTERS?
+# This mainly exists because of the approximately 1 billion different
+# ways to deploy this script.
+class FileStalker
+  def self.hunt(paths)
+    for path in paths
+      if File.exists?(File.expand_path(path))
+        return File.expand_path(path)
+      elsif File.exists?(File.expand_path(
+          File.dirname(__FILE__) + path))
+        return File.expand_path(File.dirname(__FILE__) + path)
+      elsif File.exists?(File.expand_path(
+          File.dirname(__FILE__) + "/" + path))
+        return File.expand_path(File.dirname(__FILE__) + "/" + path)
+      end
+    end
+    return nil
+  end
+end
+# :startdoc:
+
+$:.unshift(FileStalker.hunt([
+  "/../vendor/feedupdater/lib",
+  "vendor/feedupdater/lib",
+  "../lib"]))
+$:.uniq!
+
+require 'yaml'
+require 'feed_updater'
+
+if ARGV[0].nil?
+  # Exit quickly if no command line arguments given.
+  
+  feed_updater_version = FeedTools::FEED_UPDATER_VERSION::STRING
+  feed_tools_version = "<error: not loaded>"
+  if defined?(FeedTools::FEED_TOOLS_VERSION::STRING)
+    feed_tools_version = FeedTools::FEED_TOOLS_VERSION::STRING
+  elsif defined?(FEED_TOOLS_VERSION)
+    feed_tools_version = FEED_TOOLS_VERSION
+  end
+  puts ""
+  puts "FeedUpdater #{feed_updater_version} / FeedTools #{feed_tools_version}"
+  puts "USAGE: feed_updater <subcommand>"
+  puts ""
+  puts "Subcommands:"
+  puts "  start"
+  puts "    This will start the daemon if it hasn't already been started."
+  puts ""
+  puts "  stop"
+  puts "    This will stop the daemon if it's already running."
+  puts ""
+  puts "  restart"
+  puts "    This will restart the daemon."
+  puts ""
+  puts "  install"
+  puts "    If the current directory is a Rails installation, this will"
+  puts "    install FeedUpdater into the vendor folder and add a new script"
+  puts "    to the scripts/ directory for running FeedUpdater.  Rerunning"
+  puts "    the install subcommand will update the files against the latest"
+  puts "    version of the FeedUpdater gem file you have installed.  Rails"
+  puts "    is NOT required to use FeedUpdater."
+  puts ""
+  exit
+elsif ARGV[0] == "install"
+  rails_root = nil
+  install_vendor_directory = nil
+  install_config_directory = nil
+  install_script_directory = nil
+  
+  current_directory = File.expand_path(".")
+  loop do
+    entries = Dir.entries(current_directory)
+    if entries.include?("vendor") &&
+        entries.include?("config") &&
+        entries.include?("script")
+      rails_root = current_directory
+      break
+    else
+      break if current_directory == "/"
+      current_directory = File.expand_path(current_directory + "/..")
+    end
+  end
+  if rails_root.nil?
+    puts "You must run the install command when the current directory is"
+    puts "a Rails application."
+  else
+    install_vendor_directory = File.expand_path(rails_root + "/vendor")
+    install_config_directory = File.expand_path(rails_root + "/config")
+    install_script_directory = File.expand_path(rails_root + "/script")
+    
+    # Install FeedUpdater gem to application vendor directory
+    system("rm -rf #{install_vendor_directory}/feedupdater")
+    unpack_string = `gem unpack feedupdater`
+    feed_updater_version =
+      unpack_string.scan(/\'feedupdater\-(.*)\'/).flatten[0]
+    if feed_updater_version.nil? || feed_updater_version == ""
+      puts "You need to have the feedupdater gem installed for the install"
+      puts "subcommand to run correctly."
+      exit
+    end
+    puts "Installing FeedUpdater #{feed_updater_version} " +
+      "to vendor directory..."
+    
+    system("cp -rp feedupdater-#{feed_updater_version} " +
+      "#{install_vendor_directory}")
+    system("rm -rf feedupdater-#{feed_updater_version}")
+    system("mv #{install_vendor_directory}/" + 
+      "feedupdater-#{feed_updater_version} " +
+      "#{install_vendor_directory}/feedupdater")
+
+    # Install FeedTools gem to application vendor directory
+    system("rm -rf #{install_vendor_directory}/feedtools")
+    unpack_string = `gem unpack feedtools`
+    feed_tools_version =
+      unpack_string.scan(/\'feedtools\-(.*)\'/).flatten[0]
+    if feed_tools_version.nil? || feed_tools_version == ""
+      puts "You need to have the feedtools gem installed for the install"
+      puts "subcommand to run correctly."
+      exit
+    end
+    puts "Installing FeedTools #{feed_tools_version} to vendor directory..."
+
+    system("cp -rp feedtools-#{feed_tools_version} " +
+      "#{install_vendor_directory}")
+    system("rm -rf feedtools-#{feed_tools_version}")
+    system("mv #{install_vendor_directory}/" + 
+      "feedtools-#{feed_tools_version} " +
+      "#{install_vendor_directory}/feedtools")
+
+    # Copying the default config file to the config directory
+    if !File.exists?(File.expand_path(install_config_directory +
+        "/feed_updater.yml"))
+      puts "Installing default config file..."
+      config_file =
+        File.expand_path(File.dirname(__FILE__) + '/../config/feed_updater.yml')
+      system("cp -p #{config_file} #{install_config_directory}")
+    else
+      puts "Config file already exists, skipping..."
+    end
+
+    # Copying this executable to the script directory
+    puts "Copying executable to script directory..."
+    system("rm -rf #{install_script_directory}/feed_updater")
+    system("cp -p #{__FILE__} #{install_script_directory}")
+    system("chmod +x #{install_script_directory}/feed_updater")
+    
+    puts "Use script/feed_updater to run FeedUpdater."
+  end
+  exit
+end
+
+config_file = FileStalker.hunt([
+  "./feed_updater.yml",
+  "../config/feed_updater.yml"
+])
+if config_file.nil?
+  puts "Could not locate feed_updater.yml config file."
+  exit
+end
+
+config_hash = YAML.load(File.open(config_file, "r").read)
+load_script_path = config_hash["load_script"]
+if !load_script_path.nil?
+  result = FileStalker.hunt([
+    load_script_path,
+    File.dirname(config_file) + "/" + load_script_path,
+    File.dirname(config_file) + load_script_path,
+    File.dirname(__FILE__) + "/../" + load_script_path,
+    File.dirname(__FILE__) + "/.." + load_script_path
+  ])
+  if result.nil?
+    puts "Could not locate #{load_script_path.inspect} file."
+    exit
+  else
+    load_script_path = result
+  end
+end
+
+pid_file_path = config_hash["pid_file_path"]
+if !pid_file_path.nil?
+  result = FileStalker.hunt([
+    pid_file_path,
+    File.dirname(config_file) + "/" + pid_file_path,
+    File.dirname(config_file) + pid_file_path,
+    File.dirname(__FILE__) + "/../" + pid_file_path,
+    File.dirname(__FILE__) + "/.." + pid_file_path,
+    "../" + pid_file_path
+  ])
+  pid_file_path = result if !result.nil?
+else
+  result = FileStalker.hunt([
+    File.dirname(__FILE__) + "/../config",
+    "./config",
+    File.dirname(__FILE__) + "/config"
+  ])
+  pid_file_path = result if !result.nil?
+end
+
+log_file_path = config_hash["log_file_path"]
+if !log_file_path.nil?
+  result = FileStalker.hunt([
+    log_file_path,
+    File.dirname(config_file) + "/" + log_file_path,
+    File.dirname(config_file) + log_file_path,
+    File.dirname(__FILE__) + "/../" + log_file_path,
+    File.dirname(__FILE__) + "/.." + log_file_path,
+    "../" + log_file_path
+  ])
+  log_file_path = result if !result.nil?
+else
+  result = FileStalker.hunt([
+    File.dirname(__FILE__) + "/../log",
+    "./log",
+    File.dirname(__FILE__) + "/log",
+    File.dirname(__FILE__) + "/../logs",
+    "./logs",
+    File.dirname(__FILE__) + "/logs"
+  ])
+  log_file_path = result if !result.nil?
+end
+
+script_class = nil
+existing_subclasses = []
+ObjectSpace.each_object do |instance|
+  if instance.class.name == "Class" &&
+      instance.ancestors.include?(FeedTools::FeedUpdater)
+    existing_subclasses << instance
+  end
+end
+load(load_script_path)
+loaded_subclasses = []
+ObjectSpace.each_object do |instance|
+  if instance.class.name == "Class" &&
+      instance.ancestors.include?(FeedTools::FeedUpdater)
+    loaded_subclasses << instance
+  end
+end
+script_classes = (loaded_subclasses - existing_subclasses)
+if script_classes.size == 0
+  puts "There do not appear to be any subclasses of FeedTools::FeedUpdater."
+  exit
+elsif script_classes.size > 1
+  puts "There appear to be multiple subclasses of FeedTools::FeedUpdater."
+  exit
+else
+  script_class = script_classes[0]
+end
+
+updater = nil
+if load_script_path.nil?
+  updater = FeedTools::FeedUpdater.new
+  FeedTools::FeedUpdater.on_update do |feed, seconds|
+    updater.logger.info("Loaded '#{feed.href}'.")
+    updater.logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
+  end
+  FeedTools::FeedUpdater.on_error do |href, error|
+    updater.logger.info("Error updating '#{href}':")
+    updater.logger.info(error)
+  end
+else
+  updater = script_class.new
+end
+
+for key in config_hash.keys
+  key_sym = key.to_s.to_sym
+  updater.updater_options[key_sym] = config_hash[key]
+end
+updater.pid_file_dir = pid_file_path
+updater.log_file_dir = log_file_path
+
+case ARGV[0]
+when "start"
+  updater.start()
+when "stop"
+  updater.stop()
+when "restart"
+  updater.restart()
+end

data/config/feed_updater.pid

@@ -0,0 +1 @@
+617

data/config/feed_updater.yml

@@ -0,0 +1,12 @@
+# This ensures that you don't break shared servers.
+start_delay: false
+
+# This is the path to the custom handler for updated feeds.
+# See the documentation for details.
+load_script: example/custom_updater.rb
+
+# This is the path to store the pid files.
+pid_file_path: config
+
+# This is the path to store the log files.
+log_file_path: log

data/example/custom_updater.rb

@@ -0,0 +1,14 @@
+class CustomUpdater < FeedTools::FeedUpdater
+  on_update do |feed, seconds|
+    logger.info("Loaded '#{feed.href}'.")
+    logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
+  end
+  
+  on_error do |href, error|
+    logger.info("Error updating '#{href}':")
+    logger.info(error)
+  end
+
+  on_complete do |updated_feed_hrefs|
+  end
+end

data/lib/feed_updater/vendor/daemons/application.rb

@@ -0,0 +1,298 @@
+require 'daemons/pidfile'
+require 'daemons/pidmem'
+
+
+module Daemons
+
+  class Application
+  
+    attr_accessor :app_argv
+    attr_accessor :controller_argv
+    
+    # the Pid instance belonging to this application
+    attr_reader :pid
+    
+    # the ApplicationGroup the application belongs to
+    attr_reader :group
+    
+    # my private options
+    attr_reader :options
+    
+    
+    def initialize(group, add_options = {}, pid = nil)
+      @group = group
+      @options = group.options.dup
+      @options.update(add_options)
+      
+      unless @pid = pid
+        if dir = pidfile_dir
+          @pid = PidFile.new(pidfile_dir(), @group.app_name, @group.multiple)
+        else
+          @pid = PidMem.new
+        end
+      end
+    end
+    
+    def script
+      @script || @group.script
+    end
+    
+    def pidfile_dir
+      Pid.dir(@dir_mode || @group.dir_mode, @dir || @group.dir, @script || @group.script)
+    end
+    
+    def logfile
+      (options[:log_output] && pidfile_dir()) ? File.join(pidfile_dir(), @group.app_name + '.output') : nil
+    end
+    
+    def start_none
+      unless options[:ontop]
+        Daemonize.daemonize #(logfile)
+      else
+        Daemonize.simulate
+      end
+      
+      @pid.pid = Process.pid
+      
+      
+      # We need this to remove the pid-file if the applications exits by itself.
+      # Note that <tt>at_text</tt> will only be run if the applications exits by calling 
+      # <tt>exit</tt>, and not if it calls <tt>exit!</tt> (so please don't call <tt>exit!</tt>
+      # in your application!
+      #
+      at_exit {
+        @pid.cleanup rescue nil
+        
+        # If the option <tt>:backtrace</tt> is used and the application did exit by itself
+        # create a exception log.
+        if options[:backtrace] and not options[:ontop] and not $daemons_sigterm
+          exception_log() rescue nil
+        end
+          
+      }
+      
+      # This part is needed to remove the pid-file if the application is killed by 
+      # daemons or manually by the user.
+      # Note that the applications is not supposed to overwrite the signal handler for
+      # 'TERM'.
+      #
+      trap('TERM') {
+        @pid.cleanup rescue nil
+        $daemons_sigterm = true
+        
+        exit
+      }
+    end
+    
+    def start_exec
+      unless options[:ontop]
+        Daemonize.daemonize(logfile)
+      else
+        Daemonize.simulate(logfile)
+      end
+      
+      @pid.pid = Process.pid
+        
+      ENV['DAEMONS_ARGV'] = @controller_argv.join(' ')      
+      # haven't tested yet if this is really passed to the exec'd process...
+      
+      Kernel.exec(script(), *ARGV)
+    end
+    
+    def start_load
+      unless options[:ontop]
+        Daemonize.daemonize(logfile)
+      else
+        Daemonize.simulate(logfile)
+      end
+      
+      @pid.pid = Process.pid
+      
+      
+      # We need this to remove the pid-file if the applications exits by itself.
+      # Note that <tt>at_text</tt> will only be run if the applications exits by calling 
+      # <tt>exit</tt>, and not if it calls <tt>exit!</tt> (so please don't call <tt>exit!</tt>
+      # in your application!
+      #
+      at_exit {
+        @pid.cleanup rescue nil
+        
+        # If the option <tt>:backtrace</tt> is used and the application did exit by itself
+        # create a exception log.
+        if options[:backtrace] and not options[:ontop] and not $daemons_sigterm
+          exception_log() rescue nil
+        end
+          
+      }
+      
+      # This part is needed to remove the pid-file if the application is killed by 
+      # daemons or manually by the user.
+      # Note that the applications is not supposed to overwrite the signal handler for
+      # 'TERM'.
+      #
+      trap('TERM') {
+        @pid.cleanup rescue nil
+        $daemons_sigterm = true
+        
+        exit
+      }
+      
+      # Know we really start the script...
+      $DAEMONS_ARGV = @controller_argv
+      ENV['DAEMONS_ARGV'] = @controller_argv.join(' ')
+      
+      ARGV.clear
+      ARGV.concat @app_argv if @app_argv
+      
+      # TODO: begin - rescue - end around this and exception logging
+      load script()
+    end
+    
+    def start_proc
+      return unless options[:proc]
+      
+      unless options[:ontop]
+        @pid.pid = Daemonize.call_as_daemon(options[:proc], logfile)
+      else
+#         Daemonize.simulate(logfile)
+#         
+#         @pid.pid = Process.pid
+#         
+#         Thread.new(&options[:proc])
+        unless @pid.pid = Process.fork
+          Daemonize.simulate(logfile)
+          options[:proc].call
+          exit
+        else
+          Process.detach(@pid.pid)
+        end
+      end
+    end
+    
+    
+    def start
+      @group.create_monitor(@group.applications[0] || self)
+      
+      case options[:mode]
+        when :none
+          start_none
+        when :exec
+          start_exec
+        when :load
+          start_load
+        when :proc
+          start_proc
+        else
+          start_load
+      end
+    end
+    
+#     def run
+#       if @group.controller.options[:exec]
+#         run_via_exec()
+#       else
+#         run_via_load()
+#       end
+#     end
+#      
+#     def run_via_exec
+#       
+#     end
+#     
+#     def run_via_load
+#       
+#     end
+
+
+    # This is a nice little function for debugging purposes:
+    # In case a multi-threaded ruby script exits due to an uncaught exception
+    # it may be difficult to find out where the exception came from because
+    # one cannot catch exceptions that are thrown in threads other than the main
+    # thread.
+    #
+    # This function searches for all exceptions in memory and outputs them to STDERR
+    # (if it is connected) and to a log file in the pid-file directory.
+    #
+    def exception_log
+      require 'logger'
+      
+      l_file = Logger.new(File.join(pidfile_dir(), @group.app_name + '.log'))
+      
+      
+      # the code below only logs the last exception
+#       e = nil
+#       
+#       ObjectSpace.each_object {|o|
+#         if ::Exception === o
+#           e = o
+#         end
+#       }
+#       
+#       l_file.error e
+#       l_file.close
+      
+      # this code logs every exception found in memory
+      ObjectSpace.each_object {|o|
+        if ::Exception === o
+          l_file.error o
+        end
+      }
+      
+      l_file.close
+    end
+    
+    
+    def stop
+      if options[:force] and not running?
+        self.zap
+        return
+      end
+      
+      # Catch errors when trying to kill a process that doesn't
+      # exist. This happens when the process quits and hasn't been
+      # restarted by the monitor yet. By catching the error, we allow the
+      # pid file clean-up to occur.
+      begin
+        Process.kill('TERM', @pid.pid)
+      rescue Errno::ESRCH => e
+        puts "#{e} #{@pid.pid}"
+        puts "deleting pid-file."
+      end
+      
+      # We try to remove the pid-files by ourselves, in case the application
+      # didn't clean it up.
+      @pid.cleanup rescue nil
+      
+    end
+    
+    def zap
+      @pid.cleanup
+    end
+    
+    def zap!
+      @pid.cleanup rescue nil
+    end
+    
+    def show_status
+      running = self.running?
+      
+      puts "#{self.group.app_name}: #{running ? '' : 'not '}running#{(running and @pid.exists?) ? ' [pid ' + @pid.pid.to_s + ']' : ''}#{(@pid.exists? and not running) ? ' (but pid-file exists: ' + @pid.pid.to_s + ')' : ''}"
+    end
+    
+    # This function implements a (probably too simle) method to detect
+    # whether the program with the pid found in the pid-file is still running.
+    # It just searches for the pid in the output of <tt>ps ax</tt>, which
+    # is probably not a good idea in some cases.
+    # Alternatives would be to use a direct access method the unix process control
+    # system.
+    #
+    def running?
+      if @pid.exists?
+        return Pid.running?(@pid.pid)
+      end
+      
+      return false
+    end
+  end
+  
+end