KeeperPat-feedupdater 0.2.6

data/CHANGELOG

@@ -0,0 +1,21 @@
+== FeedUpdater 0.2.6
+ * Updated for compatibility with RubyGems >= 1.0
+== FeedUpdater 0.2.5
+ * fixed bug with redefinition of file searching system
+== FeedUpdater 0.2.4
+ * added more code for detecting the location of the config file
+== FeedUpdater 0.2.3
+ * added additional logging messages
+== FeedUpdater 0.2.2
+ * better distribution of work across threads
+ * fixed major threading bug
+ * added configuration variable for sleep time
+== FeedUpdater 0.2.1
+ * no longer attempts to update feeds that have been updated very recently
+ * fixed logging levels
+ * added on_begin block
+== FeedUpdater 0.2.0
+ * multithreaded
+ * unchanged feeds will no longer call on_update
+== FeedUpdater 0.1.0
+ * initial version

data/README

@@ -0,0 +1,34 @@
+The FeedUpdater is a simple daemon for handling the automatic updating of
+FeedTools feed objects.
+
+ USAGE: feed_updater <subcommand>
+ 
+ Subcommands:
+   start
+   stop
+   restart
+   install
+
+The FeedUpdater daemon is very easy to use, especially if you're using it
+with a Rails application.  If you're going to be using it with a Rails app,
+simply install the feedupdater gem, navigate to your Rails app, and run
+the 'feed_updater install' command.  This will install FeedUpdater.  Then
+edit the feed_updater.yml config file as necessary.  You will probably need
+to define a custom updater class, which should probably go in your lib
+directory.  It's recommended that you use the example/custom_updater.rb file
+as a starting point.  You should using the updater events to copy any
+required fields to feed subscription tables or item tables as necessary so
+that a full feed parse isn't required every time the feed is displayed.
+
+The config file also has a start delay option.  This should almost always be
+set to true.  You should only turn it off when you are debugging a custom
+updater class, and it should probably never be turned off in production.
+Keeping it on prevents FeedUpdater from hitting the CPU too hard immediately
+after a server reboot, and also prevents hundreds of update processes from
+kicking off simultaneously on a shared server.
+
+If you chose to use FeedUpdater outside a Rails application, you should try
+to use a similar directory structure to what would be found in a Rails
+setup.  FeedUpdater will usually look in the current directory if it cannot
+find a file its looking for though, so if FeedUpdater can't find a file it 
+needs, changing directories will probably fix the problem.

data/bin/feed_updater

@@ -0,0 +1,317 @@
+#!/usr/bin/env ruby
+
+#--
+# Copyright (c) 2005 Robert Aman
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#++
+
+if RUBY_PLATFORM =~ /mswin/
+  puts "FeedUpdater is not supported on Windows due to a lack of fork()."
+  exit
+end
+
+# :stopdoc:
+# ROFLCOPTERS?
+# This mainly exists because of the approximately 1 billion different
+# ways to deploy this script.
+class FileStalker
+  def self.hunt(paths)
+    for path in paths
+      if File.exists?(File.expand_path(path))
+        return File.expand_path(path)
+      elsif File.exists?(File.expand_path(
+          File.dirname(__FILE__) + path))
+        return File.expand_path(File.dirname(__FILE__) + path)
+      elsif File.exists?(File.expand_path(
+          File.dirname(__FILE__) + "/" + path))
+        return File.expand_path(File.dirname(__FILE__) + "/" + path)
+      end
+    end
+    return nil
+  end
+end
+# :startdoc:
+
+$:.unshift(FileStalker.hunt([
+  "/../vendor/feedupdater/lib",
+  "vendor/feedupdater/lib",
+  "../lib"]))
+$:.uniq!
+
+require 'yaml'
+require 'feed_updater'
+
+if ARGV[0].nil?
+  # Exit quickly if no command line arguments given.
+
+  feed_updater_version = FeedTools::FEED_UPDATER_VERSION::STRING
+  feed_tools_version = "<error: not loaded>"
+  if defined?(FeedTools::FEED_TOOLS_VERSION::STRING)
+    feed_tools_version = FeedTools::FEED_TOOLS_VERSION::STRING
+  elsif defined?(FEED_TOOLS_VERSION)
+    feed_tools_version = FEED_TOOLS_VERSION
+  end
+  puts ""
+  puts "FeedUpdater #{feed_updater_version} / FeedTools #{feed_tools_version}"
+  puts "USAGE: feed_updater <subcommand>"
+  puts ""
+  puts "Subcommands:"
+  puts "  start"
+  puts "    This will start the daemon if it hasn't already been started."
+  puts ""
+  puts "  stop"
+  puts "    This will stop the daemon if it's already running."
+  puts ""
+  puts "  restart"
+  puts "    This will restart the daemon."
+  puts ""
+  puts "  install"
+  puts "    If the current directory is a Rails installation, this will"
+  puts "    install FeedUpdater into the vendor folder and add a new script"
+  puts "    to the scripts/ directory for running FeedUpdater.  Rerunning"
+  puts "    the install subcommand will update the files against the latest"
+  puts "    version of the FeedUpdater gem file you have installed.  Rails"
+  puts "    is NOT required to use FeedUpdater."
+  puts ""
+  exit
+elsif ARGV[0] == "install"
+  rails_root = nil
+  install_vendor_directory = nil
+  install_config_directory = nil
+  install_script_directory = nil
+
+  current_directory = File.expand_path(".")
+  loop do
+    entries = Dir.entries(current_directory)
+    if entries.include?("vendor") &&
+        entries.include?("config") &&
+        entries.include?("script")
+      rails_root = current_directory
+      break
+    else
+      break if current_directory == "/"
+      current_directory = File.expand_path(current_directory + "/..")
+    end
+  end
+  if rails_root.nil?
+    puts "You must run the install command when the current directory is"
+    puts "a Rails application."
+  else
+    install_vendor_directory = File.expand_path(rails_root + "/vendor")
+    install_config_directory = File.expand_path(rails_root + "/config")
+    install_script_directory = File.expand_path(rails_root + "/script")
+
+    # Install FeedUpdater gem to application vendor directory
+    system("rm -rf #{install_vendor_directory}/feedupdater")
+    unpack_string = `gem unpack feedupdater`
+    feed_updater_version =
+      unpack_string.scan(/feedupdater\-(.*)\'/).flatten[0]
+    if feed_updater_version.nil? || feed_updater_version == ""
+      puts "You need to have the feedupdater gem installed for the install"
+      puts "subcommand to run correctly."
+      exit
+    end
+    puts "Installing FeedUpdater #{feed_updater_version} " +
+      "to vendor directory..."
+
+    system("cp -rp feedupdater-#{feed_updater_version} " +
+      "#{install_vendor_directory}")
+    system("rm -rf feedupdater-#{feed_updater_version}")
+    system("mv #{install_vendor_directory}/" +
+      "feedupdater-#{feed_updater_version} " +
+      "#{install_vendor_directory}/feedupdater")
+
+    # Install FeedTools gem to application vendor directory
+    system("rm -rf #{install_vendor_directory}/feedtools")
+    unpack_string = `gem unpack feedtools`
+    feed_tools_version =
+      unpack_string.scan(/feedtools\-(.*)\'/).flatten[0]
+    if feed_tools_version.nil? || feed_tools_version == ""
+      puts "You need to have the feedtools gem installed for the install"
+      puts "subcommand to run correctly."
+      exit
+    end
+    puts "Installing FeedTools #{feed_tools_version} to vendor directory..."
+
+    system("cp -rp feedtools-#{feed_tools_version} " +
+      "#{install_vendor_directory}")
+    system("rm -rf feedtools-#{feed_tools_version}")
+    system("mv #{install_vendor_directory}/" +
+      "feedtools-#{feed_tools_version} " +
+      "#{install_vendor_directory}/feedtools")
+
+    # Copying the default config file to the config directory
+    if !File.exists?(File.expand_path(install_config_directory +
+        "/feed_updater.yml"))
+      puts "Installing default config file..."
+      config_file =
+        File.expand_path(File.dirname(__FILE__) + '/../config/feed_updater.yml')
+      system("cp -p #{config_file} #{install_config_directory}")
+    else
+      puts "Config file already exists, skipping..."
+    end
+
+    # Copying this executable to the script directory
+    puts "Copying executable to script directory..."
+    system("rm -rf #{install_script_directory}/feed_updater")
+    system("cp -p #{__FILE__} #{install_script_directory}")
+    system("chmod +x #{install_script_directory}/feed_updater")
+
+    puts "Use script/feed_updater to run FeedUpdater."
+  end
+  exit
+end
+
+config_file = FileStalker.hunt([
+  "../config/feed_updater.yml",
+  "./feed_updater.yml"
+])
+# Reportedly, in some cases, the example config is found instead.
+if config_file =~ /example/
+  config_file = FileStalker.hunt([
+    "../../../config/feed_updater.yml",
+    "../config/feed_updater.yml",
+    "./feed_updater.yml"
+  ])
+end
+if config_file.nil?
+  puts "Could not locate feed_updater.yml config file."
+  exit
+end
+
+config_hash = YAML.load(File.open(config_file, "r").read)
+load_script_path = config_hash["load_script"]
+if !load_script_path.nil?
+  result = FileStalker.hunt([
+    load_script_path,
+    File.dirname(config_file) + "/" + load_script_path,
+    File.dirname(config_file) + load_script_path,
+    File.dirname(__FILE__) + "/../" + load_script_path,
+    File.dirname(__FILE__) + "/.." + load_script_path
+  ])
+  if result.nil?
+    puts "Could not locate #{load_script_path.inspect} file."
+    exit
+  else
+    load_script_path = result
+  end
+end
+
+pid_file_path = config_hash["pid_file_path"]
+if !pid_file_path.nil?
+  result = FileStalker.hunt([
+    pid_file_path,
+    File.dirname(config_file) + "/" + pid_file_path,
+    File.dirname(config_file) + pid_file_path,
+    File.dirname(__FILE__) + "/../" + pid_file_path,
+    File.dirname(__FILE__) + "/.." + pid_file_path,
+    "../" + pid_file_path
+  ])
+  pid_file_path = result if !result.nil?
+else
+  result = FileStalker.hunt([
+    File.dirname(__FILE__) + "/../config",
+    "./config",
+    File.dirname(__FILE__) + "/config"
+  ])
+  pid_file_path = result if !result.nil?
+end
+
+log_file_path = config_hash["log_file_path"]
+if !log_file_path.nil?
+  result = FileStalker.hunt([
+    log_file_path,
+    File.dirname(config_file) + "/" + log_file_path,
+    File.dirname(config_file) + log_file_path,
+    File.dirname(__FILE__) + "/../" + log_file_path,
+    File.dirname(__FILE__) + "/.." + log_file_path,
+    "../" + log_file_path
+  ])
+  log_file_path = result if !result.nil?
+else
+  result = FileStalker.hunt([
+    File.dirname(__FILE__) + "/../log",
+    "./log",
+    File.dirname(__FILE__) + "/log",
+    File.dirname(__FILE__) + "/../logs",
+    "./logs",
+    File.dirname(__FILE__) + "/logs"
+  ])
+  log_file_path = result if !result.nil?
+end
+
+script_class = nil
+existing_subclasses = []
+ObjectSpace.each_object do |instance|
+  if instance.class.name == "Class" &&
+      instance.ancestors.include?(FeedTools::FeedUpdater)
+    existing_subclasses << instance
+  end
+end
+load(load_script_path)
+loaded_subclasses = []
+ObjectSpace.each_object do |instance|
+  if instance.class.name == "Class" &&
+      instance.ancestors.include?(FeedTools::FeedUpdater)
+    loaded_subclasses << instance
+  end
+end
+script_classes = (loaded_subclasses - existing_subclasses)
+if script_classes.size == 0
+  puts "There do not appear to be any subclasses of FeedTools::FeedUpdater."
+  exit
+elsif script_classes.size > 1
+  puts "There appear to be multiple subclasses of FeedTools::FeedUpdater."
+  exit
+else
+  script_class = script_classes[0]
+end
+
+updater = nil
+if load_script_path.nil?
+  updater = FeedTools::FeedUpdater.new
+  FeedTools::FeedUpdater.on_update do |feed, seconds|
+    updater.logger.info("Loaded '#{feed.href}'.")
+    updater.logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
+  end
+  FeedTools::FeedUpdater.on_error do |href, error|
+    updater.logger.info("Error updating '#{href}':")
+    updater.logger.info(error)
+  end
+else
+  updater = script_class.new
+end
+
+for key in config_hash.keys
+  key_sym = key.to_s.to_sym
+  updater.updater_options[key_sym] = config_hash[key]
+end
+updater.pid_file_dir = pid_file_path
+updater.log_file_dir = log_file_path
+
+case ARGV[0]
+when "start"
+  updater.start()
+when "stop"
+  updater.stop()
+when "restart"
+  updater.restart()
+end

data/config/feed_updater.yml

@@ -0,0 +1,22 @@
+# This ensures that you don't break shared servers.
+start_delay: false
+
+# This is the path to the custom handler for updated feeds.
+# See the documentation for details.
+load_script: example/custom_updater.rb
+
+# Keep this set to 1 unless you're certain you need to set it higher.
+# Never set it higher than 1 if you're on a shared server.
+threads: 5
+
+# The amount of time to sleep between updates.  Expressed in minutes.
+sleep_time: 65
+
+# This is the path to store the pid files.
+pid_file_path: config
+
+# This is the path to store the log files.
+log_file_path: log
+
+# Log level
+log_level: 0

data/example/custom_updater.rb

@@ -0,0 +1,24 @@
+class CustomUpdater < FeedTools::FeedUpdater
+  on_begin do
+#     self.feed_href_list = [
+#       "http://www.gigaom.com/feed/rss2/",
+#       "http://feeds.feedburner.com/ManeuverMarketingCommunique",
+#       "http://www.afp.com/english/rss/stories.xml",
+#       "http://www.railheaddesign.com/rss/railhead.xml",
+#       "http://www.nateanddi.com/rssfeed.xml"
+#     ]
+  end
+  
+  on_update do |feed, seconds|
+    self.logger.info("Loaded '#{feed.href}'.")
+    self.logger.info("=> Updated (#{feed.title}) in #{seconds} seconds.")
+  end
+  
+  on_error do |href, error|
+    self.logger.info("Error updating '#{href}':")
+    self.logger.info(error)
+  end
+
+  on_complete do |updated_feed_hrefs|
+  end
+end