audrey2 0.2.1 → 0.3.0

data/README.rdoc

@@ -7,6 +7,20 @@ Audrey 2.0 is a gem for feed processing and aggregation.
 The initial codebase is an adaptation of several other prototypes. It suffers from
 severe shortages of documentation and testing. Both are coming soon.
 
+== Release notes 
+
+The 0.3.0 version of Audrey 2.0 makes three changes which could break compatibility with
+existing installations. They are all very easy to deal with:
+
+1) The default location of the configuration file has moved from /etc/audrey2/audrey2.conf
+to /etc/audrey2.conf
+
+2) In a theme's entry.haml when you reference data in the entry hash you now need to index 
+the hash using symbols rather than strings: entry[:title] replaces entry['title'].
+
+3) If email notification of errors is configured via SMTP you now need to identify the 
+SMTP server with the key 'address' instead of 'server' in Audrey 2.0's configuration file.
+
 == Installation
 
 gem install audrey2

data/Rakefile

@@ -5,13 +5,15 @@ begin
   require 'jeweler'
   Jeweler::Tasks.new do |gem|
     gem.name        = "audrey2"
-    gem.summary     = "Gem for feed processing and aggregation"
-    gem.description = "Gem for feed processing and aggregation"
+    gem.summary     = "Feed processing and aggregation"
+    gem.description = "Audrey 2.0 is a command-line utility for customizable feed processing, aggregation, and output."
     gem.email       = "sven.aas@gmail.com"
     gem.homepage    = "http://github.com/svenaas/audrey2"
     gem.authors     = ["Sven Aas"]
     gem.add_dependency "feed-normalizer", "~>1.5.2"
     gem.add_dependency "haml", "~>3.0.13"    
+    gem.add_development_dependency "shoulda", ">= 2.11.1"
+    gem.add_development_dependency "mocha", ">= 0.9.8"    
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.3.0

data/audrey2.conf.sample

@@ -31,7 +31,7 @@ themes_folder:   ./themes
 #   from: Audrey 2.0 <noreply@test.com>
 #   via: smtp
 #   smtp:
-#     server: smtp.test.com
+#     address: smtp.test.com
 #     port: 25
 #     user_name: test
 #     password: password

data/audrey2.gemspec

@@ -5,13 +5,13 @@
 
 Gem::Specification.new do |s|
   s.name = %q{audrey2}
-  s.version = "0.2.1"
+  s.version = "0.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Sven Aas"]
-  s.date = %q{2010-07-30}
+  s.date = %q{2010-08-17}
   s.default_executable = %q{feedme}
-  s.description = %q{Gem for feed processing and aggregation}
+  s.description = %q{Audrey 2.0 is a command-line utility for customizable feed processing, aggregation, and output.}
   s.email = %q{sven.aas@gmail.com}
   s.executables = ["feedme"]
   s.extra_rdoc_files = [
@@ -28,13 +28,33 @@ Gem::Specification.new do |s|
      "audrey2.conf.sample",
      "audrey2.gemspec",
      "bin/feedme",
-     "lib/audrey2.rb"
+     "lib/audrey2.rb",
+     "test/helper.rb",
+     "test/test_config.rb",
+     "test/test_email.rb",
+     "test/test_misc.rb",
+     "test/test_options.rb",
+     "test/test_parse.rb",
+     "test/test_recipes.rb",
+     "test/test_sort.rb",
+     "test/test_themes.rb"
   ]
   s.homepage = %q{http://github.com/svenaas/audrey2}
   s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
-  s.summary = %q{Gem for feed processing and aggregation}
+  s.summary = %q{Feed processing and aggregation}
+  s.test_files = [
+    "test/helper.rb",
+     "test/test_config.rb",
+     "test/test_email.rb",
+     "test/test_misc.rb",
+     "test/test_options.rb",
+     "test/test_parse.rb",
+     "test/test_recipes.rb",
+     "test/test_sort.rb",
+     "test/test_themes.rb"
+  ]
 
   if s.respond_to? :specification_version then
     current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
@@ -43,13 +63,19 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<feed-normalizer>, ["~> 1.5.2"])
       s.add_runtime_dependency(%q<haml>, ["~> 3.0.13"])
+      s.add_development_dependency(%q<shoulda>, [">= 2.11.1"])
+      s.add_development_dependency(%q<mocha>, [">= 0.9.8"])
     else
       s.add_dependency(%q<feed-normalizer>, ["~> 1.5.2"])
       s.add_dependency(%q<haml>, ["~> 3.0.13"])
+      s.add_dependency(%q<shoulda>, [">= 2.11.1"])
+      s.add_dependency(%q<mocha>, [">= 0.9.8"])
     end
   else
     s.add_dependency(%q<feed-normalizer>, ["~> 1.5.2"])
     s.add_dependency(%q<haml>, ["~> 3.0.13"])
+    s.add_dependency(%q<shoulda>, [">= 2.11.1"])
+    s.add_dependency(%q<mocha>, [">= 0.9.8"])
   end
 end
 

data/bin/feedme

@@ -1,45 +1,14 @@
 #!/usr/bin/ruby
-
 require 'rubygems'
 require 'audrey2'
-require 'optparse'
-
-options = {}
-
-opts = OptionParser.new do |opts|
-  opts.banner = "Usage: feedme [OPTIONS] recipes"
-  
-  options[:config] = '/etc/audrey2/audrey2.conf'
-  opts.on( '--config CONFIGFILE', "Location of config file", "(default: /etc/audrey2/audrey2.conf)" )  do |f|
-    options[:config] = f
-  end
-
-  opts.on_tail( '-h', '--help', 'Display this screen' ) do
-    puts opts
-    exit
-  end
-end
-
-begin 
-  opts.parse! ARGV
-  options
-rescue OptionParser::InvalidOption => e
-  $stderr.puts e
-  $stderr.puts opts
-  exit 1
-rescue OptionParser::MissingArgument => e
-  $stderr.puts e
-  $stderr.puts opts
-  exit 1
-end
 
+options = Audrey2::Options.parse(ARGV)   
+ 
 recipes = ARGV
-if recipes.length == 0
-  $stderr.puts "You must specify at least one recipe to feed me"
-  $stderr.puts opts
-  exit 1
-end
 
-audrey2 = Audrey2::Aggregator.new(options[:config])
-
-recipes.each { |recipe| audrey2.feed_me(recipe) }
+if recipes.empty?
+  abort "You must specify at least one recipe to feed me"
+else
+  audrey2 = Audrey2::Aggregator.new(options[:config])
+  recipes.each { |recipe| audrey2.feed_me(recipe) }
+end

data/lib/audrey2.rb

@@ -3,8 +3,59 @@ require 'yaml'
 require 'feed-normalizer'
 require 'open-uri'
 require 'haml'
+require 'optparse'
+
+module HashExtensions # Adapted from http://gist.github.com/151324 by Avdi Grimm and Paul Berry
+  def recursively_symbolize_keys
+    inject({}) do |acc, (k,v)|
+      key = String === k ? k.to_sym : k
+      value = case v
+        when Hash
+          v.recursively_symbolize_keys
+        when Array
+          v.collect { |e| Hash === e ? e.recursively_symbolize_keys : e }
+        else
+          v
+        end
+      acc[key] = value
+      acc
+    end
+  end
+end
+Hash.send(:include, HashExtensions)
 
 module Audrey2
+  class Options
+    def self.parse(args)
+      options = {}
+
+      opts = OptionParser.new do |opts|
+        opts.banner = "Usage: feedme [OPTIONS] recipes"
+
+        options[:config] = '/etc/audrey2.conf'
+        opts.on( '--config CONFIGFILE', "Location of config file", "(default: /etc/audrey2.conf)" )  do |f|
+          options[:config] = f
+        end
+
+        opts.on_tail( '-h', '--help', 'Display this screen' ) do
+          puts opts
+          exit
+        end
+
+        begin
+          opts.parse! args
+          options
+        rescue OptionParser::ParseError => e
+          warn e.message
+          $stderr.puts opts
+          exit 1
+        end
+      end
+
+      options
+    end
+  end
+
 
   class Aggregator
     def initialize(configfile)
@@ -15,19 +66,21 @@ module Audrey2
       begin
         # Load recipe and theme and make sure everything is in order
         recipe = load_recipe(recipe_name)
-        init_theme(recipe['theme'])
-        output_file = recipe['output_file']
+        output_file = recipe[:output_file]
         verify_output_file(output_file)
-        max_entries = recipe['max_entries'] || 1000
+        max_entries = recipe[:max_entries] || 1000
+
+        # Load theme
+        init_theme(recipe[:theme])
 
         # Download and parse the feeds
         entry_sources = {}
-        feeds = recipe['feeds'].collect { |feed| parse_feed(feed, entry_sources) }
+        feeds = recipe[:feeds].collect { |feed| parse_feed(feed, entry_sources) }
 
         # Aggregate and sort the entries
         entries = []
         feeds.each { |feed| entries += feed.entries }
-        sort!(entries)
+        entries.sort! &entry_sort_comparator
 
         # Prepare template evaluation scope including any helper code defined in the theme
         scope = Object.new
@@ -42,8 +95,9 @@ module Audrey2
         end
 
         File.open(output_file, 'w') { |f| f << output }
-
       rescue Exception => e
+        # NOTE: This also catches SystemExit as can be raise by Kernel#exit when recipes
+        # and themes are loaded and verified. Is it good to handle those events here? Perhaps ...
         if @email
           email(<<-EOF
 An exception occurred while running recipe #{recipe_name}
@@ -67,24 +121,25 @@ EOF
       return unless @email
 
       mail = Mail.new
-      mail[:from] =    @email['from']
-      mail[:to] =      @email['to']
+      mail[:from] =    @email[:from]
+      mail[:to] =      @email[:to]
       mail[:subject] = "[AUDREY 2.0] Exception Notification"
       mail[:body]    = e
-
-      case @email['via']
-      when 'sendmail'
+      
+      @email[:via] ||= 'sendmail' # Do this somewhere else?
+      case @email[:via].to_sym    # Do this somewhere else too?
+      when :sendmail
         mail.delivery_method :sendmail
-      when 'smtp'
-        raise "Missing SMTP configuration" unless @email['smtp']
+      when :smtp
+        raise "Missing SMTP configuration" unless @email[:smtp]
         smtp = {
-          :address        => @email['smtp']['server']         || 'localhost',
-          :port           => @email['smtp']['port']           || 25
+          :address        => @email[:smtp][:address]        || 'localhost',
+          :port           => @email[:smtp][:port]           || 25
         }
-        smtp[:domain]         = @email['smtp']['domain']         if @email['smtp']['domain']
-        smtp[:user_name]      = @email['smtp']['user_name']      if @email['smtp']['user_name']
-        smtp[:password]       = @email['smtp']['password']       if @email['smtp']['password']
-        smtp[:authentication] = @email['smtp']['authentication'] if @email['smtp']['authentication']
+        smtp[:domain]         = @email[:smtp][:domain]         if @email[:smtp][:domain]
+        smtp[:user_name]      = @email[:smtp][:user_name]      if @email[:smtp][:user_name]
+        smtp[:password]       = @email[:smtp][:password]       if @email[:smtp][:password]
+        smtp[:authentication] = @email[:smtp][:authentication] if @email[:smtp][:authentication]
         mail.delivery_method :smtp, smtp
       end
 
@@ -129,38 +184,47 @@ EOF
       @helper_code = nil
       if File.exist? helper_file
         if ! File.readable? helper_file
-          $stderr.puts "ERROR: Helper file #{helper_file} is not readable"
+          $stderr.puts "ERROR: Helpers file #{helper_file} is not readable"
           exit(1)
         end
         @helper_code = File.open(helper_file) { |f| f.read }
       end
     end
 
-    # Uses the sort order specified in configuration
-    def sort!(entries)
-      case @sort
-      when 'reverse-chronological'
-        entries.sort! {|a, b| b.date_published <=> a.date_published } # Reverse chronological
+    # Implements sort orders which may be specified in configuration    
+    def entry_sort_comparator(sort = @sort) # Defaults to the sort order specified in configuration
+      case sort
+      when :reverse_chronological
+        Proc.new {|a, b| b.date_published <=> a.date_published } 
+      when :chronological
+        Proc.new {|a, b| a.date_published <=> b.date_published } 
       end
     end
 
     def parse_feed(feed, entry_sources)
       remote_feed = nil
       begin
-        remote_feed = open(feed['url'], "User-Agent" => @user_agent)
+        remote_feed = open(feed[:url], "User-Agent" => @user_agent)
       rescue Exception => e
-        raise "Exception occurred when opening feed #{feed['name']} at #{feed['url']}:\n\n" + e.to_s
+        raise "Exception occurred when opening feed #{feed[:name]} at #{feed[:url]}:\n\n" + e.to_s
       end
 
       parsed_feed = nil
-      begin
+      begin :test
         parsed_feed = FeedNormalizer::FeedNormalizer.parse remote_feed
       rescue Exception => e
-        raise "Exception occurred when parsing feed #{feed['name']} which was downloaded from #{feed['url']}:\n\n" + e.to_s
+        raise "Exception occurred when parsing feed #{feed[:name]} which was downloaded from #{feed[:url]}:\n\n" + e.to_s
       end
 
-      raise "Feed #{feed['name']} at #{feed['url']} does not appear to be a parsable feed" unless parsed_feed
+      raise "Feed #{feed[:name]} at #{feed[:url]} does not appear to be a parsable feed" unless parsed_feed
 
+      # Sort and truncate the entries if max_entries argument is present
+      if feed[:max_entries]
+        parsed_feed.entries.sort!(&entry_sort_comparator)
+        parsed_feed.entries.slice!(feed[:max_entries], parsed_feed.entries.size - feed[:max_entries])
+      end
+      
+      # Store the entry sources. TODO: Store this information in the entries themselves
       parsed_feed.entries.each { |entry| entry_sources[entry] = feed }
 
       parsed_feed
@@ -169,13 +233,24 @@ EOF
     def load_recipe(recipe)
       recipefile = File.join(@recipes_folder, recipe)
       if ! File.exist? recipefile
-        $stderr.puts "ERROR: Recipe #{recipefile} does not exist"
+        $stderr.puts "ERROR: Recipe file #{recipefile} does not exist"
         exit(1)
       elsif ! File.readable? recipefile
         $stderr.puts "ERROR: Recipe file #{recipefile} is not readable"
         exit(1)
       end
-      YAML::load_file(recipefile)
+
+      recipe = {}
+
+      begin
+        recipe = YAML::load_file(recipefile).recursively_symbolize_keys
+      rescue Exception => e
+        $stderr.puts "ERROR: Problem parsing recipe file #{recipefile}"
+        $stderr.puts e
+        exit(1)
+      end
+
+      recipe
     end
 
     def init_config(configfile)
@@ -187,9 +262,17 @@ EOF
         exit(1)
       end
 
-      config = YAML::load_file(configfile)
+      config = {}
+
+      begin
+        config = YAML::load_file(configfile).recursively_symbolize_keys
+      rescue Exception => e
+        $stderr.puts "ERROR: Problem parsing configuration file #{configfile}"
+        $stderr.puts e
+        exit(1)
+      end
 
-      @recipes_folder = config['recipes_folder']
+      @recipes_folder = config[:recipes_folder]
       if ! File.exist? @recipes_folder
         $stderr.puts "ERROR: Recipes folder #{@recipes_folder} does not exist"
         exit(1)
@@ -198,7 +281,7 @@ EOF
         exit(1)
       end
 
-      @themes_folder = config['themes_folder']
+      @themes_folder = config[:themes_folder]
       if ! File.exist? @themes_folder
         $stderr.puts "ERROR: Themes folder #{@themes_folder} does not exist"
         exit(1)
@@ -207,10 +290,10 @@ EOF
         exit(1)
       end
 
-      @user_agent = config['user_agent'] || 'Audrey 2.0 Feed Aggregator'
-      @sort = config['sort'] || 'reverse-chronological'
+      @user_agent = config[:user_agent] || 'Audrey 2.0 Feed Aggregator'
+      @sort = (config[:sort] || 'reverse_chronological').to_sym
 
-      if @email = config['email']
+      if @email = config[:email]
         gem 'mail', '~> 2.2.5'
         require 'mail'
         # TODO: Check for required/consistent email config

data/test/helper.rb

@@ -0,0 +1,35 @@
+require 'rubygems'
+gem 'test-unit'
+require 'test/unit'
+require 'shoulda'
+require 'mocha'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'bin'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'audrey2'
+
+# The following trick is via http://thinkingdigitally.com/archive/capturing-output-from-puts-in-ruby/
+require 'stringio'
+module Kernel
+  def capture_stderr
+    err = StringIO.new
+    $stderr = err
+    yield
+    return err
+  ensure
+    $stderr = STDERR
+  end
+
+  def capture_stdout
+    out = StringIO.new
+    $stdout = out
+    yield
+    return out
+  ensure
+    $stdout = STDOUT
+  end
+end
+
+class Test::Unit::TestCase
+end

data/test/test_config.rb

@@ -0,0 +1,148 @@
+require 'helper'
+
+class TestConfig < Test::Unit::TestCase
+  context "Initializing an Aggregator" do
+
+    context "without a configfile" do
+      should 'report error and exit' do
+        File.stubs(:exist?).with('configfile').returns(false)
+        err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+        assert_match /ERROR: Configuration file configfile does not exist/, err.string
+      end
+    end
+
+    context "with an unreadable configfile" do
+      should 'report error and exit' do
+        File.stubs(:exist?).with('configfile').returns(true)
+        File.expects(:readable?).with('configfile').returns(false)
+        err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+        assert_match /ERROR: Configuration file configfile is not readable/, err.string
+      end
+    end
+
+    context "with an unparsable configfile" do
+      should 'report error and exit' do
+        File.stubs(:exist?).with('configfile').returns(true)
+        File.stubs(:readable?).with('configfile').returns(true)
+        YAML.expects(:load_file).with('configfile').raises(Exception)
+        err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+        assert_match /ERROR: Problem parsing configuration file configfile/, err.string
+      end
+    end
+
+    context "with a valid configfile" do
+      setup do
+        @config
+        File.stubs(:exist?).with('configfile').returns(true)
+        File.stubs(:readable?).with('configfile').returns(true)
+        YAML.stubs(:load_file).with('configfile').returns({
+          'recipes_folder' => 'recipes_folder',
+          'themes_folder'  => 'themes_folder'
+        })
+      end
+
+      context "without a recipes folder" do
+        should 'report error and exit' do
+          File.expects(:exist?).with('recipes_folder').returns(false)
+          err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+          assert err.string =~ /ERROR: Recipes folder recipes_folder does not exist/
+        end
+      end
+
+      context "with an unreadable recipes folder" do
+        should 'report error and exit' do
+          File.stubs(:exist?).with('recipes_folder').returns(true)
+          File.expects(:readable?).with('recipes_folder').returns(false)
+          err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+          assert err.string =~ /ERROR: Recipes folder recipes_folder is not readable/
+        end
+      end
+
+      context "and recipes folder" do
+        setup do
+          File.stubs(:exist?).with('recipes_folder').returns(true)
+          File.stubs(:readable?).with('recipes_folder').returns(true)
+        end
+
+        context "without a themes folder" do
+          should 'report error and exit' do
+            File.expects(:exist?).with('themes_folder').returns(false)
+            err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+            assert err.string =~ /ERROR: Themes folder themes_folder does not exist/
+          end
+        end
+
+        context "with an unreadable themes folder" do
+          should 'report error and exit' do
+            File.stubs(:exist?).with('themes_folder').returns(true)
+            File.expects(:readable?).with('themes_folder').returns(false)
+            err = capture_stderr { assert_raise(SystemExit) { Audrey2::Aggregator.new('configfile') } }
+            assert err.string =~ /ERROR: Themes folder themes_folder is not readable/
+          end
+        end
+
+        context "and themes folder" do
+          setup do
+            File.stubs(:exist?).with('themes_folder').returns(true)
+            File.stubs(:readable?).with('themes_folder').returns(true)
+            @aggregator = Audrey2::Aggregator.new('configfile')
+          end
+
+          should "return a valid aggregator" do
+            assert_not_nil @aggregator
+          end
+
+          should "use the default user agent" do
+            assert_equal 'Audrey 2.0 Feed Aggregator', @aggregator.instance_variable_get('@user_agent')
+          end
+
+          should "use the default sort" do
+            assert_equal :reverse_chronological, @aggregator.instance_variable_get('@sort')
+          end
+
+          should "not setup email" do
+            assert_nil @aggregator.instance_variable_get('@email')
+          end
+        end
+      end
+    end
+    
+    context "with a complete config" do
+      setup do
+        @config
+        File.stubs(:exist?).with('configfile').returns(true)
+        File.stubs(:readable?).with('configfile').returns(true)        
+        YAML.stubs(:load_file).with('configfile').returns({
+          'recipes_folder' => 'recipes_folder',
+          'themes_folder'  => 'themes_folder',
+          'user_agent'     => 'user_agent',
+          'sort'           => 'sort',
+          'email'          => 'email'
+        })
+        File.stubs(:exist?).with('recipes_folder').returns(true)
+        File.stubs(:readable?).with('recipes_folder').returns(true)  
+        File.stubs(:exist?).with('themes_folder').returns(true)
+        File.stubs(:readable?).with('themes_folder').returns(true) 
+        @aggregator = Audrey2::Aggregator.new('configfile')                     
+      end
+
+      should "return a valid aggregator" do
+        assert_not_nil @aggregator
+      end
+
+      should "use the specified user agent" do
+        assert_equal 'user_agent', @aggregator.instance_variable_get('@user_agent')
+      end
+
+      should "use the specified sort" do
+        assert_equal :sort, @aggregator.instance_variable_get('@sort')
+      end
+
+      # TODO: Fix this one later once email config consistency checking (and testing) are implemented
+      should "setup email" do
+        assert_equal 'email', @aggregator.instance_variable_get('@email')
+      end
+      
+    end    
+  end
+end