rss2mail 0.0.1

data/README

@@ -0,0 +1,41 @@
+= rss2mail - Send RSS feeds as e-mail
+
+== VERSION
+
+This documentation refers to rss2mail version 0.0.1
+
+
+== DESCRIPTION
+
+Sends new entries for configured RSS feeds as e-mail to any number of
+recipients. See <tt>example/</tt> directory for a sample configuration file.
+
+
+== LINKS
+
+<b></b>
+Documentation::     <http://rss2mail.rubyforge.org/>
+Source code::       <http://github.com/blackwinter/rss2mail>
+Rubyforge project:: <http://rubyforge.org/projects/rss2mail>
+
+
+== AUTHORS
+
+* Jens Wille <mailto:ww@blackwinter.de>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2007-2008 Jens Wille
+
+rss2mail is free software: you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free Software
+Foundation, either version 3 of the License, or (at your option) any later
+version.
+
+rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along with
+rss2mail. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,26 @@
+require %q{lib/rss2mail/version}
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    :rubyforge => {
+      :project  => %q{rss2mail},
+      :package  => %q{rss2mail},
+      :rdoc_dir => nil
+    },
+
+    :gem => {
+      :version      => RSS2Mail::VERSION,
+      :summary      => %q{Send RSS feeds as e-mail},
+      :homepage     => %q{http://rss2mail.rubyforge.org/},
+      :files        => FileList['lib/**/*.rb', 'bin/*'].to_a,
+      :extra_files  => FileList['[A-Z]*', 'example/*'].to_a,
+      :dependencies => %w[simple-rss hpricot unidecode ruby-nuggets]
+    }
+  }}
+rescue LoadError
+  abort "Please install the 'hen' gem first."
+end
+
+### Place your custom Rake tasks here.

data/bin/rss2mail

@@ -0,0 +1,104 @@
+#! /usr/bin/ruby
+
+#--
+###############################################################################
+#                                                                             #
+# rss2mail -- Send RSS feeds as e-mail                                        #
+#                                                                             #
+# Copyright (C) 2007-2008 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <ww@blackwinter.de>                                          #
+#                                                                             #
+# rss2mail is free software; you can redistribute it and/or modify it under   #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with rss2mail. If not, see <http://www.gnu.org/licenses/>.                  #
+#                                                                             #
+###############################################################################
+#++
+
+require 'optparse'
+require 'yaml'
+
+base = File.join(File.dirname(__FILE__), '..')
+$: << File.join(base, 'lib')
+
+require 'rss2mail'
+
+USAGE = "Usage: #{$0} [options] <target>"
+abort USAGE if ARGV.empty?
+
+options = {
+  :files   => nil,
+  :reload  => false,
+  :verbose => false
+}
+
+OptionParser.new { |opts|
+  opts.banner = USAGE
+  opts.separator ''
+
+  opts.on('-d', '--directory DIRECTORY', 'Process all feeds in directory') { |d|
+    abort "Not a directory: #{d}"      unless File.directory?(d)
+    abort "Can't read directory: #{d}" unless File.readable?(d)
+
+    options[:files] = Dir[File.join(d, '*.yaml')]
+  }
+
+  opts.on('-r', '--reload', 'Reload feeds') {
+    options[:reload] = true
+  }
+
+  opts.on('-v', '--verbose', 'Be verbose') {
+    options[:verbose] = true
+  }
+}.parse!
+
+if target = ARGV.shift
+  target = target.to_sym
+else
+  abort "No feeds target given\n#{USAGE}"
+end
+
+templates = Hash.new { |h, k|
+  t = File.join(base, 'templates', "#{k}.erb")
+
+  begin
+    h[k] = File.read(t)
+  rescue Errno::ENOENT
+    # silently ignore
+  end
+}
+
+feeds_files = options.delete(:files) || [File.join(base, 'feeds.yaml')]
+feeds_files.each { |feeds_file|
+  feeds = begin
+    YAML.load_file(feeds_file)
+  rescue Errno::ENOENT
+    warn "Feeds file not found: #{feeds_file}"
+    next
+  end
+
+  unless target_feeds = feeds[target]
+    warn "Feeds target not found: #{target} (in #{feeds_file})"
+    next
+  end
+
+  target_feeds.each { |feed|
+    RSS2Mail::Feed.new(feed, options).deliver(templates) unless feed[:skip]
+  }
+
+  # write updated feed information
+  File.open(feeds_file, 'w') { |file|
+    YAML.dump(feeds, file)
+  }
+}

data/example/feeds.yaml

@@ -0,0 +1,30 @@
+--- 
+:"twice-a-day": 
+- :url: http://feeds.feedburner.com/Bildblog
+  :to: my.secret@e.mail
+  :title: BILDblog
+- :url: http://newsfeed.zeit.de/
+  :to: my.secret@e.mail
+  :title: DIE ZEIT
+- :url: http://www.spiegel.de/schlagzeilen/rss/index.xml
+  :to: my.secret@e.mail
+  :title: SPIEGEL ONLINE
+:hourly: 
+- :url: http://www.heise.de/newsticker/heise-atom.xml
+  :to: my.secret@e.mail
+  :title: heise online News
+  :body: heisetext
+  :body_encoding: ISO-8859-1
+- :url: http://log.netbib.de/feed/
+  :to: my.secret@e.mail
+  :title: netbib weblog
+:daily: 
+- :url: http://aktuell.de.selfhtml.org/weblog/rss-feed
+  :to: my.secret@e.mail
+  :title: SELFHTML Aktuell Weblog
+- :url: http://rss.slashdot.org/Slashdot/slashdotLinux
+  :to: my.secret@e.mail
+  :title: "Slashdot: Linux"
+- :url: http://rss.slashdot.org/Slashdot/slashdotBookReviews
+  :to: my.secret@e.mail
+  :title: "Slashdot: Book Reviews"

data/lib/rss2mail/feed.rb

@@ -0,0 +1,225 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of rss2mail, the RSS to e-mail forwarder.                       #
+#                                                                             #
+# Copyright (C) 2007-2008 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <ww@blackwinter.de>                                          #
+#                                                                             #
+# rss2mail is free software; you can redistribute it and/or modify it under   #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with rss2mail. If not, see <http://www.gnu.org/licenses/>.                  #
+#                                                                             #
+###############################################################################
+#++
+
+require 'open-uri'
+require 'erb'
+
+require 'rubygems'
+require 'hpricot'
+require 'unidecode'
+require 'nuggets/util/i18n'
+require 'nuggets/string/evaluate'
+
+require 'rss2mail/rss'
+
+module RSS2Mail
+
+  class Feed
+
+    SUBSTITUTIONS = {
+      '–'     => '--',
+      '«'     => '<<',
+      '&amp;' => '&'
+    }
+
+    SUBSTITUTIONS_RE = %r{Regexp.union(*SUBSTITUTIONS.keys)}o
+
+    TAGS_TO_KEEP = %w[a p br h1 h2 h3 h4]
+
+    attr_reader :feed, :verbose, :reload, :simple, :updated, :content, :rss
+
+    def initialize(feed, options = {})
+      raise TypeError, "Hash expected, got #{feed.class}" unless feed.is_a?(Hash)
+
+      @feed    = feed
+      @simple  = feed[:simple]
+      @updated = feed[:updated]
+
+      @verbose = options[:verbose]
+      @reload  = options[:reload]
+
+      required = [:url, :to, :title]
+      required.delete_if { |i| feed.has_key?(i) }
+
+      raise ArgumentError, "feed incomplete: #{required.join(', ')} missing" unless required.empty?
+    end
+
+    def deliver(templates)
+      unless get && parse
+        warn "[#{feed[:title]}] Nothing to send" if verbose
+        return
+      end
+
+      if rss.items.empty?
+        warn "[#{feed[:title]}] No new items" if verbose
+        return
+      end
+
+      to = [*feed[:to]]
+      if to.empty?
+        warn "[#{feed[:title]}] No one to send to" if verbose
+        return
+      end
+
+      feed_title   = feed[:title]
+      content_type = feed[:content_type] || 'text/html'
+      encoding     = feed[:encoding]     || 'UTF-8'
+
+      feed[:sent] ||= []
+
+      content_type_header = "Content-type: #{content_type}; charset=#{encoding}"
+
+      unless template = templates[content_type[/\/(.*)/, 1]]
+        warn "[#{feed[:title]}] Template not found: #{content_type}" if verbose
+        return
+      end
+
+      cmd = [
+        '/usr/bin/mail',
+        '-e',
+        "-a '#{content_type_header}'",
+        "-a 'From: rss2mail@blackwinter.de'",
+        "-s '[#{feed_title}] \#{subject}'",
+        *to
+      ].join(' ')
+
+      sent = 0
+
+      rss.items.each { |item|
+        title       = item.title
+        link        = item.link
+        description = item.description
+        date        = item.date
+        author      = item.author
+
+        if description && feed[:unescape_html]
+          description.gsub!(/&lt;/, '<')
+          description.gsub!(/&gt;/, '>')
+        end
+
+        if tag = feed[:body]
+          body = case tag
+            when true: open(link).read
+            else       Hpricot(open(link)).at(tag).to_s
+          end.gsub(/<\/?(.*?)>/) { |m|
+            m if TAGS_TO_KEEP.include?($1.split.first.downcase)
+          }.gsub(/<a\s+href=['"](?!http:).*?>(.*?)<\/a>/mi, '\1')
+
+          if body_encoding = feed[:body_encoding]
+            body = Iconv.conv('UTF-8', body_encoding, body)
+          end
+        end
+
+        subject = title ? clean_subject(title) : 'NO TITLE'
+
+        _cmd = cmd.evaluate(binding)
+
+        begin
+          IO.popen(_cmd, 'w') { |mail| mail.puts ERB.new(template).result(binding) }
+          feed[:sent] << link
+          sent += 1
+        rescue Errno::EPIPE => err
+          warn "[#{feed[:title]}] Error while sending mail (#{err.class}): #{_cmd}"
+        end
+      }
+
+      # only keep the last 100 entries
+      feed[:sent].slice!(0...-100)
+
+      warn "[#{feed[:title]}] #{sent} items sent" if verbose
+      sent
+    end
+
+    private
+
+    def get(reload = reload)
+      if reload
+        @content = nil
+        conditions = {}
+      else
+        conditions = case
+          when etag  = feed[:etag]:  { 'If-None-Match'     => etag  }
+          when mtime = feed[:mtime]: { 'If-Modified-Since' => mtime }
+          else                         {}
+        end
+      end
+
+      begin
+        open(feed[:url], conditions) { |uri|
+          case
+            when etag  = uri.meta['etag']:  feed[:etag]    = etag
+            when mtime = uri.last_modified: feed[:mtime]   = mtime.rfc822
+            else                            feed[:updated] = Time.now
+          end
+
+          @content ||= uri.read
+        }
+      rescue OpenURI::HTTPError
+        warn "[#{feed[:title]}] Feed not found or unchanged" if verbose
+      rescue Timeout::Error, Errno::ETIMEDOUT, Errno::ECONNRESET => err
+        warn "[#{feed[:title]}] Error while getting feed: #{err} (#{err.class})"
+      end
+
+      @content
+    end
+
+    def parse(reload = reload)
+      @rss = nil if reload
+
+      if content && @rss ||= begin
+        RSS2Mail::RSS.new(content, simple)
+      rescue SimpleRSSError => err
+        warn "[#{feed[:title]}] Error while parsing feed: #{err} (#{err.class})"
+      end
+        sent = feed[:sent]
+
+        unless reload
+          @rss.items.delete_if { |item|
+            if updated && date = item.date
+              date <= updated
+            else
+              sent && sent.include?(item.link)
+            end
+          }
+        end
+      else
+        warn "[#{feed[:title]}] Nothing to parse" if verbose
+      end
+
+      @rss
+    end
+
+    def clean_subject(string)
+      string.
+        replace_diacritics.
+        gsub(SUBSTITUTIONS_RE) { |m| SUBSTITUTIONS[m] }.
+        to_ascii.
+        gsub(/'/, "'\\\\''")
+    end
+
+  end
+
+end

data/lib/rss2mail/rss.rb

@@ -0,0 +1,95 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of rss2mail, the RSS to e-mail forwarder.                       #
+#                                                                             #
+# Copyright (C) 2007-2008 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <ww@blackwinter.de>                                          #
+#                                                                             #
+# rss2mail is free software; you can redistribute it and/or modify it under   #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with rss2mail. If not, see <http://www.gnu.org/licenses/>.                  #
+#                                                                             #
+###############################################################################
+#++
+
+require 'rss'
+
+require 'rubygems'
+require 'simple-rss'
+
+module RSS2Mail
+
+  class RSS
+
+    attr_reader :content, :rss
+
+    def initialize(content, simple = false)
+      @content = content
+      @simple  = simple
+
+      @rss = simple ? simple_parse : parse
+    end
+
+    def simple?
+      @simple
+    end
+
+    def items
+      @items ||= rss.items.map { |item| Item.new(item) }
+    end
+
+    def parse
+      ::RSS::Parser.parse(content, false) || simple_parse
+    end
+
+    def simple_parse
+      SimpleRSS.parse(content)
+    end
+
+    class Item
+
+      ALIASES = {
+        :title       => %w[],
+        :link        => %w[],
+        :description => %w[summary content],
+        :date        => %w[pubDate updated],
+        :author      => %w[dc_creator]
+      }
+
+      def initialize(item)
+        @item = item
+      end
+
+      def method_missing(method, *args, &block)
+        if aliases = ALIASES[method]
+          [method, *aliases].each { |name|
+            begin
+              res = @item.send(name)
+              return res if res
+            rescue NoMethodError
+            end
+          }
+
+          nil
+        else
+          super
+        end
+      end
+
+    end
+
+  end
+
+end

data/lib/rss2mail/util.rb

@@ -0,0 +1,65 @@
+#--
+###############################################################################
+#                                                                             #
+# A component of rss2mail, the RSS to e-mail forwarder.                       #
+#                                                                             #
+# Copyright (C) 2007-2008 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <ww@blackwinter.de>                                          #
+#                                                                             #
+# rss2mail is free software; you can redistribute it and/or modify it under   #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with rss2mail. If not, see <http://www.gnu.org/licenses/>.                  #
+#                                                                             #
+###############################################################################
+#++
+
+require 'open-uri'
+require 'uri'
+
+require 'rubygems'
+require 'hpricot'
+
+module RSS2Mail
+
+  module Util
+
+    extend self
+
+    FEED_REGEXP = %r{\Aapplication/(?:atom|rss)\+xml\z}io
+
+    # cf. <http://www.rssboard.org/rss-autodiscovery>
+    def discover_feed(url, or_self = false)
+      default = or_self ? url : nil
+
+      unless url.nil? || url.empty? || url == 'about:blank'
+        doc = Hpricot(open(url))
+
+        if feed_element = doc.search('//link[@rel="alternate"').find { |link|
+          link[:type] =~ FEED_REGEXP
+        }
+          if feed_href = feed_element[:href]
+            return feed_href if feed_href =~ URI.regexp(%w[http https])
+
+            base_href = doc.at('base')[:href] rescue url
+            return URI.join(base_href, feed_href).to_s
+          end
+        end
+      end
+
+      default
+    end
+
+  end
+
+end

data/lib/rss2mail/version.rb

@@ -0,0 +1,27 @@
+module RSS2Mail
+
+  module Version
+
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 1
+
+    class << self
+
+      # Returns array representation.
+      def to_a
+        [MAJOR, MINOR, TINY]
+      end
+
+      # Short-cut for version string.
+      def to_s
+        to_a.join('.')
+      end
+
+    end
+
+  end
+
+  VERSION = Version.to_s
+
+end

data/lib/rss2mail.rb

@@ -0,0 +1,30 @@
+#--
+###############################################################################
+#                                                                             #
+# rss2mail -- Send RSS feeds as e-mail                                        #
+#                                                                             #
+# Copyright (C) 2007-2008 Jens Wille                                          #
+#                                                                             #
+# Authors:                                                                    #
+#     Jens Wille <ww@blackwinter.de>                                          #
+#                                                                             #
+# rss2mail is free software; you can redistribute it and/or modify it under   #
+# the terms of the GNU General Public License as published by the Free        #
+# Software Foundation; either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# rss2mail is distributed in the hope that it will be useful, but WITHOUT ANY #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more       #
+# details.                                                                    #
+#                                                                             #
+# You should have received a copy of the GNU General Public License along     #
+# with rss2mail. If not, see <http://www.gnu.org/licenses/>.                  #
+#                                                                             #
+###############################################################################
+#++
+
+require 'rss2mail/feed'
+
+module RSS2Mail
+end