feed2imap 1.2.6

data/lib/feed2imap/config.rb

@@ -0,0 +1,167 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+require 'cgi'
+require 'yaml'
+require 'uri'
+require 'feed2imap/imap'
+require 'feed2imap/maildir'
+require 'etc'
+require 'socket'
+require 'set'
+
+# Default cache file
+DEFCACHE = ENV['HOME'] + '/.feed2imap.cache'
+
+# Hostname and login name of the current user
+HOSTNAME = Socket.gethostname
+LOGNAME = Etc.getlogin
+
+# Feed2imap configuration
+class F2IConfig
+  attr_reader :imap_accounts, :cache, :feeds, :dumpdir, :updateddebug, :max_failures, :include_images, :default_email, :hostname, :reupload_if_updated, :parts, :timeout
+
+  # Load the configuration from the IO stream
+  # TODO should do some sanity check on the data read.
+  def initialize(io)
+    @conf = YAML::load(io)
+    @cache = @conf['cache'] || DEFCACHE
+    @dumpdir = @conf['dumpdir'] || nil
+    @conf['feeds'] ||= []
+    @feeds = []
+    @max_failures = (@conf['max-failures'] || 10).to_i
+
+    @updateddebug = false
+    @updateddebug = @conf['debug-updated'] if @conf.has_key?('debug-updated')
+
+    @parts = %w(text html)
+    @parts = Array(@conf['parts']) if @conf.has_key?('parts') && !@conf['parts'].empty?
+    @parts = Set.new(@parts)
+
+    @include_images = true
+    @include_images = @conf['include-images'] if @conf.has_key?('include-images')
+    @parts << 'html' if @include_images && ! @parts.include?('html')
+
+    @reupload_if_updated = true
+    @reupload_if_updated = @conf['reupload-if-updated'] if @conf.has_key?('reupload-if-updated')
+
+    @timeout = if @conf['timeout'] == nil then 30 else @conf['timeout'].to_i end
+
+    @default_email = (@conf['default-email'] || "#{LOGNAME}@#{HOSTNAME}")
+    ImapAccount.no_ssl_verify = (@conf.has_key?('disable-ssl-verification') and @conf['disable-ssl-verification'] == true)
+    @hostname = HOSTNAME # FIXME: should this be configurable as well?
+    @imap_accounts = ImapAccounts::new
+    maildir_account = MaildirAccount::new
+    @conf['feeds'].each do |f|
+      f['name'] = f['name'].to_s
+      if f['disable'].nil?
+        uri = URI::parse(Array(f['target']).join(''))
+        path = CGI::unescape(uri.path)
+        if uri.scheme == 'maildir'
+          @feeds.push(ConfigFeed::new(f, maildir_account, path, self))
+        else
+          # remove leading slash from IMAP mailbox names
+          path = path[1..-1] if path[0,1] == '/'
+          @feeds.push(ConfigFeed::new(f, @imap_accounts.add_account(uri), path, self))
+        end
+      end
+    end
+  end
+
+  def to_s
+    s =  "Your Feed2Imap config :\n"
+    s += "=======================\n"
+    s += "Cache file: #{@cache}\n\n"
+    s += "Imap accounts I'll have to connect to :\n"
+    s += "---------------------------------------\n"
+    @imap_accounts.each_value { |i| s += i.to_s + "\n" }
+    s += "\nFeeds :\n"
+    s +=   "-------\n"
+    i = 1
+    @feeds.each do |f|
+      s += "#{i}. #{f.name}\n"
+      s += "    URL: #{f.url}\n"
+      s += "    IMAP Account: #{f.imapaccount}\n"
+      s += "    Folder: #{f.folder}\n"
+
+      if not f.wrapto
+        s += "    Not wrapped.\n"
+      end
+
+      s += "\n"
+      i += 1
+    end
+    s
+  end
+end
+
+# A configured feed. simple data container.
+class ConfigFeed
+  attr_reader :name, :url, :imapaccount, :folder, :always_new, :execurl, :filter, :ignore_hash, :dumpdir, :wrapto, :include_images, :reupload_if_updated
+  attr_accessor :body
+
+  def initialize(f, imapaccount, folder, f2iconfig)
+    @name = f['name']
+    @url = f['url']
+    @url.sub!(/^feed:/, '') if @url =~ /^feed:/
+    @imapaccount = imapaccount
+    @folder = encode_utf7 folder
+    @freq = f['min-frequency']
+
+    @always_new = false
+    @always_new = f['always-new'] if f.has_key?('always-new')
+
+    @execurl = f['execurl']
+    @filter = f['filter']
+
+    @ignore_hash = false
+    @ignore_hash = f['ignore-hash'] if f.has_key?('ignore-hash')
+
+    @freq = @freq.to_i if @freq
+    @dumpdir = f['dumpdir'] || nil
+    @wrapto = if f['wrapto'] == nil then 72 else f['wrapto'].to_i end
+
+    @include_images = f2iconfig.include_images
+    @include_images = f['include-images'] if f.has_key?('include-images')
+
+    @reupload_if_updated = f2iconfig.reupload_if_updated
+    @reupload_if_updated = f['reupload-if-updated'] if f.has_key?('reupload-if-updated')
+
+  end
+
+  def needfetch(lastcheck)
+    return true if @freq.nil?
+    return (lastcheck + @freq * 3600) < Time::now
+  end
+
+  def encode_utf7(s)
+    if "foo".respond_to?(:force_encoding)
+      return Net::IMAP::encode_utf7 s
+    else
+      # this is a copy of the Net::IMAP::encode_utf7 w/o the force_encoding
+      return s.gsub(/(&)|([^\x20-\x7e]+)/u) {
+        if $1
+          "&-"
+        else
+          base64 = [$&.unpack("U*").pack("n*")].pack("m")
+          "&" + base64.delete("=\n").tr("/", ",") + "-"
+        end }
+    end
+  end
+end

data/lib/feed2imap/feed2imap.rb

@@ -0,0 +1,297 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+F2I_WARNFETCHTIME = 10
+
+require 'feed2imap/version'
+require 'feed2imap/config'
+require 'feed2imap/cache'
+require 'feed2imap/httpfetcher'
+require 'logger'
+require 'thread'
+require 'feedparser'
+require 'feed2imap/itemtomail'
+require 'open3'
+
+class Feed2Imap
+  def Feed2Imap.version
+    return Feed2Imap::VERSION
+  end
+
+  def initialize(verbose, cacherebuild, configfile)
+    @logger = Logger::new(STDOUT)
+    if verbose == :debug
+      @logger.level = Logger::DEBUG
+      require 'pp'
+    elsif verbose == true
+      @logger.level = Logger::INFO
+    else
+      @logger.level = Logger::WARN
+    end
+    @logger.info("Feed2Imap V.#{Feed2Imap::VERSION} started")
+    # reading config
+    @logger.info('Reading configuration file ...')
+    if not File::exist?(configfile)
+      @logger.fatal("Configuration file #{configfile} not found.")
+      exit(1)
+    end
+    if (File::stat(configfile).mode & 044) != 0
+      @logger.warn("Configuration file is readable by other users. It " +
+        "probably contains your password.")
+    end
+    begin
+      File::open(configfile) { 
+        |f| @config = F2IConfig::new(f)
+      }
+    rescue
+      @logger.fatal("Error while reading configuration file, exiting: #{$!}")
+      exit(1)
+    end
+    if @logger.level == Logger::DEBUG
+      @logger.debug("Configuration read:")
+      pp(@config)
+    end
+
+    # init cache
+    @logger.info('Initializing cache ...')
+    @cache = ItemCache::new(@config.updateddebug)
+    if not File::exist?(@config.cache + '.lock')
+      f = File::new(@config.cache + '.lock', 'w')
+      f.close
+    end
+    if File::new(@config.cache + '.lock', 'w').flock(File::LOCK_EX | File::LOCK_NB) == false
+      @logger.fatal("Another instance of feed2imap is already locking the cache file")
+      exit(1)
+    end
+    if not File::exist?(@config.cache) 
+      @logger.warn("Cache file #{@config.cache} not found, using a new one")
+    else
+      File::open(@config.cache) do |f|
+        @cache.load(f)
+      end
+    end
+
+    # connecting all IMAP accounts
+    @logger.info('Connecting to IMAP accounts ...')
+    @config.imap_accounts.each_value do |ac|
+      begin
+        ac.connect
+      rescue
+        @logger.fatal("Error while connecting to #{ac}, exiting: #{$!}")
+        exit(1)
+      end
+    end
+
+    # for each feed, fetch, upload to IMAP and cache
+    @logger.info("Fetching and filtering feeds ...")
+    ths = []
+    mutex = Mutex::new
+    sparefetchers = 16 # max number of fetchers running at the same time.
+    sparefetchers_mutex = Mutex::new
+    sparefetchers_cond = ConditionVariable::new
+    @config.feeds.each do |f|
+      ths << Thread::new(f) do |feed|
+        begin
+          mutex.lock
+          lastcheck = @cache.get_last_check(feed.name) 
+          if feed.needfetch(lastcheck)
+            mutex.unlock
+            sparefetchers_mutex.synchronize do
+              while sparefetchers <= 0
+                sparefetchers_cond.wait(sparefetchers_mutex)
+              end
+              sparefetchers -= 1
+            end
+            fetch_start = Time::now
+            if feed.url
+              fetcher = HTTPFetcher::new
+              fetcher::timeout = @config.timeout
+              s = fetcher::fetch(feed.url, @cache.get_last_check(feed.name))
+            elsif feed.execurl
+              # avoid running more than one command at the same time.
+              # We need it because the called command might not be
+              # thread-safe, and we need to get the right exitcode
+              mutex.lock
+              s = %x{#{feed.execurl}}
+              if $? && $?.exitstatus != 0
+                @logger.warn("Command for #{feed.name} exited with status #{$?.exitstatus} !")
+              end
+              mutex.unlock
+            else
+              @logger.warn("No way to fetch feed #{feed.name} !")
+            end
+            if feed.filter and s != nil
+              # avoid running more than one command at the same time.
+              # We need it because the called command might not be
+              # thread-safe, and we need to get the right exitcode.
+              mutex.lock
+              # hack hack hack, avoid buffering problems
+              begin
+                stdin, stdout, stderr = Open3::popen3(feed.filter)
+                inth = Thread::new do
+                  stdin.puts s
+                  stdin.close
+                end
+                output = nil
+                outh = Thread::new do
+                  output = stdout.read
+                end
+                inth.join
+                outh.join
+                s = output
+                if $? && $?.exitstatus != 0
+                  @logger.warn("Filter command for #{feed.name} exited with status #{$?.exitstatus}. Output might be corrupted !")
+                end
+              ensure
+                mutex.unlock
+              end
+            end
+            if Time::now - fetch_start > F2I_WARNFETCHTIME
+              @logger.info("Fetching feed #{feed.name} took #{(Time::now - fetch_start).to_i}s")
+            end
+            sparefetchers_mutex.synchronize do
+              sparefetchers += 1
+              sparefetchers_cond.signal
+            end
+            mutex.lock
+            feed.body = s
+            @cache.set_last_check(feed.name, Time::now)
+          else
+            @logger.debug("Feed #{feed.name} doesn't need to be checked again for now.")
+          end
+          mutex.unlock
+          # dump if requested
+          if @config.dumpdir
+            mutex.synchronize do
+              if feed.body
+                fname = @config.dumpdir + '/' + feed.name + '-' + Time::now.xmlschema
+                File::open(fname, 'w') { |file| file.puts feed.body }
+              end
+            end
+          end
+          # dump this feed if requested
+          if feed.dumpdir
+            mutex.synchronize do
+              if feed.body
+                fname = feed.dumpdir + '/' + feed.name + '-' + Time::now.xmlschema
+                File::open(fname, 'w') { |file| file.puts feed.body }
+              end
+            end
+          end
+        rescue Timeout::Error
+          mutex.synchronize do
+            n = @cache.fetch_failed(feed.name)
+            m = "Timeout::Error while fetching #{feed.url}: #{$!} (failed #{n} times)"
+            if n > @config.max_failures
+              @logger.fatal(m)
+            else
+              @logger.info(m)
+            end
+          end
+        rescue
+          mutex.synchronize do
+            n = @cache.fetch_failed(feed.name)
+            m = "Error while fetching #{feed.url}: #{$!} (failed #{n} times)"
+            if n > @config.max_failures
+              @logger.fatal(m)
+            else
+              @logger.info(m)
+            end
+          end
+        end
+      end
+    end
+    ths.each { |t| t.join }
+    @logger.info("Parsing and uploading ...")
+    @config.feeds.each do |f|
+      if f.body.nil? # means 304
+        @logger.debug("Feed #{f.name} did not change.")
+        next
+      end
+      begin
+        feed = FeedParser::Feed::new(f.body.force_encoding('UTF-8'), f.url)
+      rescue Exception
+        n = @cache.parse_failed(f.name)
+        m = "Error while parsing #{f.name}: #{$!} (failed #{n} times)"
+        if n > @config.max_failures
+          @logger.fatal(m)
+        else
+          @logger.info(m)
+        end
+        next
+      end
+      begin
+        newitems, updateditems = @cache.get_new_items(f.name, feed.items, f.always_new, f.ignore_hash)
+      rescue
+        @logger.fatal("Exception caught when selecting new items for #{f.name}: #{$!}")
+        puts $!.backtrace
+        next
+      end
+      @logger.info("#{f.name}: #{newitems.length} new items, #{updateditems.length} updated items.") if newitems.length > 0 or updateditems.length > 0 or @logger.level == Logger::DEBUG
+      begin
+        if !cacherebuild
+          fn = f.name.gsub(/[^0-9A-Za-z]/,'')
+          updateditems.each do |i|
+            id = "<#{fn}-#{i.cacheditem.index}@#{@config.hostname}>"
+            email = item_to_mail(@config, i, id, true, f.name, f.include_images, f.wrapto)
+            f.imapaccount.updatemail(f.folder, email,
+                                     id, i.date || Time::new, f.reupload_if_updated)
+          end
+          # reverse is needed to upload older items first (fixes gna#8986)
+          newitems.reverse.each do |i|
+            id = "<#{fn}-#{i.cacheditem.index}@#{@config.hostname}>"
+            email = item_to_mail(@config, i, id, false, f.name, f.include_images, f.wrapto)
+            f.imapaccount.putmail(f.folder, email, i.date || Time::new)
+          end
+        end
+      rescue
+        @logger.fatal("Exception caught while uploading mail to #{f.folder}: #{$!}")
+        puts $!.backtrace
+        @logger.fatal("We can't recover from IMAP errors, so we are exiting.")
+        exit(1)
+      end
+      begin
+        @cache.commit_cache(f.name)
+      rescue
+        @logger.fatal("Exception caught while updating cache for #{f.name}: #{$!}")
+        next
+      end
+    end
+    @logger.info("Finished. Saving cache ...")
+    begin
+      File::open("#{@config.cache}.new", 'w') { |f| @cache.save(f) }
+    rescue
+      @logger.fatal("Exception caught while writing new cache to #{@config.cache}.new: #{$!}")
+    end
+    begin
+      File::rename("#{@config.cache}.new", @config.cache)
+    rescue
+      @logger.fatal("Exception caught while renaming #{@config.cache}.new to #{@config.cache}: #{$!}")
+    end
+    @logger.info("Closing IMAP connections ...")
+    @config.imap_accounts.each_value do |ac|
+      begin
+        ac.disconnect
+      rescue
+        # servers tend to cause an exception to be raised here, hence the INFO level.
+        @logger.info("Exception caught while closing connection to #{ac.to_s}: #{$!}")
+      end
+    end
+  end
+end