feed2imap 1.2.6

data/lib/feed2imap/html2text-parser.rb

@@ -0,0 +1,99 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+require 'feed2imap/sgml-parser'
+
+# this class provides a simple SGML parser that removes HTML tags
+class HTML2TextParser < SGMLParser
+
+  attr_reader :savedata
+
+  def initialize(verbose = false)
+    @savedata = ''
+    @pre = false
+    @href = nil
+    @links = []
+    super(verbose)
+  end
+
+  def handle_data(data)
+    # let's remove all CR
+    data.gsub!(/\n/, '') if not @pre
+ 
+    @savedata << data
+  end
+
+  def unknown_starttag(tag, attrs)
+    case tag
+    when 'p'
+      @savedata << "\n\n"
+    when 'br'
+      @savedata << "\n"
+    when 'b'
+      @savedata << '*'
+    when 'u'
+      @savedata << '_'
+    when 'i'
+      @savedata << '/'
+    when 'pre'
+      @savedata << "\n\n"
+      @pre = true
+    when 'a'
+      # find href in args
+      @href = nil
+      attrs.each do |a|
+        if a[0] == 'href'
+          @href = a[1]
+        end
+      end
+      if @href
+        @links << @href.gsub(/^("|'|)(.*)("|')$/,'\2')
+      end
+    end
+  end
+
+  def close
+    super
+    if @links.length > 0
+      @savedata << "\n\n"
+      @links.each_index do |i|
+        @savedata << "[#{i+1}] #{@links[i]}\n"
+      end
+    end
+  end
+
+  def unknown_endtag(tag)
+    case tag
+    when 'b'
+      @savedata << '*'
+    when 'u'
+      @savedata << '_'
+    when 'i'
+      @savedata << '/'
+    when 'pre'
+      @savedata << "\n\n"
+      @pre = false
+    when 'a'
+      if @href
+        @savedata << "[#{@links.length}]"
+        @href = nil
+      end
+    end
+  end
+end

data/lib/feed2imap/httpfetcher.rb

@@ -0,0 +1,122 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+require 'zlib'
+require 'net/http'
+# get openssl if available
+begin
+require 'net/https'
+rescue LoadError
+end
+require 'uri'
+
+
+# max number of redirections
+MAXREDIR = 5
+
+HTTPDEBUG = false
+
+# Class used to retrieve the feed over HTTP
+class HTTPFetcher
+
+  @timeout = 30 # should be enough for everybody...
+
+  def timeout=(value)
+    @timeout = value
+  end
+
+  def fetcher(baseuri, uri, lastcheck, recursion)
+    proxy_host = nil
+    proxy_port = nil
+    proxy_user = nil
+    proxy_pass = nil
+    if ENV['http_proxy']
+        proxy_uri = URI.parse(ENV['http_proxy'])
+        proxy_host = proxy_uri.host
+        proxy_port = proxy_uri.port
+        proxy_user, proxy_pass = proxy_uri.userinfo.split(/:/) if proxy_uri.userinfo
+    end
+
+    http = Net::HTTP::Proxy(proxy_host,
+                            proxy_port,
+                            proxy_user,
+                            proxy_pass ).new(uri.host, uri.port)
+    http.read_timeout = @timeout
+    http.open_timeout = @timeout
+    if uri.scheme == 'https'
+      http.use_ssl = true 
+      http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    end
+    if defined?(Feed2Imap)
+      useragent = "Feed2Imap v#{Feed2Imap.version} http://home.gna.org/feed2imap/"
+    else
+      useragent = 'Feed2Imap http://home.gna.org/feed2imap/'
+    end
+
+    headers = {
+      'User-Agent' => useragent,
+      'Accept-Encoding' => 'gzip',
+    }
+    if lastcheck != Time::at(0)
+      headers.merge!('If-Modified-Since' => lastcheck.httpdate)
+    end
+    req = Net::HTTP::Get::new(uri.request_uri, headers)
+    if uri.userinfo
+      login, pw = uri.userinfo.split(':')
+      req.basic_auth(login, pw)
+    # workaround. eg. wikini redirects and loses auth info.
+    elsif uri.host == baseuri.host and baseuri.userinfo
+      login, pw = baseuri.userinfo.split(':')
+      req.basic_auth(login, pw)
+    end
+    begin
+      response = http.request(req)
+    rescue Timeout::Error
+      raise "Timeout while fetching #{baseuri.to_s}"
+    end
+    case response
+    when Net::HTTPSuccess
+      case response['Content-Encoding']
+      when 'gzip'
+        return Zlib::GzipReader.new(StringIO.new(response.body)).read
+      else
+        return response.body
+      end
+    when Net::HTTPRedirection
+      # if not modified
+      if Net::HTTPNotModified === response
+        puts "HTTPNotModified on #{uri}" if HTTPDEBUG
+        return nil 
+      end
+      if recursion > 0
+        redir = URI::join(uri.to_s, response['location'])
+        return fetcher(baseuri, redir, lastcheck, recursion - 1)
+      else
+        raise "Too many redirections while fetching #{baseuri.to_s}"
+      end
+    else
+      raise "#{response.code}: #{response.message} while fetching #{baseuri.to_s}"
+    end
+  end
+    
+  def fetch(url, lastcheck)
+    uri = URI::parse(url)
+    return fetcher(uri, uri, lastcheck, MAXREDIR)
+  end
+end

data/lib/feed2imap/imap.rb

@@ -0,0 +1,166 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+# Imap connection handling
+require 'net/imap'
+begin
+require 'openssl'
+rescue LoadError
+end
+require 'cgi'
+require 'uri'
+
+# This class is a container of IMAP accounts.
+# Thanks to it, accounts are re-used : several feeds
+# using the same IMAP account will create only one
+# IMAP connection.
+class ImapAccounts < Hash
+
+  def add_account(uri)
+    u = URI::Generic::build({ :scheme => uri.scheme,
+                              :userinfo => uri.userinfo,
+                              :host => uri.host,
+                              :port => uri.port })
+    if not include?(u)
+      ac = ImapAccount::new(u)
+      self[u] = ac
+    end
+    return self[u]
+  end
+end
+
+# This class is an IMAP account, with the given fd
+# once the connection has been established
+class ImapAccount
+  attr_reader :uri
+
+  @@no_ssl_verify = false
+  def ImapAccount::no_ssl_verify=(v)
+    @@no_ssl_verify = v
+  end
+
+  def initialize(uri)
+    @uri = uri
+    @existing_folders = []
+    self
+  end
+
+  # connects to the IMAP server
+  # raises an exception if it fails
+  def connect
+    port = 143
+    usessl = false
+    if uri.scheme == 'imap'
+      port = 143
+      usessl = false
+    elsif uri.scheme == 'imaps'
+      port = 993
+      usessl = true
+    else
+      raise "Unknown scheme: #{uri.scheme}"
+    end
+    # use given port if port given
+    port = uri.port if uri.port 
+    @connection = Net::IMAP::new(uri.host, port, usessl, nil, !@@no_ssl_verify)
+    user, password = CGI::unescape(uri.userinfo).split(':',2)
+    @connection.login(user, password)
+    self
+  end
+
+  # disconnect from the IMAP server
+  def disconnect
+    if @connection
+      @connection.logout
+      @connection.disconnect
+    end
+  end
+
+  # tests if the folder exists and create it if not
+  def create_folder_if_not_exists(folder)
+    return if @existing_folders.include?(folder)
+    if @connection.list('', folder).nil?
+      @connection.create(folder)
+      @connection.subscribe(folder)
+    end
+    @existing_folders << folder
+  end
+
+  # Put the mail in the given folder
+  # You should check whether the folder exist first.
+  def putmail(folder, mail, date = Time::now)
+    create_folder_if_not_exists(folder)
+    @connection.append(folder, mail.gsub(/\n/, "\r\n"), nil, date)
+  end
+
+  # update a mail
+  def updatemail(folder, mail, id, date = Time::now, reupload_if_updated = true)
+    create_folder_if_not_exists(folder)
+    @connection.select(folder)
+    searchres = @connection.search(['HEADER', 'Message-Id', id])
+    flags = nil
+    if searchres.length > 0
+      # we get the flags from the first result and delete everything
+      flags = @connection.fetch(searchres[0], 'FLAGS')[0].attr['FLAGS']
+      searchres.each { |m| @connection.store(m, "+FLAGS", [:Deleted]) }
+      @connection.expunge
+      flags -= [ :Recent ] # avoids errors with dovecot
+    elsif not reupload_if_updated
+      # mail not present, and we don't want to re-upload it
+      return
+    end
+    @connection.append(folder, mail.gsub(/\n/, "\r\n"), flags, date)
+  end
+
+  # convert to string
+  def to_s
+    u2 = uri.clone
+    u2.password = 'PASSWORD'
+    u2.to_s
+  end
+
+  # remove mails in a folder according to a criteria
+  def cleanup(folder, dryrun = false)
+    puts "-- Considering #{folder}:"
+    @connection.select(folder)
+    a = ['SEEN', 'NOT', 'FLAGGED', 'BEFORE', (Date::today - 3).strftime('%d-%b-%Y')]
+    todel = @connection.search(a)
+    todel.each do |m|
+      f = @connection.fetch(m, "FULL")
+      d = f[0].attr['INTERNALDATE']
+      s = f[0].attr['ENVELOPE'].subject
+      if s =~ /^=\?utf-8\?b\?/
+        s = Base64::decode64(s.gsub(/^=\?utf-8\?b\?(.*)\?=$/, '\1')).force_encoding('utf-8')
+      elsif s =~ /^=\?iso-8859-1\?b\?/
+        s = Base64::decode64(s.gsub(/^=\?iso-8859-1\?b\?(.*)\?=$/, '\1')).force_encoding('iso-8859-1').encode('utf-8')
+      end
+      if dryrun
+        puts "To remove: #{s} (#{d})"
+      else
+        puts "Removing: #{s} (#{d})"
+        @connection.store(m, "+FLAGS", [:Deleted])
+      end
+    end
+    puts "-- Deleted #{todel.length} messages."
+    if not dryrun
+      @connection.expunge
+    end
+    return todel.length
+  end
+end
+

data/lib/feed2imap/itemtomail.rb

@@ -0,0 +1,129 @@
+=begin
+Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server
+Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net>
+
+This file contains classes to parse a feed and store it as a Channel object.
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+=end
+
+require 'rexml/document'
+require 'time'
+require 'rmail'
+require 'feedparser'
+require 'feedparser/text-output'
+require 'feedparser/html-output'
+require 'feed2imap/version'
+require 'base64'
+require 'rmail'
+require 'digest/md5'
+
+class String
+  def needMIME
+    utf8 = false
+    begin
+      self.unpack('U*').each do |c|
+        if c > 127
+          utf8 = true
+          break
+        end
+      end
+    rescue
+      # safe fallback in case of problems
+      utf8 = true
+    end
+    utf8
+  end
+end
+
+def item_to_mail(config, item, id, updated, from = 'Feed2Imap', inline_images = false, wrapto = false)
+  message = RMail::Message::new
+  if item.creator and item.creator != ''
+    if item.creator.include?('@')
+      message.header['From'] = item.creator.chomp
+    else
+      message.header['From'] = "=?utf-8?b?#{Base64::encode64(item.creator.chomp).gsub("\n",'')}?= <#{config.default_email}>"
+    end
+  else
+    message.header['From'] = "=?utf-8?b?#{Base64::encode64(from).gsub("\n",'')}?= <#{config.default_email}>"
+  end
+  message.header['To'] = "=?utf-8?b?#{Base64::encode64(from).gsub("\n",'')}?= <#{config.default_email}>"
+
+  if item.date.nil?
+    message.header['Date'] = Time::new.rfc2822
+  else
+    message.header['Date'] = item.date.rfc2822
+  end
+  message.header['X-Feed2Imap-Version'] = Feed2Imap::VERSION
+  message.header['Message-Id'] = id
+  message.header['X-F2IStatus'] = "Updated" if updated
+  # treat subject. Might need MIME encoding.
+  subj = item.title or (item.date and item.date.to_s) or item.link
+  if subj
+    if subj.needMIME
+      message.header['Subject'] = "=?utf-8?b?#{Base64::encode64(subj).gsub("\n",'')}?="
+    else
+      message.header['Subject'] = subj
+    end
+  end
+  textpart = htmlpart = nil
+  parts = config.parts
+  if parts.include?('text')
+    textpart = parts.size == 1 ? message : RMail::Message::new
+    textpart.header['Content-Type'] = 'text/plain; charset=utf-8; format=flowed'
+    textpart.header['Content-Transfer-Encoding'] = '8bit'
+    textpart.body = item.to_text(true, wrapto, false)
+  end
+  if parts.include?('html')
+    htmlpart = parts.size == 1 ? message : RMail::Message::new
+    htmlpart.header['Content-Type'] = 'text/html; charset=utf-8'
+    htmlpart.header['Content-Transfer-Encoding'] = '8bit'
+    htmlpart.body = item.to_html
+  end
+
+  # inline images as attachments
+  imgs = []
+  if inline_images
+    cids = []
+    htmlpart.body.gsub!(/(<img[^>]+)src="(\S+?\/([^\/]+?\.(png|gif|jpe?g)))"([^>]*>)/i) do |match|
+      # $2 contains url, $3 the image name, $4 the image extension
+      begin
+        fetcher = HTTPFetcher.new
+        image = Base64.encode64(fetcher.fetch($2, Time.at(0)).chomp)
+        "#{$1}src=\"data:image/#{$4};base64,#{image}\"#{$5}"
+      rescue
+        #print "Error while fetching image #{$2}: #{$!}...\n"
+        $& # don't modify on exception
+      end
+    end
+  end
+  if imgs.length > 0
+    message.header.set('Content-Type', 'multipart/related', 'type'=> 'multipart/alternative')
+    texthtml = RMail::Message::new
+    texthtml.header.set('Content-Type', 'multipart/alternative')
+    texthtml.add_part(textpart)
+    texthtml.add_part(htmlpart)
+    message.add_part(texthtml)
+    imgs.each do |i|
+      message.add_part(i)
+    end
+  elsif parts.size != 1
+    message.header['Content-Type'] = 'multipart/alternative'
+    message.add_part(textpart)
+    message.add_part(htmlpart)
+  end
+  return message.to_s
+end
+