imap-feeder 0.1.0

data/Rakefile

@@ -0,0 +1,4 @@
+require 'config/requirements'
+require 'config/hoe' # setup Hoe + all gem configuration
+
+Dir['tasks/**/*.rake'].each { |rake| load rake }

data/TODO.txt

@@ -0,0 +1,11 @@
+Ideas for further development:
+* Adding new feeds through rssimap, without manually editing the file.
+* HTML-Mails (really?)
+* make the position of the link to the original configurable
+* embed images (e.g. for dilbert)
+
+* user-hooks that can interact with the feed processing:
+ * to ignore items
+ * custom body, e.g. fetch content with hpricot
+ * code directly in the feedy.yml
+  * or an external file, with an ID to identify the feed

data/bin/imap-feeder

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'imap-feeder'
+require 'imap-feeder/createconfigparser'
+require 'imap-feeder/imapfeederconfig'
+require 'imap-feeder/fakeserver.rb'
+
+opts = CreateConfigParser.parse(ARGV)
+
+if opts.create
+  $stdout.puts "Creating new configuration."
+  ImapFeederConfig.create(opts.create_file, opts.folder)
+  $stdout.puts "Please edit your settings.rb file now."
+
+elsif opts.check
+  $stdout.puts "Checking configuration #{opts.check}"
+  $log = Logger.new($stdout)
+  ImapFeederConfig.check(File.open(opts.check))
+
+elsif ARGV.first and load(ARGV.first)
+  configuration = [$host, $user, $pass, $temp, $config]
+  if not configuration.all?
+    $stderr.puts "Your settings are incomplete, please review settings.rb:"
+    $stderr.puts configuration.inspect
+    exit 1
+  end
+
+  if File.exist? $running_instance
+    $stderr.puts "Already running (#{$running_instance})! Aborting..."
+    exit 1
+  else
+    File.new($running_instance, "w")
+  end
+
+  store = MessageStore.new($temp)
+
+  server_options = {
+    :host => $host,
+    :user => $user,
+    :pass => $pass,
+    :port => $port || "143",
+    :use_ssl => $use_ssl || false
+  }
+
+  if opts.pretend
+    @server = FakeServer.new(server_options)
+    def store.save
+    end
+    $log = Logger.new(STDOUT)
+  else
+    begin
+      @server = Server.new(server_options)
+    rescue Exception, Errno::ECONNRESET => e
+      $log.error "Could not open initial connection to server: #{e}"
+    end
+  end
+
+  config = File.open($config)
+  imapfeeder = ImapFeeder.new(@server, store, config)
+  imapfeeder.run
+  @server.disconnect
+
+  File.delete $running_instance
+
+else
+  $stdout.puts "No options given, maybe you need --help?"
+end
+

data/config/hoe.rb

@@ -0,0 +1,77 @@
+require 'imap-feeder/version'
+
+AUTHOR = 'Mirko Stocker'  # can also be an array of Authors
+EMAIL = "me@misto.ch"
+DESCRIPTION = "Imap-feeder pushes your RSS and Atom feeds to an IMAP server."
+GEM_NAME = 'imap-feeder' # what ppl will type to install your gem
+RUBYFORGE_PROJECT = 'imap-feeder' # The unix name for your project
+HOMEPATH = "http://#{RUBYFORGE_PROJECT}.rubyforge.org"
+DOWNLOAD_PATH = "http://rubyforge.org/projects/#{RUBYFORGE_PROJECT}"
+
+@config_file = "~/.rubyforge/user-config.yml"
+@config = nil
+RUBYFORGE_USERNAME = "unknown"
+def rubyforge_username
+  unless @config
+    begin
+      @config = YAML.load(File.read(File.expand_path(@config_file)))
+    rescue
+      puts <<-EOS
+ERROR: No rubyforge config file found: #{@config_file}
+Run 'rubyforge setup' to prepare your env for access to Rubyforge
+ - See http://newgem.rubyforge.org/rubyforge.html for more details
+      EOS
+      exit
+    end
+  end
+  RUBYFORGE_USERNAME.replace @config["username"]
+end
+
+
+REV = nil 
+# UNCOMMENT IF REQUIRED: 
+# REV = `svn info`.each {|line| if line =~ /^Revision:/ then k,v = line.split(': '); break v.chomp; else next; end} rescue nil
+VERS = ImapFeeder::VERSION::STRING + (REV ? ".#{REV}" : "")
+RDOC_OPTS = ['--quiet', '--title', 'imap-feeder documentation',
+    "--opname", "index.html",
+    "--line-numbers", 
+    "--main", "README",
+    "--inline-source"]
+
+class Hoe
+  def extra_deps 
+    @extra_deps.reject! { |x| Array(x).first == 'hoe' } 
+    @extra_deps
+  end 
+end
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+hoe = Hoe.new(GEM_NAME, VERS) do |p|
+  p.developer(AUTHOR, EMAIL)
+  p.description = DESCRIPTION
+  p.summary = DESCRIPTION
+  p.url = HOMEPATH
+  p.rubyforge_name = RUBYFORGE_PROJECT if RUBYFORGE_PROJECT
+  p.test_globs = ["test/**/test_*.rb"]
+  p.clean_globs |= ['**/.*.sw?', '*.gem', '.config', '**/.DS_Store']  #An array of file patterns to delete on clean.
+  
+  # == Optional
+  p.changes = p.paragraphs_of("History.txt", 0..1).join("\n\n")
+
+  p.extra_deps = [
+    ['actionmailer', '>= 1.3.2'],
+    ['hpricot',      '>= 0.5'],
+    ['htmlentities', '>= 3.0.1'],
+    ['tidy',         '>= 1.1.2'],
+    ['simple-rss',   '>= 1.1']
+  ]
+
+  #p.spec_extras = {}    # A hash of extra values to set in the gemspec.
+
+end
+
+CHANGES = hoe.paragraphs_of('History.txt', 0..1).join("\\n\\n")
+PATH    = (RUBYFORGE_PROJECT == GEM_NAME) ? RUBYFORGE_PROJECT : "#{RUBYFORGE_PROJECT}/#{GEM_NAME}"
+hoe.remote_rdoc_dir = File.join(PATH.gsub(/^#{RUBYFORGE_PROJECT}\/?/,''), 'rdoc')
+hoe.rsync_args = '-av --delete --ignore-errors'

data/config/requirements.rb

@@ -0,0 +1,17 @@
+require 'fileutils'
+include FileUtils
+
+require 'rubygems'
+%w[rake hoe newgem rubigen].each do |req_gem|
+  begin
+    require req_gem
+  rescue LoadError
+    puts "This Rakefile requires the '#{req_gem}' RubyGem."
+    puts "Installation: gem install #{req_gem} -y"
+    exit
+  end
+end
+
+$:.unshift(File.join(File.dirname(__FILE__), %w[.. lib]))
+
+require 'imap-feeder'

data/lib/imap-feeder.rb

@@ -0,0 +1,87 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'imap-feeder/server'
+require 'imap-feeder/message'
+require 'imap-feeder/feedreader'
+require 'imap-feeder/messagestore'
+
+$KCODE="U"
+
+class ImapFeeder
+
+  def initialize(server, store, config)
+    @server = server
+    @store = store
+    @config = config
+  end
+
+  def run
+    $log.info "Started"
+    feeds = YAML.load(@config)
+
+    feeds.each do |feed|
+      path = feed['feed']['path']
+      $log.info "Processing #{path}"
+
+      create_folder(path) unless check_folder_exists(path)
+
+      begin
+        url = feed['feed']['url']
+        reader = FeedReader.new(url)
+      rescue OpenURI::HTTPError => e
+        $log.warn "Error retrieving #{url}: #{e.message}"
+        next
+      rescue Exception => e
+        $log.error "Unexpected error while retrieving #{path}: #{e.message}"
+        next
+      end
+
+      archive = get_archived(path)
+      messages = reader.get_new(archive)
+
+      unless messages.empty?
+        $log.debug "already processed messages: '#{archive.join("', '")}'"
+        send_messages(messages, path, reader.number_of_entries)
+      end
+    end
+
+    $log.info "Finished"
+  end
+
+  private
+
+  def send_messages messages, path, number_of_entries
+    $log.info "#{messages.size} new message(s)"
+    messages.each do |msg|
+      send_message(msg, path)
+    end
+    message_sent(messages, path, number_of_entries)
+  end
+
+  def message_sent(messages, path, number_of_entries)
+    identifiers = messages.collect do |msg|
+      msg.generate_identifier
+    end
+
+    @store.add_new(path, identifiers, number_of_entries)
+    @store.save
+  end
+
+  def send_message(msg, complete_path)
+    @server.send(msg, complete_path)
+    $log.info "Found in #{complete_path.split(".").last}: #{msg.generate_identifier}"
+  end
+
+  def get_archived(path)
+    @store.get_archived path
+  end
+
+  def create_folder(path)
+    $log.info "Creating #{path}"
+    @server.create_folder path
+  end
+
+  def check_folder_exists(path)
+    @server.has_folder? path
+  end
+end

data/lib/imap-feeder/createconfigparser.rb

@@ -0,0 +1,40 @@
+require 'optparse'
+require 'ostruct'
+
+class CreateConfigParser
+  def self.parse(args)
+    options = OpenStruct.new
+
+    opts = OptionParser.new do |opts|
+      opts.banner = <<EOF
+Usage: #{$0} [options] to create or check your configuration
+or     #{$0} SETTINGS_FILE to run the script
+EOF
+
+      opts.on("-c", "--check-settings SETTINGS_FILE",
+              "Check an existing configuration") do |file|
+        options.check = file
+      end
+
+      opts.on("-r", "--imap-root [FOLDER]",
+              "Use this as the root folder for all feeds") do |folder|
+        options.folder = folder
+      end
+
+      opts.on("-n", "--new-config [OPML_FILE]",
+              "Create a new configuration") do |file|
+        options.create = true
+        options.create_file = file
+      end
+
+      opts.on("-p", "--pretend",
+              "Don't do anything, just pretend. " +
+              "Prints new items to the console.") do |pretend|
+        options.pretend = pretend
+      end
+    end
+    opts.parse!(args)
+
+    options
+  end
+end

data/lib/imap-feeder/fakeserver.rb

@@ -0,0 +1,23 @@
+class FakeServer
+
+  attr_reader :connected
+
+  def initialize(params)
+    puts "Would connect to #{params[:host]}:#{params[:port]}, with ssl? #{params[:use_ssl]}"
+    puts "Would login with #{params[:user]}/#{params[:pass]}"
+  end
+
+  def disconnect
+  end
+
+  def send(message, folder="INBOX")
+    puts "Send message to folder #{folder}: #{message.title}"
+  end
+
+  def create_folder(folder)
+  end
+
+  def has_folder?(folder)
+    true
+  end
+end

data/lib/imap-feeder/feedfolder.rb

@@ -0,0 +1,23 @@
+FeedUrl = Struct.new(:name, :url)
+
+class FeedFolder
+  attr_accessor :name, :children, :urls
+    
+  def initialize(name)
+    @name = name
+    @children = []
+    @urls = []
+  end
+  
+  def add_sub(child)
+    @children << child
+  end
+  
+  def add_url(url)
+    @urls << url
+  end
+  
+  def [](index)
+    @children[index]
+  end
+end

data/lib/imap-feeder/feedreader.rb

@@ -0,0 +1,81 @@
+require 'tempfile'
+require 'open-uri'
+require 'simple-rss'
+require 'htmlentities'
+require 'iconv'
+
+$KCODE="U"
+
+# Overwrite SimpleRSS::unescape because of an open bug(#10852)
+class SimpleRSS
+  def unescape(content)
+    content.gsub(/(<!\[CDATA\[|\]\]>)/,'').strip
+  end
+end
+
+class FeedReader
+  attr_reader :messages
+
+  def initialize(feed_url)
+    @feed_url = feed_url
+    @feed = SimpleRSS.parse(open(feed_url))
+
+    @encoding = @feed.source[/encoding=["'](.*?)["']/, 1]
+    if not @encoding
+      $log.warn "No encoding found for #{feed_url}, defaulting to UTF-8."
+      @encoding = "UTF-8"
+    end
+  end
+
+  def conv(str)
+    Iconv.iconv("UTF-8", @encoding, str).first
+  rescue Iconv::IllegalSequence => e
+    $log.error "IConv reports an IllegalSequence: #{e.message} from #{str}"
+    return str
+  end
+
+  def number_of_entries
+    @feed.entries.size
+  end
+
+  def get_new(archive)
+    return [] if not @feed
+
+    archive ||= []
+    if archive.include?("")
+      $log.warn "Title is empty, that should never happen! Aborting #{@feed_url}."
+      return []
+    end
+
+    messages = []
+    @feed.entries.each do |item|
+
+      body = conv(item.content_encoded || item.content ||
+                  item.summary || item.description)
+      message = Message.new(
+        :title => conv(item.title),
+        :time => time_from(item),
+        :body => body,
+        :from => conv(item.author),
+        :url => conv(item.link)
+      )
+
+      item_identifier = message.generate_identifier
+
+      if archive.include? item_identifier
+        short_name = message.title[0..30]
+        short_name << "…" if message.title.length > 30
+        $log.debug "Already have '#{short_name}'."
+      else
+        messages << message
+      end
+    end
+
+    messages
+  end
+
+  def time_from item
+    return nil if not item
+    item.published || item.pubDate || item.date_published || item.updated || nil
+  end
+end

data/lib/imap-feeder/imapfeederconfig.rb

@@ -0,0 +1,74 @@
+require 'yaml'
+require 'uri'
+require 'net/http'
+require 'ftools'
+
+require 'imap-feeder/opmlreader'
+
+class ImapFeederConfig
+  def self.create(opml_file, root_folder)
+
+    root_folder = root_folder ? "INBOX.#{root_folder}" : "INBOX"
+
+    if opml_file
+      items = process(OpmlReader.get(File.open(opml_file)), "#{root_folder}").flatten
+    else
+      items = [
+        {"feed" => {"url" => "http://rubyforge.org/export/rss_sfnews.php",    "path" => "#{root_folder}.rubyforge"}},
+        {"feed" => {"url" => "http://feeds.feedburner.com/DilbertDailyStrip", "path" => "#{root_folder}.dilbert"}}
+      ]
+    end
+
+    File.open("#{Dir.pwd}/feeds.yml", "w+") do |file|
+      YAML.dump(items, file)
+    end
+
+    File.copy "#{File.dirname(__FILE__)}/../../settings.rb.example", "#{Dir.pwd}/settings.rb"
+  end
+
+  def self.check(configuration)
+    YAML.load(configuration).each do |conf_item|
+      check_url_connection(conf_item['feed']['url'])
+      check_path_name(conf_item['feed']['path'])
+    end
+  end
+   
+  private
+  def self.process(folder, parent_path)
+    items = []
+    path = parent_path + folder.name
+      
+    folder.urls.each do |child|
+      feed_path = "#{path}.#{child.name}"
+      $log.debug "#{feed_path}: #{child.url}"
+      items << {"feed" => {"url" => child.url, "path" => feed_path}}
+    end
+  
+    folder.children.each do |child|
+      items << process(child, path + '.')
+    end
+    items
+  end
+  
+  def self.check_url_connection(url)
+    begin
+      uri = URI.parse url
+      uri.path = "/" if uri.path.empty?
+
+      response = Net::HTTP.new(uri.host, uri.port).head(uri.path, nil)
+      if response.code =~ /^[^2]\d/
+        $log.info "Connecting to #{url}: #{response.message}, code: #{response.code}"
+      else
+        $log.info "Connecting to #{url}: OK"
+      end
+    rescue Exception => e
+      $log.warn "Exception while connecting to #{url}: #{e}."
+    end
+  end
+  
+  def self.check_path_name(path)
+    path.scan(/[^#{IMAP_CHARS}\.]+/) do |char|
+      $log.error "Invalid character found in \'#{path}\': #{char}"
+    end
+  end
+end