RubyGems - imap-feeder - Versions diffs - 0.1.0 - Mend

imap-feeder 0.1.0

Files changed (51) hide show

data.tar.gz.sig +0 -0
data/.project +23 -0
data/History.txt +4 -0
data/License.txt +341 -0
data/Manifest.txt +48 -0
data/README.txt +77 -0
data/Rakefile +4 -0
data/TODO.txt +11 -0
data/bin/imap-feeder +68 -0
data/config/hoe.rb +77 -0
data/config/requirements.rb +17 -0
data/lib/imap-feeder.rb +87 -0
data/lib/imap-feeder/createconfigparser.rb +40 -0
data/lib/imap-feeder/fakeserver.rb +23 -0
data/lib/imap-feeder/feedfolder.rb +23 -0
data/lib/imap-feeder/feedreader.rb +81 -0
data/lib/imap-feeder/imapfeederconfig.rb +74 -0
data/lib/imap-feeder/message.rb +152 -0
data/lib/imap-feeder/messagestore.rb +35 -0
data/lib/imap-feeder/opmlreader.rb +50 -0
data/lib/imap-feeder/server.rb +77 -0
data/lib/imap-feeder/version.rb +9 -0
data/script/destroy +14 -0
data/script/generate +14 -0
data/script/txt2html +77 -0
data/settings.rb.example +58 -0
data/setup.rb +1585 -0
data/tasks/deployment.rake +34 -0
data/tasks/environment.rake +7 -0
data/tasks/website.rake +15 -0
data/test/data/encoded.rss +22 -0
data/test/data/erroneous.yml +7 -0
data/test/data/last_messages.yaml +0 -0
data/test/data/rss20_no_body.xml +10 -0
data/test/data/rss20_one_entry.xml +11 -0
data/test/data/rss20_two_entries.xml +17 -0
data/test/data/rss20_with_authors.xml +21 -0
data/test/data/simple.opml +9 -0
data/test/functional_test_server.rb +95 -0
data/test/test_feedreader.rb +92 -0
data/test/test_imap-feeder.rb +66 -0
data/test/test_imapfeederconfig.rb +56 -0
data/test/test_message.rb +312 -0
data/test/test_messagestore.rb +54 -0
data/test/test_opmlreader.rb +73 -0
data/test/testlogger.rb +22 -0
data/website/javascripts/rounded_corners_lite.inc.js +285 -0
data/website/stylesheets/screen.css +137 -0
data/website/template.rhtml +49 -0
metadata +175 -0
metadata.gz.sig +0 -0

@@ -0,0 +1,152 @@
+require 'base64'
+require 'time'
+require 'action_mailer'
+require 'hpricot'
+require 'tidy'
+require 'htmlentities'
+require 'digest/md5'
+$KCODE="U"
+class Message
+  include ActionMailer::Quoting
+  def initialize(params)
+    @params = params
+  end
+  def title
+    @title ||= (dec( (@params[:title] || "").gsub(/(\r\n)|\r|\n/, " ")) || "")
+  end
+  def from
+    @from ||= (dec(@params[:from]) || "Unknown <spam@example.org>")
+  end
+  def body
+    @body ||= strip_html(@params[:body] || @params[:url] || "")
+  end
+  def time
+    @time ||= (@params[:time] || Time.now.localtime).rfc2822
+  end
+  def quote(str)
+    return "" if not str
+    str.gsub(/[^a-zA-Z0-9 -_:,\.]+/) {|to_quote| quote_if_necessary(to_quote, "UTF-8")}
+  end
+  def generate_identifier
+    @cached_identifier ||= "#{title}##{Digest::MD5.hexdigest(body())}"
+  end
+  def format
+    url   = @params[:url]
+    return <<-EOF
+Date: #{time}
+Subject: #{quote(title)}
+From: #{quote(from)}
+Content-Type: text/plain;
+  charset="utf-8"
+Content-Transfer-Encoding: 8bit
+#{body}#{"\n\n" + url if url}
+EOF
+  end
+  private
+  def replace(doc, element)
+    doc.search(element) do |found|
+      replacement = block_given? ? yield(found) : found.innerHTML
+      found.swap(replacement)
+    end
+  end
+  def tidy(body)
+    begin
+      Tidy.path = $tidy_path unless Tidy.path
+    rescue LoadError => e
+      $log.warning "Tidy not available: #{e.message}"
+      return body
+    end
+    tidy_html = Tidy.open(:show_warnings => true) do |tidy|
+      tidy.options.markup = true
+      tidy.options.wrap = 0
+      tidy.options.logical_emphasis = true
+      tidy.options.drop_font_tags = true
+      tidy.options.output_encoding = "utf8"
+      tidy.options.input_encoding = "utf8"
+      tidy.options.doctype = "omit"
+      tidy.clean(body)
+    end
+    tidy_html.strip!
+    tidy_html.gsub!(/^<html>(.|\n)*<body>/, "")
+    tidy_html.gsub!(/<\/body>(.|\n)*<\/html>$/, "")
+    tidy_html.gsub!("\t", "  ")
+    tidy_html
+  end
+  def dec(html)
+    HTMLEntities.decode_entities(html).strip if html
+  end
+  def strip_html(body)
+    doc = Hpricot(tidy(dec(body)))
+    replace(doc, 'p')      {|paragraph| "\n#{paragraph.innerHTML}\n"}
+    replace(doc, 'strong') {|strong| "*#{strong.innerHTML}*"}
+    replace(doc, 'b')      {|bold| "*#{bold.innerHTML}*"}
+    replace(doc, 'em')     {|em| "*#{em.innerHTML}*"}
+    replace(doc, 'li')     {|li| "- #{li.innerHTML}"}
+    replace(doc, 'i')
+    replace(doc, 'ol')
+    replace(doc, 'ul')
+    replace(doc, 'abr')
+    replace(doc, 'font')
+    replace(doc, 'span')
+    replace(doc, 'div')
+    replace(doc, 'br')     {|br| "\n"}
+    replace(doc, 'img')    {|img| img.attributes['alt'] || ""}
+    replace(doc, 'abbr')   {|abbr| abbr.innerHTML + (" (#{abbr.attributes['title']})" || "")}
+    urls = gather_urls(doc)
+    body = doc.to_html
+    unless urls.empty?
+      body << "\n"
+      max_length = "[0]".length + Math.log10(urls.size).floor + 1
+      urls.each_with_index do |url, i|
+        index = i + 1
+        str = "\n[#{index}]".ljust max_length
+        body << "#{str} #{url}"
+      end
+    end
+    doc = Hpricot(body)
+    replace(doc, 'a')
+    body = doc.to_html
+    #sanitize newlines
+    body.gsub!(/(\n\s*){3,}/, "\n\n")
+    dec(body)
+  end
+  def gather_urls(doc)
+    urls = []
+    doc.search('a') do |link|
+      href = URI(link.attributes['href']) rescue nil
+      next if not href && href.host
+      next if link.innerHTML.strip == href.to_s.strip
+      urls << href
+      link.swap(link.innerHTML.strip + "[#{urls.length}]")
+    end
+    urls
+  end
+end

data/lib/imap-feeder/messagestore.rb ADDED

@@ -0,0 +1,35 @@
+require 'yaml'
+class MessageStore
+  MESSAGES_TO_STORE = 100
+  def initialize(file)
+    @file = file
+    @root = {}
+    if File.exist? @file
+      File.open(@file) do |f|
+        @root = YAML.load(f) || {}
+      end
+    else
+      $log.warn "#{file} does not exist, creating a new one."
+    end
+  end
+  def add_new(folder, titles, number_of_entries = MESSAGES_TO_STORE)
+    @root[folder] ||= []
+    @root[folder].unshift(*titles)
+    @root[folder].slice!((number_of_entries * 2)..-1)
+    @root[folder].compact!
+  end
+  def get_archived(folder)
+    @root[folder] || []
+  end
+  def save
+    File.open(@file, "w") do |f|
+      YAML.dump(@root, f)
+    end
+  end
+end

data/lib/imap-feeder/opmlreader.rb ADDED

@@ -0,0 +1,50 @@
+require 'rexml/document'
+require 'rexml/parsers/sax2parser'
+require 'rexml/sax2listener'
+require 'imap-feeder/feedfolder'
+#
+# Defines the characters that can be used as part of an IMAP Folder
+#
+IMAP_CHARS = "\\w:,\\-= "
+#
+# Parses an OPML File and extracts the folders and urls of the feeds. A
+# tree like structure is built with FoodFolders and FeedUrls as leafes.
+#
+class OpmlReader
+  #
+  # Entry point for the parsing process. Takes the filecontent as a string and returns the root element.
+  #
+  def self.get(file)
+    opml = REXML::Document.new(file)
+    parse_opml(opml.elements['opml/body'])
+  end
+  private
+  #
+  # Replaces the disallowed characters from the folder name
+  #
+  def self.replace_bad_chars(name)
+    name.gsub(/[^#{IMAP_CHARS}]+/, "_")
+  end
+  #
+  # Parses the given node and recursively traverses through the children
+  #
+  def self.parse_opml(opml_node, folder = FeedFolder.new(""))
+    opml_node.elements.each('outline') do |element|
+      if element.attributes['isOpen'] != nil || element.attributes['type'] == "folder"
+        child_folder = FeedFolder.new(replace_bad_chars(element.attributes['text']))
+        folder.add_sub(child_folder)
+        parse_opml(element, child_folder)
+      else
+        folder.add_url(FeedUrl.new(replace_bad_chars(element.attributes['title']), element.attributes['xmlUrl']))
+      end
+    end
+    folder
+  end
+end

data/lib/imap-feeder/server.rb ADDED

@@ -0,0 +1,77 @@
+require 'base64'
+require 'net/imap'
+require 'imap-feeder/message'
+$KCODE="U"
+class Server
+  def initialize(params)
+    @connection = Net::IMAP.new(params[:host], params[:port], params[:use_ssl])
+    result = @connection.login(params[:user], params[:pass])
+  end
+  def disconnect
+    @connection.disconnect if @connection
+  end
+  def send(message, folder="INBOX")
+    @connection.select(folder)
+    message = message.format.gsub(/\n/, "\r\n")
+    @connection.append(folder, message, nil, Time.now)
+  end
+  def retrieve(title, folder="INBOX")
+    @connection.examine(folder)
+    found = @connection.search(["SUBJECT", title]).first
+    return if not found
+    imap_header = @connection.fetch([found], "BODY[HEADER.FIELDS (SUBJECT)]")
+    retr_title = imap_header.first.attr["BODY[HEADER.FIELDS (SUBJECT)]"]
+    retr_title.gsub!(/(^Subject: )|[\n\r]/, "")
+    Message.new(:title => base64decode(retr_title), :id => found)
+  end
+  def base64decode(subject)
+    encoded = subject[/^=\?utf-8\?b\?(.*?)$/, 1]
+    if encoded
+      Base64.decode64(encoded)
+    else
+      subject
+    end
+  end
+  private :base64decode
+  def has?(title, folder)
+    retrieve(title, folder) != nil
+  end
+  def has_folder?(folder)
+    @connection.examine(folder)
+    true
+  rescue Net::IMAP::NoResponseError
+    false
+  end
+  def create_folder(folder)
+    path = ''
+    folder.split('.').each do |part|
+      path << part
+      @connection.create(path) unless has_folder?(path)
+      path << '.'
+    end
+  end
+  def delete_folder(folder)
+    #Switch to root so we can delete the folder
+    @connection.examine("INBOX")
+    @connection.delete(folder)
+  end
+  def delete(message, folder="INBOX")
+    @connection.select(folder)
+    @connection.store(message.id, "+FLAGS", [:Deleted])
+    @connection.expunge
+  end
+end

data/lib/imap-feeder/version.rb ADDED

@@ -0,0 +1,9 @@
+class ImapFeeder #:nodoc:
+  module VERSION #:nodoc:
+    MAJOR = 0
+    MINOR = 1
+    TINY  = 0
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+end

data/script/destroy ADDED

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate ADDED

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/script/txt2html ADDED

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+begin
+  require 'newgem'
+rescue LoadError
+  puts "\n\nGenerating the website requires the newgem RubyGem"
+  puts "Install: gem install newgem\n\n"
+  exit(1)
+end
+require 'redcloth'
+require 'syntax/convertors/html'
+require 'erb'
+require File.dirname(__FILE__) + '/../lib/imap-feeder/version.rb'
+version  = ImapFeeder::VERSION::STRING
+download = 'http://rubyforge.org/projects/imap-feeder'
+class Fixnum
+  def ordinal
+    # teens
+    return 'th' if (10..19).include?(self % 100)
+    # others
+    case self % 10
+    when 1: return 'st'
+    when 2: return 'nd'
+    when 3: return 'rd'
+    else    return 'th'
+    end
+  end
+end
+class Time
+  def pretty
+    return "#{mday}#{mday.ordinal} #{strftime('%B')} #{year}"
+  end
+end
+def convert_syntax(syntax, source)
+  return Syntax::Convertors::HTML.for_syntax(syntax).convert(source).gsub(%r!^<pre>|</pre>$!,'')
+end
+if ARGV.length >= 1
+  src, template = ARGV
+  template ||= File.join(File.dirname(__FILE__), '/../website/template.rhtml')
+else
+  puts("Usage: #{File.split($0).last} source.txt [template.rhtml] > output.html")
+  exit!
+end
+template = ERB.new(File.open(template).read)
+title = nil
+body = nil
+File.open(src) do |fsrc|
+  title_text = fsrc.readline.gsub(/^=/, "")
+  body_text = fsrc.read
+  syntax_items = []
+  body_text.gsub!(%r!<(pre|code)[^>]*?syntax=['"]([^'"]+)[^>]*>(.*?)</\1>!m){
+    ident = syntax_items.length
+    element, syntax, source = $1, $2, $3
+    syntax_items << "<#{element} class='syntax'>#{convert_syntax(syntax, source)}</#{element}>"
+    "syntax-temp-#{ident}"
+  }
+  body_text.gsub!(/^===/, "h3.")
+  body_text.gsub!(/^==/, "h2.")
+  body_text.gsub!(/^- /, "* ")
+  title = RedCloth.new(title_text).to_html.gsub(%r!<.*?>!,'').strip
+  body = RedCloth.new(body_text).to_html
+  body.gsub!(%r!(?:<pre><code>)?syntax-temp-(\d+)(?:</code></pre>)?!){ syntax_items[$1.to_i] }
+end
+stat = File.stat(src)
+created = stat.ctime
+modified = stat.mtime
+$stdout << template.result(binding)

data/settings.rb.example ADDED

@@ -0,0 +1,58 @@
+# Configuration for imap-feeder
+# IMAP connection settings.
+$host = ""
+$user = ""
+$pass = ""
+# If you want to use a secure connection using ssl/tls, then uncomment the
+# following two declarations.
+#$port = "993"
+#$use_ssl = true
+# Base directory for imap-feeder
+BASEDIR="#{ENV['HOME']}/.imap-feeder"
+# We use html-tidy to improve the structure of the feed
+$tidy_path = "/usr/lib/libtidy.so"
+# imap-feeder needs a file to store the last message for each feed
+# so it doesn't fetch old entries. Where should that file be?
+$temp = "#{BASEDIR}/processed_feeds.yml"
+# The configuration file that was generated
+$config = "#{BASEDIR}/feeds.yml"
+# The file that contains information about the currently running instance
+$running_instance = "#{BASEDIR}/running_instance"
+# Configuration for the logger (from `ri Logger.new`):
+#
+#------------------------------------------------------------ Logger::new
+#     Logger::new(logdev, shift_age = 0, shift_size = 1048576)
+#------------------------------------------------------------------------
+#     Synopsis
+#       Logger.new(name, shift_age = 7, shift_size = 1048576)
+#       Logger.new(name, shift_age = 'weekly')
+#
+#     Args
+#     +logdev+:     The log device. This is a filename (String) or IO
+#                   object (typically +STDOUT+, +STDERR+, or an open
+#                   file).
+#
+#     +shift_age+:  Number of old log files to keep, *or* frequency of
+#                   rotation (+daily+, +weekly+ or +monthly+).
+#
+#     +shift_size+: Maximum logfile size (only applies when +shift_age+
+#                   is a number).
+#
+$log = Logger.new "#{BASEDIR}/log.txt", 10
+# The level can be set to: DEBUG < INFO < WARN < ERROR < FATAL
+$log.level = Logger::WARN
+$log.formatter = proc { |severity, datetime, progname, msg|
+  "#{severity} (#{datetime}): #{msg}\n"
+}