fastreader 1.0.0

data/lib/fastreader.rb

@@ -0,0 +1,283 @@
+$KCODE = 'u'
+require 'jcode'
+require 'rubygems' # I thought of using datamapper, but using activerecord will avoid installation
+# problems for people with macs
+require 'simple-rss'
+require 'activerecord'
+require 'action_view/helpers/date_helper.rb'
+require 'yaml'
+require 'feed-normalizer'
+require 'open-uri'
+require 'hpricot'
+$:.unshift File.dirname(__FILE__)
+require 'curses'
+require 'curses_extensions'
+require 'feed'
+require 'virtual_feed'
+require 'entry'
+require 'autodiscovery'
+require 'character_cleaner'
+require 'display'
+require 'entry_controller'
+require 'feeds_controller'
+require 'entries_controller'
+require 'curses_controller'
+require 'menu_window'
+require 'entry_window'
+require 'command_window'
+require 'menu_pager'
+require 'opml'
+require 'logger'
+require 'fileutils'
+include FileUtils
+include ActionView::Helpers::DateHelper
+
+class Fastreader
+  VERSION = '1.0.0'
+
+  attr_accessor :database_path
+
+  def setup_database(database_path)
+    unless File.exist?(database_path)
+      # copy the stock starter feed database (which just contains a few feed
+      # subscriptions) to the database_path
+      default_db = File.dirname(__FILE__) + "/../db/default.sqlite3"
+      cp(default_db, database_path)
+    end
+  end
+
+  def initialize(options={})
+    @curses = options[:curses] # If true it is in curses mode
+    @debug = false # change if there is an option
+    @environment = options[:environment] 
+
+    if ['test', 'development'].include?(@environment)
+      config = File.open(File.dirname(__FILE__) + '/../config/database.yml')
+      dbconfig = YAML::load(config)[@environment]
+    else
+      dbconfig = {"timeout"=>5000, "adapter"=>"sqlite3"}.merge({'database' => options[:database]})
+      database_path = options[:database] || ENV['HOME'] + '/fastreader.sqlite3'
+      setup_database(database_path)
+    end
+
+    ActiveRecord::Base.establish_connection(dbconfig)
+
+    # establish logging if in development mode
+    if @debug
+      log_file_path = File.dirname(__FILE__) + "/../log/#{@environment}.log"
+      log_file = File.open(log_file_path, 'a') 
+      log_file.sync = true
+    else
+      log_file = STDOUT
+    end
+    ActiveRecord::Base.logger = Logger.new(log_file)
+    ActiveRecord::Base.logger.level = Logger::INFO
+
+
+    # get Display object
+    @display = Display.new(options)
+  end
+
+  def parse(command)
+    self.instance_eval(command)
+  end
+
+  def auto_discover_and_subscribe(url)
+    uri = URI.parse(url)
+    feed_url = Autodiscovery.new(fetch(url)).discover
+    if feed_url
+      feed_url = uri.merge(feed_url).to_s
+      puts "Found feed: #{feed_url}" 
+      return feed_url
+    else
+      puts "Can't find feed for #{url}" 
+      return nil
+    end
+  end
+
+  def puts(string)
+    if @output_block
+      @output_block.call(string)
+    else
+      STDOUT.puts( string )
+    end
+  end
+
+  def import_opml(opml)
+    importer = OPMLImporter.new(opml)
+    feeds = importer.feed_urls.each do | url |
+      subscribe(url)
+    end
+  end
+ 
+  def subscribe(feed_url, &block)
+
+    if @output_block.nil? && (block_given? || block)
+      @output_block = block
+    end
+    
+    # try to repair the URL if possible
+    unless feed_url =~ /^http:\/\//
+      feed_url = "http://" + feed_url
+    end
+
+    puts "Subscribing to #{feed_url}"
+    begin
+      xml = fetch(feed_url)
+    rescue SocketError
+      puts "Error trying to load page at #{feed_url}"
+      return
+    end
+    if xml.nil?
+      puts "Can't find any resource at #{feed_url}"
+      return
+    end
+
+    LOGGER.debug( "xml length: %s, feed_url: %s, block: %s" % [xml.length, feed_url, block.class])
+    feed = Feed.create_feed( xml, feed_url.strip, &block )
+    LOGGER.debug(feed.class)
+
+    if feed.nil?
+    
+      puts "Can't find feed at #{feed_url}"
+      puts "Attempting autodiscovery..."
+   
+      feed_url = auto_discover_and_subscribe(feed_url)
+      if feed_url
+        puts "Subscribing to #{feed_url}"
+        xml = fetch(feed_url)
+
+        feed = Feed.create_feed( xml, feed_url.strip, &block )
+      end
+    end
+    feed
+  end
+
+  def update(options = {}, &block)
+    if @output_block.nil? && (block_given? || block)
+      @output_block = block
+    end
+
+    num = 0 
+    if feed_id = options[:feed_id]
+
+      f = Feed.find(feed_id)
+
+      puts "Updating from #{f.feed_url}"
+
+      result =  f.update_self( fetch(f.feed_url), options[:force], &block ) 
+      num += result || 0
+
+    else
+      Feed.find(:all).each {|f| 
+        
+        begin 
+          puts f.feed_url
+          result = f.update_self( fetch(f.feed_url) )  
+          num += result || 0
+        rescue
+          puts "Error trying to update from #{f.feed_url}! Skipping for now."
+        end
+       
+      }
+    end
+    # Return the number updated 
+    return num
+  end
+
+  def delete_all
+    Feed.delete_all
+  end
+
+  # Shows the +number+ most recent posts across all feeds
+  def most_recent(number=10)
+    entries = Entry.find(:all,
+                         :order => "last_updated desc",
+                         :limit => number)
+    @display.display_entries(entries)
+  end
+
+  def list
+    # Add virtual feeds here
+    feeds = Feed.feeds_list
+    @display.list_feeds( feeds )
+  end
+
+  alias_method :ls, :list
+
+  # a simple wrapper over open-uri call. Easier to mock in testing.
+  def fetch(url)
+    begin
+      open(url).read
+    rescue Timeout::Error 
+      puts "-> attempt to fetch #{url} timed out"
+    rescue Exception => e
+      puts "-> error trying to fetch #{url}: #{$!}"
+    end
+  end
+
+  def get_binding
+    return binding()
+  end
+end
+
+
+# for development
+def reload
+  puts "Reloading " + __FILE__ 
+  load __FILE__
+end
+
+def preprocess(command_string)
+  
+  # Preprocessing steps to make the command a valid Ruby statement:
+ 
+  # If the command is simply a url, then subscribe to it
+  if command_string.strip =~ /^http:/
+    command_string = "subscribe " + command_string
+  end
+  
+  
+  # Surround any url with quotes:
+  command_string = command_string.gsub(/(http:[^\s]*)/, '"\1"')
+
+  # default action is list feeds
+  if command_string.strip == ""
+    command_string = "ls"
+  end
+  command_string
+end
+
+# set up logging, especially for the curses part
+#logfile = File.open(File.dirname(__FILE__) + "/../textfeeds_development.log", "a")
+logfile = STDOUT #production mode
+LOGGER = Logger.new(logfile)
+LOGGER.level = Logger::INFO
+
+database = ENV['HOME'] + '/fastreader.sqlite3'
+FASTREADER_CONTROLLER = Fastreader.new(:database => database,
+                          :no_links => true, 
+                          :simple => true, 
+                          :curses => true, 
+                          :width => 60)
+
+def run(argv)
+  # If there are arguments, then interpret them directly. Otherwise, start
+  # an interactive session.
+  
+  command = preprocess(argv.join(' '))
+
+  # If the command is an OPML file, import the feeds in it 
+  if command.strip =~ /\.opml$/
+    puts "Importing OPML: #{command}"
+    FASTREADER_CONTROLLER.import_opml File.read(command.strip)
+  else
+    eval(command, FASTREADER_CONTROLLER.get_binding)
+  end
+
+end
+
+if __FILE__ == $0
+  run(ARGV)
+end
+

data/lib/feed.rb

@@ -0,0 +1,222 @@
+class Feed < ActiveRecord::Base
+  serialize :urls
+  serialize :authors
+  serialize :skip_hours
+  serialize :skip_days
+  has_many :entries, 
+    :order => "date_published desc, created_at desc, id desc", 
+    :dependent => :destroy # TODO make sure the user can override this
+  
+  # Takes a url an creates a feed object and subscription
+  def self.create_feed(xml, feed_url, &block)
+
+    if block_given? || block
+      @output_block = block
+    end
+
+    feed = FeedNormalizer::FeedNormalizer.parse(xml, :force_parser => FeedNormalizer::SimpleRssParser)
+    return nil unless feed.is_a?(FeedNormalizer::Feed)
+
+    puts "Looking for #{feed_url} in the database"
+    if found_feed=Feed.find_by_feed_url(feed_url)
+
+      puts "Feed already exists"
+
+      # Update it
+      puts found_feed.import_entries(feed)
+
+      return found_feed
+
+    end
+    puts "Not found. Subscribing."
+
+    new_feed = Feed.create(:feed_id => feed.id,
+                :title => feed.title.strip,
+                # It's very importannt that this is feed_url and not feed.url:
+                :feed_url => feed_url.strip, 
+                :urls => feed.urls.map {|x| x.strip},
+                :parser => feed.parser,
+                :last_updated => feed.last_updated || Time.now,
+                :authors => feed.authors,
+                :copyright => feed.copyright,
+                :image => feed.image,
+                :generator => feed.generator,
+                :ttl => feed.ttl,
+                :skip_hours => feed.skip_hours,
+                :skip_days => feed.skip_days)
+    # create entries
+    new_feed.import_entries(feed)
+    new_feed
+  end
+
+  def self.feeds_list
+    feeds = []
+    feeds = feeds + Feed.find(:all, :order => "title asc")
+
+    flagged_entries = VirtualFeed.new
+    flagged_entries.title = "Flagged Entries"
+    flagged_entries.finder_params = {:conditions => "flagged is not null", :order => "flagged desc"}
+
+    feeds << flagged_entries 
+
+    all_entries = VirtualFeed.new
+    all_entries.title = "All Entries"
+    all_entries.finder_params = {:order => "id desc"}
+
+    feeds << all_entries 
+
+    feeds
+  end
+
+  def puts(string)
+    if @output_block
+      @output_block.call(string)
+    else
+      STDOUT.puts( string )
+    end
+  end
+  
+  # Takes a FeedNormalizer::Feed object
+  def import_entries(feed)
+
+    num_new_items = 0
+
+    # Reverse the entries because they are most recent first.
+    feed.entries.reverse.each do |entry| 
+      # Check if the entry already exists
+      # puts "Looking for existing entry with id #{entry.id}"
+      if (existing_entry = self.entries.find(:first, 
+                                             :conditions => ["entry_id = ?", entry.id ? entry.id : entry.url ]))
+
+        # Do nothing if the entry has not been updated
+        if existing_entry.last_updated == entry.last_updated
+          #puts "Skipping #{entry.title}. Already exists."
+          next
+
+        # The entry has been updated, so update it.
+        else
+          puts "Updating #{entry.title}"
+          update_entry(existing_entry, entry)
+          next 
+        end
+
+      else 
+        puts "Importing #{entry.title}"
+        num_new_items += 1
+        import_entry(entry)
+      end
+    end
+
+    num_new_items
+  end
+
+  # Takes a FeedNormalizer::Entry object
+  def import_entry(entry)
+    unless entry.id || entry.url 
+      puts "Skipping #{entry.title}. Bad item. No entry id or url detected."
+      return 
+    end   
+
+    self.entries.create(:title => entry.title,
+                        :description => entry.description,
+                        :content => entry.content,
+                        :categories => entry.categories,
+                        :date_published => entry.date_published || entry.last_updated,
+                        :url => entry.url,
+                        :urls => entry.urls,
+                        # If the entry.id is nil, use the entry.url (this
+                        # happens for some reason on Slashdot and maybe other
+                        # websites.
+                        :entry_id => entry.id ? entry.id.strip : entry.url.strip, 
+                        :authors => entry.authors,
+                        :copyright => entry.copyright, 
+                        # Apparently entry.last_updated is a Time object
+                        :last_updated => entry.last_updated ? entry.last_updated.to_datetime : nil)
+  end
+
+  # The old entry is ActiveRecord. The new one is a FeedNormalizer::Entry
+  def update_entry(old, new)
+    old.update_attributes(:title => new.title,
+                        :description => new.description,
+                        :content => new.content,
+                        :categories => new.categories,
+                        :date_published => new.date_published,
+                        :url => new.url,
+                        :urls => new.urls,
+                        :authors => new.authors,
+                        :copyright => new.copyright, 
+                        # Apparently new.last_updated is a Time object
+                        :last_updated => new.last_updated ? new.last_updated.to_datetime : nil)
+
+  end
+
+
+  # This field is used to determine whether an entry in the feed is new, in
+  # which case it is colored in a special way
+  def previously_updated_at
+    unless self['previously_updated_at']
+      return self['created_at']
+    end
+    self['previously_updated_at']
+  end
+
+  # Takes a new version of the feed xml
+  # Can't call this "update" because that's an important ActiveRecord method
+  # The block is the output method. If no block is given it a standard block is
+  # created that just outputs to stdout.
+  # +puts+ calls the output lambda when it's available; otherwise it prints to
+  # STDOUT.
+
+  def too_soon_to_update?
+    self.updated_at.to_time > (Time.now - 3600)
+  end
+
+  def update_self(xml, force=false, &block)
+    num_new_items = 0
+    if block_given?
+      @output_block = block
+    end
+
+    unless force
+      # To be courteous, don't update feeds that have been downloaded in the last
+      # hour.
+      if too_soon_to_update?
+        puts "-> skipping. last update was with the last hour."
+        return
+      end
+    end
+
+    # :updated_at is used for this program's internal bookkeeping, and tracks when the feed was last
+    # accessed. :last_updated is the property of the feed.
+    
+    begin 
+
+      # We're forcing the SimpleRssParser because the other one led to errors with DaringFireball's Atom feed.
+      new_feed_content = FeedNormalizer::FeedNormalizer.parse(xml, :force_parser => FeedNormalizer::SimpleRssParser)
+      # Trye another parser
+      unless new_feed_content.is_a?(FeedNormalizer::Feed)
+        puts "Failed to update #{self.title}. Try again later."
+        LOGGER.debug("FAILED TO UPDATE #{self.title}")
+        LOGGER.debug(xml)
+        return
+      end
+
+      # At this point we're definitely updating the feed.
+
+      # create entries
+      # The import_entries method should silently skip entries that already exist
+      num_new_items += import_entries(new_feed_content)
+      
+      # This updates the last_updated timestamp
+      self.last_updated = Time.now 
+
+      self.save
+      puts "-> %s new items found." % num_new_items
+    rescue
+      puts "-> There was an error updating the feed #{self.feed_url}."
+      raise
+    end
+    return num_new_items 
+  end
+end
+