vnews 0.0.1

data/lib/vnews/autodiscoverer.rb

@@ -0,0 +1,24 @@
+require 'nokogiri'
+
+class Vnews
+  class AutodiscoveryFailed < StandardError;  end
+
+  module Autodiscoverer
+    def auto_discover(feed_url)
+      html = open(feed_url)
+      doc = Nokogiri::HTML.parse(html)
+      feed_url = [ 'head link[@type=application/atom+xml]', 
+        'head link[@type=application/rss+xml]', 
+        "head link[@type=text/xml]"].detect do |path|
+          doc.at(path)
+        end
+      if feed_url
+        feed_url
+      else
+        nil
+      end
+    rescue Errno::ECONNRESET, Nokogiri::CSS::SyntaxError
+      $stderr.puts $!
+    end
+  end
+end

data/lib/vnews/config.rb

@@ -0,0 +1,122 @@
+require 'vnews/sql'
+require 'yaml'
+
+class Vnews
+  def self.sql_client
+    $sql_client ||= Config.load_config
+  end
+
+  module Config
+    def self.generate_config
+      config = Vnews.sql_client.config.inject({}) do |memo, (k, v)|
+        memo[k.to_s] = v
+        memo
+      end
+      out = [config.to_yaml.sub(/---\s+/, ''), '']
+      Vnews.sql_client.configured_folders.map do |x| 
+        folder = x["folder"]
+        out << folder
+        Vnews.sql_client.feeds_in_folder(folder).each do |feed|
+          out <<  feed
+        end
+        out << ""
+      end
+      out.join("\n")
+    end
+
+    def self.rewrite_config
+      out = generate_config
+      f = File.open(File.expand_path(CONFIGPATH), 'w') {|f| f.write(out)}
+    end
+
+    def self.stub_config
+      stub = <<-END.gsub(/^ */, '')
+        host: localhost 
+        database: vnews 
+        username: root 
+        password: 
+
+        General News 
+        http://feedproxy.google.com/economist/full_print_edition
+        http://feeds.feedburner.com/TheAtlanticWire
+
+        Humor
+        http://feed.dilbert.com/dilbert/blog
+
+        Tech 
+        http://rss.slashdot.org/Slashdot/slashdot
+        http://feeds2.feedburner.com/readwriteweb
+        http://feedproxy.google.com/programmableweb
+        http://news.ycombinator.com/rss
+        http://daringfireball.net/index.xml
+        http://dailyvim.blogspot.com/feeds/posts/default
+      END
+    end
+
+    CONFIGPATH = "#{ENV['HOME']}/.vnewsrc"
+
+    def self.update_folders
+      f = File.read(File.expand_path(CONFIGPATH))
+      db, list = f.split(/^\s*$/,2)
+      current_folder = nil
+      ff = []
+      list.split("\n").each do |line|
+        line = line.strip
+        if line =~ /^\s*http/ # feed
+          ff << [line, current_folder]
+        elsif line =~ /^\s*\w+/ # folder
+          current_folder = line
+        end
+      end
+      puts "Using feeds and folders: #{ff.inspect}"
+
+      old_feeds = Vnews.sql_client.feeds(0).map {|x| x["feed_url"]}
+      new_feeds = ff.map {|feed,folder| feed}
+      rm_feeds = old_feeds - new_feeds
+      puts "Removing feeds: #{rm_feeds.inspect}"
+      rm_feeds.each {|x| Vnews.sql_client.delete_feed(x)}
+
+      # ff is an association between a feed and a folder
+      old_ff = Vnews.sql_client.configured_feeds_folders 
+      rm_ff = old_ff - ff
+      puts "Removing feed-folder associations: #{rm_ff.inspect}"
+      rm_ff.each {|feed,folder| Vnews.sql_client.delete_feed_folder(feed,folder)}
+
+      puts "Adding feeds: #{(new_feeds - old_feeds).inspect}"
+      puts "Adding folder-feed associations: #{(ff - old_ff).inspect}"
+      feeds2 = []
+      threads = []
+      ff.each do |feed_url, folder|
+        threads << Thread.new do 
+          feeds2 << Vnews::Feed.fetch_feed(feed_url, folder)
+        end
+      end
+      threads.each {|t| t.join}
+      feeds2.each do |x|
+        feed_url, f, folder = *x
+        folder ||= "Misc"
+        if f.nil?
+          $stderr.print "\nNo feed found for #{feed_url}\n"
+        else
+          Vnews::Feed.save_feed(feed_url, f, folder)
+        end
+      end
+      $stderr.puts "\nDone."
+    end
+
+    def self.load_config
+      if ! File.exists?(File.expand_path(CONFIGPATH))
+        return false
+      end
+      return if $sql_client
+      f = File.read(File.expand_path(CONFIGPATH))
+      top, bottom = f.split(/^\s*$/,2)
+      dbconfig = YAML::load top
+      Sql.new(dbconfig)
+    end
+  end
+end
+
+if __FILE__ == $0
+  puts Vnews::Config.generate_config 
+end

data/lib/vnews/display.rb

@@ -0,0 +1,157 @@
+require 'vnews/config'
+require 'yaml'
+require 'date'
+
+class Vnews
+  class Display
+
+    def initialize
+      @sqliteclient = Vnews.sql_client
+      @window_width = 140
+    end
+
+    # returns folders as a list
+    def folders
+      @sqliteclient.folders.map do |x|
+        "#{x["folder"]} (#{x['count']})"
+      end.join("\n")
+    end
+
+    # returns feeds as a list, sorted alphabetically
+    # e.g.
+    # {"title"=>"Bits",
+    # "feed_url"=>"http://bits.blogs.nytimes.com/feed/",
+    # "link"=>"http://bits.blogs.nytimes.com/"}
+    def feeds(order)
+      # '0' is alphabetical, '1' is most popular first
+      @sqliteclient.feeds(order.to_i).map.with_index do |x, idx|
+        "#{ x['title'] } (#{x['item_count']})"
+      end
+    end
+
+    # returns items as a list, most recent first
+    # e.g.
+    # {"title"=>"Episode 96: Git on Rails", "guid"=>"git-on-rails",
+    # "feed"=>"http://feeds.feedburner.com/railscasts",
+    # "feed_title"=>"Railscasts", "pub_date"=>2008-03-10 00:00:00 -0400,
+    # "word_count"=>41}
+
+    def col(string, width)
+      return unless string
+      string[0,width].ljust(width)
+    end
+
+    def format_date(d)
+      if d.nil?
+        "no date"
+      elsif d.year != Time.now.year
+        d.strftime("%b %Y")
+      elsif d.to_date == Time.now.to_date
+        d.strftime("%I:%M%P")
+      else
+        d.strftime("%b %d")
+      end 
+    end
+
+    # for item display
+    def format_long_date(d)
+      if d.nil?
+        "[no date]"
+      else
+        d.strftime("%a %m/%d/%Y at %I:%M%p %Z") 
+      end
+    end
+
+    def format_item_summary(i, width)
+      varwidth = width.to_i - 31
+      feed_title = col i['feed_title'], varwidth * 0.25
+      title = col i['title'], varwidth * 0.75
+      word_count = i['word_count'].to_s.rjust(6)
+      date = format_date(i['pub_date']).rjust(7) 
+      spacer = " " * 20 # to push guid all the way off screen
+      guid = i['guid']
+
+      flag = i['unread'] == 1 ? '+' : ' '
+      flag = i['starred'] == 1 ? '*' : flag
+      "%s | %s | %s | %s | %s | %s | %s" % [flag, feed_title, title, word_count, date, spacer, guid]
+    end
+
+    # look up feed up idx
+    def feed_items(*feed_selection)
+      window_width = feed_selection[0]
+      feed_title = feed_selection[1].split(' ')[0..-2].join(' ') 
+      @sqliteclient.feed_items(feed_title).map do |x|
+        format_item_summary x, window_width
+      end
+    end
+
+    def self.strip_item_count(folder)
+      folder.gsub(/\(\d+\)$/, '').strip
+    end
+
+    def folder_items(window_width, folder)
+      # strip off the count summary
+      folder = self.class.strip_item_count(folder)
+      @sqliteclient.folder_items(folder).map do |x|
+        format_item_summary x, window_width
+      end
+    end
+
+    def format_item(item)
+      res = <<-END
+#{item['feed']}
+#{item['feed_title']}
+#{format_long_date item['pub_date']}  #{item['word_count']} words
+ 
+
+#{item['title']}#{item['author'] ? ("\n" + item['author']) : '' }  
+
+#{item['text']}
+
+---
+#{item['link']}
+
+      END
+    end
+
+    def show_item(guid, increment_read_count=false)
+      inc_read_count = increment_read_count == '1' 
+      res = @sqliteclient.show_item(guid.strip, inc_read_count).first
+      if res
+        format_item(res)
+      else
+        "No item found"
+      end
+    end
+
+    def star_item(guid)
+      @sqliteclient.star_item guid, true
+    end
+
+    def unstar_item(guid)
+      @sqliteclient.star_item guid, false
+    end
+
+    def delete_item(guid)
+      @sqliteclient.delete_item guid
+    end
+
+    def search_items(window_width, term)
+      res = @sqliteclient.search_items(term).map do |x|
+        format_item_summary x, window_width
+      end
+      res.empty? ? "No matches" : res
+    end
+
+    def cat_items(*guids)
+      text = guids.map do |guid|
+        show_item(guid, 0) 
+      end.join( "\n+" + ('-' * 78) + "+\n" )
+    end
+  end
+
+end
+
+if __FILE__ == $0
+  puts Vnews::Display.new.send *ARGV
+end

data/lib/vnews/exceptions.rb

@@ -0,0 +1,4 @@
+class Vnews
+  class SubscribeFailed < StandardError;  end
+end
+

data/lib/vnews/feed.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require 'open-uri'
+require 'feed_yamlizer'
+#require 'vnews/autodiscoverer'
+require 'vnews/config'
+
+class Vnews
+  class Feed
+    #include Autodiscoverer
+
+    USER_AGENT = "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4"
+
+    def self.get_feed(feed_url)
+      response = open(feed_url, "User-Agent" => USER_AGENT)
+
+      xml = response.read
+      # puts response.last_modified
+      $stderr.print "#{feed_url} -> Found #{response.content_type} #{response.charset}\n"
+
+      charset = response.charset || "UTF-8"
+
+      not_xml = response.content_type !~ /xml/ && xml[0,900] !~ /<?xml|<rss/
+      if not_xml
+        return nil
+      end
+      feed_yaml = FeedYamlizer.run(xml, charset)
+      feed_yaml
+    rescue OpenURI::HTTPError, REXML::ParseException, NoMethodError, Timeout::Error
+      $stderr.puts "ERROR (#{feed_url}): #{$!} => #{$!.message}"
+    end
+
+    def self.fetch_feed(xml_url, folder=nil)
+      f = self.get_feed(xml_url)
+      [xml_url, f, folder]
+    end
+
+    # f is the feed hash
+    def self.save_feed(feed_url, f, folder=nil)
+      # if no folder, we're just updating a feed
+      if folder
+        Vnews.sql_client.insert_feed(f[:meta][:title], feed_url, f[:meta][:link], folder)
+      end
+      f[:items].each do |item|
+        if item[:guid].nil? || item[:guid].strip == ''
+          item[:guid] = [f[:meta][:link], f[:link]].join(":::")
+        end
+        Vnews.sql_client.insert_item item.merge(:feed => feed_url, :feed_title => f[:meta][:title])
+        $stderr.print "."
+      end
+    rescue
+      puts "ERROR: dump: #{feed_url.inspect}, #{folder.inspect}: #{f.inspect} "
+      raise
+    end
+
+    def self.update_feed(feed_title)
+      require 'vnews/display'
+      feed_title = Vnews::Display.strip_item_count(feed_title)
+      puts "Updating feed: #{feed_title}"
+      feed_url = Vnews.sql_client.feed_by_title feed_title
+      puts "Fetching data from feed url: #{feed_url}"
+      # puts "Deleting feed items for #{feed_url}"
+      # Vnews.sql_client.delete_feed_items feed_url
+      f = Vnews::Feed.get_feed feed_url
+      save_feed feed_url, f, nil
+      puts "\nFeed updated"
+    end
+
+    def self.log(text)
+      $stderr.puts text
+    end
+  end
+end
+
+if __FILE__ == $0
+  Vnews::Feed.new(ARGV.first, ARGV.last).fetch
+end
+

data/lib/vnews/folder.rb

@@ -0,0 +1,29 @@
+require 'vnews/feed'
+
+class Vnews
+  class Folder
+    def self.update_folder(folder)
+      if folder.strip == "Starred"
+        puts "Sorry, you can't update the starred folder."
+        return
+      else
+        require 'vnews/display'
+        folder = Vnews::Display.strip_item_count(folder)
+        puts "Updating folder: #{folder.inspect}"
+        threads = []
+        feeds = []
+        Vnews.sql_client.feeds_in_folder(folder.strip).each do |feed|
+          threads << Thread.new do 
+            feeds << Vnews::Feed.fetch_feed(feed, folder)
+          end
+        end
+        threads.each {|t| t.join}
+        puts "Saving data to database"
+        feeds.select {|x| x[1]}.compact.each do |feed_url, f, folder|
+          Vnews::Feed.save_feed feed_url, f, folder
+        end
+        puts "\nDone"
+      end
+    end
+  end
+end

data/lib/vnews/opml.rb

@@ -0,0 +1,50 @@
+require 'nokogiri'
+require 'vnews/feed'
+
+class Vnews
+  class Opml
+
+    CONCURRENCY = 18
+
+    def self.import(opml)
+      sqlclient = Vnews.sql_client
+      doc = Nokogiri::XML.parse(opml) 
+      feeds = []
+      doc.xpath('/opml/body/outline').each_slice(CONCURRENCY) do |xs|
+        threads = []
+        xs.each do |n|
+          threads << Thread.new do 
+            if n.attributes['xmlUrl']
+              feeds << Vnews::Feed.fetch_feed(n.attributes['xmlUrl'].to_s)
+            else
+              folder = n.attributes["title"].to_s
+              $stderr.print "Found folder: #{folder}\n"
+              n.xpath("outline[@xmlUrl]").each do |m|
+                feeds << Vnews::Feed.fetch_feed(m.attributes['xmlUrl'].to_s, folder)
+              end
+            end
+          end
+        end
+        threads.each {|t| t.join}
+      end
+
+      $stderr.puts "Making database records"
+      feeds.each do |x|
+        feed_url, f, folder = *x
+        folder ||= "Misc"
+        if f.nil?
+          $stderr.print "\nNo feed found for #{feed_url}\n"
+        else
+          Vnews::Feed.save_feed(feed_url, f, folder)
+        end
+      end
+      $stderr.puts "\nDone."
+
+    end
+  end
+end
+
+if __FILE__ == $0
+  opml = STDIN.read
+  Vnews::Opml.import(opml)
+end