feedbase 0.0.1

data/.gitignore

@@ -0,0 +1,2 @@
+*.swp
+pkg/

data/NOTES

@@ -0,0 +1,65 @@
+------------------------------------------------------------------------
+Fri Jun 24 08:57:18 EDT 2011
+
+This will be the feed aggregator engine for Kindlefeeder, but also
+usable for other projects.
+
+PostgreSQL
+
+db/create.sql
+
+Create a database called 'feeds'
+
+? rchardet https://rubygems.org/gems/edouard-rchardet
+
+API
+
+  FetchFeed.from_url [feed-url]
+
+  takes care of getting the feed, parsing, and storing in postgres
+
+Next step is to rebuild the Rails app around this.
+
+Users
+Subscriptions
+Schedule
+
+What's the most conservative way of doing this?
+
+Keep MySQL and the schema, 
+
+Payload generator is a discrete part. Refactor this. and keep the middle 
+
+bin/payload_generator2 is not used any more. Feeds are fetched in
+lib/feed_couch.rb #refresh method. It doesn't seem this refresh method
+can handle https
+
+RelaxedPayloadGenerator is the key class used by Delivery.
+
+Payload Specification can be turned into a TABLE and Model. Or the
+fields can be added to delivery as discrete fields (instead of blob);
+
+TODO. 
+- Check ETag when updating feed.
+- private password feeds as user requested
+- NEW MOBI generation; periodical format
+- migrate feeds from MySQL to Postgresql
+  - check new feed creation
+- clean up rails; move to rails 3; put into git; simplify deployment
+
+
+
+------------------------------------------------------------------------
+Sat Jun 25 23:40:52 EDT 2011
+
+
+postgresql functions
+curval()
+setval()
+
+I can perhaps sync the feed_id of this table with feeds.id in the rails
+database?
+
+
+
+

data/README.markdown

@@ -0,0 +1,22 @@
+# Feedbase
+
+Better Instructions forthcoming
+
+1. Create a PostgreSQL database called feeds.
+2. Load database script db/create.sql.
+
+API
+
+
+    Feedbase::Feed[feed_url: feed_url] || Feedbase::Feed.create(feed_url: feed_url)
+
+    == Instance methods and attributes
+
+    Feedbase::Feed#refresh
+
+    Feedbase::Feed#title
+    Feedbase::Feed#feed_url
+    Feedbase::Feed#items
+
+
+

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+require 'rake/testtask'
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), 'lib')
+
+desc "Run tests"
+task :test do 
+  $:.unshift File.expand_path("test")
+  require 'test_helper'
+  Dir.chdir("test") do 
+    Dir['*_test.rb'].each do |x|
+      puts "requiring #{x}"
+      require x
+    end
+  end
+
+  MiniTest::Unit.autorun
+end
+
+task :default => :test
+

data/db/create.sql

@@ -0,0 +1,40 @@
+-- this is for postgresql
+drop table if exists feeds CASCADE;
+drop table if exists feed_downloads CASCADE;
+drop table if exists items CASCADE;
+
+create table feeds (
+  feed_id serial primary key,
+  feed_url varchar UNIQUE NOT NULL,
+  title varchar,
+  alpha_title varchar,
+  subtitle varchar,
+  web_url varchar,
+  favicon_url varchar,
+  subscribers integer default 0,
+  created timestamp default now()
+);
+
+create table feed_downloads (
+  feed_download_id serial primary key,
+  feed_id integer REFERENCES feeds (feed_id) ON DELETE CASCADE,
+  download_time float,
+  headers text,
+  encoding varchar,
+  etag varchar,
+  last_modified timestamp,
+  created timestamp default now()
+);
+
+create table items (
+  item_id serial primary key,
+  feed_id integer REFERENCES feeds (feed_id) ON DELETE CASCADE,
+  guid varchar UNIQUE NOT NULL,
+  title varchar,
+  link varchar,
+  content text,
+  author varchar,
+  word_count integer,
+  pub_date timestamp default now()
+);
+

data/db/setup.sh

@@ -0,0 +1,3 @@
+dropdb feeds
+createdb feeds
+psql feeds < db/create.sql

data/feedbase.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "feedbase"
+  s.version     = '0.0.1'
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '>= 1.9.0'
+
+  s.authors     = ["Daniel Choi"]
+  s.email       = ["dhchoi@gmail.com"]
+  s.homepage    = ""
+  s.summary     = %q{A feed aggregator database}
+  s.description = %q{}
+
+  s.rubyforge_project = "feedbase"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'sequel'
+end

data/lib/feedbase/feed.rb

@@ -0,0 +1,92 @@
+require 'sequel'
+require 'feedbase/fetch_feed'
+DB = Sequel.connect 'postgres:///feeds'
+
+module Feedbase
+
+  class Redirected < StandardError; end
+
+  class Feed < Sequel::Model
+    one_to_many :items
+    one_to_many :feed_downloads
+
+    # returns number of items created
+    def refresh(force=false)
+      # check headers and etag and last modified
+      raise "Missing feed_url" if feed_url.nil?
+      ff = Feedbase::FetchFeed.new(feed_url)
+      headers = ff.headers
+      if !force 
+        if last_etag && (headers[:etag] == last_etag)
+          puts "-- #{feed_url} -- ETag cache hit"
+          return
+        end
+      end
+      data = ff.fetch 
+      params = data[:feed_params].merge(:alpha_title => make_alpha_title(data[:feed_params][:title])) 
+      if params[:feed_url] != self[:feed_url]
+        if x = self.class.filter(:feed_url => params[:feed_url]).first
+          raise Redirected.new("Redirected to existing feed: #{x.feed_url}")
+        end
+      end
+      params.delete(:feed_url) 
+      begin Sequel::DatabaseError
+        update params
+      rescue StandardError # PGError
+        puts "The offending record is #{self.inspect}"
+        raise
+      end
+
+      Feedbase::FeedDownload.create({feed_id: feed_id}.merge(data[:download_params])) 
+      items_created = data[:items].
+        select {|item| Feedbase::Item[:guid => item[:guid]].nil?}.
+        map { |item|
+          params = {
+            feed_id: feed_id,
+            title: item[:title].encode("utf-8"), 
+            guid: item[:guid], 
+            link: item[:link],
+            content: item[:content],
+            author: item[:author],
+            word_count: item[:word_count],
+            pub_date: item[:pub_date]
+          }
+          Feedbase::Item.create params
+        }
+      # caller can extract an item count from this
+      items_created
+    end
+
+    def last_download
+      @last_download ||= FeedDownload.filter(feed_id: feed_id).first
+    end
+
+    def last_etag
+      last_download && last_download.etag
+    end
+
+    def last_modified
+      last_download && last_download.last_modified
+    end
+
+    def make_alpha_title(s)
+      return if s.nil?
+      s.gsub(/^(The|A|An)\s/, '')
+    end
+  end
+
+  class Item < Sequel::Model
+    many_to_one :feed
+  end
+
+  class FeedDownload < Sequel::Model
+  end
+
+end
+
+if __FILE__ == $0
+  feed = Feedbase::Feed[feed_url: ARGV.first] || Feedbase::Feed.create(feed_url: ARGV.first)
+  puts feed
+  puts feed.refresh
+end
+

data/lib/feedbase/feed_parser.rb

@@ -0,0 +1,153 @@
+require 'rexml/streamlistener'
+require 'rexml/document'
+require 'pp'
+require 'iconv'
+require 'feedbase/html_simplifier'
+require 'date'
+
+module Feedbase
+
+  class FeedParser
+    # Try to have the XML in UTF-8 when you call this.
+    def initialize(xml)
+      @xml = xml
+      @listener = FeedListener.new
+      REXML::Document.parse_stream(@xml, @listener)
+    end
+
+    def result
+      tidy(@listener.result)
+    end
+
+    def tidy(feed)
+      feed[:items] = feed[:items].map do |item|
+        body = item[:content] || item[:summary] || ""
+        new_body = HtmlSimplifier.new(body, "utf-8").result.
+           gsub(%r{<p>(\n|<br/>)+</p>}, '').  
+           strip + "\n\n"
+        item.delete(:summary)
+        item[:content] = new_body
+        item[:word_count] = word_count(new_body)
+        item
+      end
+      feed
+    end
+
+    def word_count(string)
+      string.gsub(%{</?[^>]+>}, '').split(/\s+/).size
+    end
+  end
+
+  class FeedListener
+    include REXML::StreamListener
+
+    FEED_TITLE_TAGS = %w[ feed/title rss/channel/title rdf:RDF/channel/title ]
+    
+    FEED_LINK_TAGS = %w[ rss/channel/link rdf:RDF/channel/link ]
+    
+    ITEM_START_TAGS = %w[ feed/entry rss/channel/item rdf:RDF/item ] 
+    
+    ITEM_TITLE_TAGS = %w[ feed/entry/title rss/channel/item/title rdf:RDF/item/title ]
+    
+    ITEM_AUTHOR_TAGS = %w[ feed/entry/author/name rss/channel/item/author rdf:RDF/item/dc:creator ]
+    
+    ITEM_GUID_TAGS = %w[ feed/entry/id rss/channel/item/guid rdf:RDF/item/guid rdf:RDF/item/feedburner:origLink ]
+    
+    ITEM_PUB_DATE_TAGS = %w[ feed/entry/published feed/entry/created feed/entry/modified rss/channel/item/pubDate rdf:RDF/item/dc:date ]
+    
+    ITEM_LINK_TAGS = %w[ rss/channel/item/link rdf:RDF/item/link ] 
+    
+    ITEM_SUMMARY_TAGS = %w[ feed/entry/summary rss/channel/item/description rdf:RDF/item/description ] 
+    ITEM_CONTENT_TAGS = [ %r{feed/entry/content}, %r{rss/channel/item/content}, %r{rss/channel/item/content:encoded},  %r{rss/item/content}, %r{rdf:RDF/item/content} ]
+
+    def initialize
+      @nested_tags = []
+      @x = {:items => []}
+    end
+
+    def result; @x; end
+
+    def tag_start(name, attrs)
+      @nested_tags.push name
+      case path
+      when 'feed/link'
+        @x[:link] = encode attrs['href']
+      when *ITEM_START_TAGS
+        @current_item = {}
+      when 'feed/entry/link'
+        @current_item[:link] = encode attrs['href']
+      end
+    end
+
+    def tag_end(name)
+      case path
+      when *ITEM_START_TAGS
+        @x[:items] << @current_item
+        @current_item = nil
+      end
+      @nested_tags.pop
+    end
+
+    def text(text)
+      case path
+      when *FEED_TITLE_TAGS
+        @x[:title] = encode text.strip
+      when *FEED_LINK_TAGS 
+        @x[:link] = encode text.strip
+      when *ITEM_TITLE_TAGS 
+        @current_item[:title] = encode(text.strip)
+      when *ITEM_AUTHOR_TAGS 
+        @current_item[:author] = encode(text.strip)
+      when *ITEM_GUID_TAGS 
+        @current_item[:guid] = encode(text)
+      when *ITEM_PUB_DATE_TAGS
+        @current_item[:pub_date] = DateTime.parse(encode(text))
+      when *ITEM_LINK_TAGS 
+        @current_item[:link] = encode(text)
+      when *ITEM_SUMMARY_TAGS 
+        if @current_item[:summary] 
+          @current_item[:summary] << encode(text)
+        else
+          @current_item[:summary] = encode(text)
+        end
+      when *ITEM_CONTENT_TAGS
+        if @current_item[:content] 
+          @current_item[:content]  << encode(text)
+        else
+          @current_item[:content] = encode(text)
+        end
+      end
+    end
+    alias_method :cdata, :text
+
+    def xmldecl(decl, encoding, extra)
+      if encoding 
+        @x[:orig_encoding] = encoding.downcase
+      else
+        @x[:orig_encoding] = "UTF-8"
+      end
+    end
+
+
+    def path
+      @nested_tags.join('/')
+    end
+
+
+    # encoding method
+    def encode(string)
+      string
+    end
+
+
+  end
+end
+
+if __FILE__ == $0
+  feeds = ARGV
+  feeds.each do |feed|
+    xml = File.read feed
+    f = Feedbase::FeedParser.new(xml)
+    pp f.result
+  end
+end

data/lib/feedbase/fetch_feed.rb

@@ -0,0 +1,81 @@
+require 'feedbase/feed_parser'
+require 'timeout'
+require 'iconv'
+
+module Feedbase
+  class FetchFeed
+
+    attr_accessor :feed_url
+
+    def initialize(feed_url)
+      @feed_url = feed_url
+    end
+
+    def headers
+      if @headers
+        return @headers 
+      end
+      _headers = begin
+                   Timeout::timeout(20) do
+                     agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092416 Firefox/3.0.3"
+                     # get headers and any redirects
+                     res = `curl -sIL -A'#{agent}' '#{feed_url}'`.gsub("\r\n", "\n")
+                     if res !~ /^HTTP.*200 OK$/
+                       puts res.inspect
+                       raise "Response not OK"
+                     end
+                     res
+                   end 
+                 end
+
+      #TODO check for xml 
+      @headers = { headers: _headers, 
+        encoding: _headers[/^Content-Type:.*charset=(.*)$/i, 1],
+        etag: _headers[/^ETag: (.*)$/,1],
+        last_modified: ((x = _headers[/Last-Modified: (.*)/, 1]) && DateTime.parse(x)) }
+    end
+
+    def fetch
+      url = fix_url(feed_url)
+      start_time = Time.now
+      result = begin
+                 Timeout::timeout(20) do
+                   agent = "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.3) Gecko/2008092416 Firefox/3.0.3" 
+                   headers
+                   # get headers and any redirects
+                   `curl -sL -A'#{agent}' '#{url}'`
+                   end
+                 rescue StandardError, Timeout::Error => ex
+                   raise
+                 end
+      elapsed = Time.now - start_time
+      if !(x = headers[:headers].scan(/^Location: (.*)$/).flatten).empty?
+        #puts "Redirected to #{x.last}"
+        feed_url = x.last
+      end
+      result2 = Iconv.conv("UTF-8//TRANSLIT//IGNORE", (headers[:encoding] || 'iso-8859-1'), result)
+      f = FeedParser.new(result2).result
+      feed_params = {:feed_url => feed_url, :title => f[:title], :web_url => f[:link]}
+      items = f[:items]
+
+      { feed_params: feed_params,
+        items: f[:items],
+        download_params: headers.merge(download_time: elapsed) }
+    end
+
+    def fix_url(url)
+      unless url =~ /^https?:\/\//
+        url = "http://" + url
+      end
+      url
+    end
+  end
+
+end
+
+
+if __FILE__ == $0
+  puts Feedbase::FetchFeed.new(ARGV.first).fetch
+end
+
+

data/lib/feedbase/html_simplifier.rb

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby19
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+
+# Takes output of feed_file_generator.rb encoded in INPUT_ENCODING as input and
+# strips superfluous markup from the feed item bodies.
+
+#require 'feed_file_generator'
+require 'fileutils'
+require 'rexml/streamlistener'
+require 'rexml/document'
+require 'open3'
+
+# NOTE requires the htmltidy program
+# http://tidy.sourceforge.net/docs/Overview.html
+
+INPUT_ENCODING = 'utf-8'
+
+module Feedbase
+  class HtmlSimplifier
+    include FileUtils::Verbose
+    attr :result
+
+    # Takes feed data as hash. Generate this with FeedParser
+    def initialize(html, orig_encoding)
+      @orig_encoding = orig_encoding
+      @xml = tidy(pre_cleanup(html))
+      @result = parse.gsub(/<http[^>]+>/, "")
+    end
+
+    def parse
+      @listener = FeedHtmlListener.new
+      REXML::Document.parse_stream(@xml, @listener)
+      @listener.result + "\n\n"
+    end
+
+    def pre_cleanup(html)
+      html.gsub!("<o:p></o:p>", "")
+      html
+    end
+
+    def self.tidy(html, orig_encoding)
+      # assumes input encoding of latin 1
+      #output = Open3.popen3("tidy -q -n -wrap 120 -asxml -latin1") do |stdin, stdout, stderr|
+      #output = IO.popen("tidy -q -n -wrap 120 -asxml -latin1", "r+") do |pipe|
+      #output = IO.popen("tidy -q -wrap 120 -raw -asxml ", "r+") do |pipe| # if from latin1
+
+      tidy = "tidy -q -wrap 120 -n -utf8 -asxml 2>/dev/null"
+      output = IO.popen(tidy, "r+") do |pipe| 
+        input = <<-END
+  <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+  <html xmlns="http://www.w3.org/1999/xhtml">
+  <head><title></title></head><body>#{html}</body></html>
+        END
+        pipe.puts input
+        pipe.close_write
+        #$stderr.puts stderr.read
+        pipe.read
+      end
+      output
+    end
+
+    def tidy(html)
+      self.class.tidy html, @orig_encoding
+    end
+  end
+
+
+  class FeedHtmlListener
+    include REXML::StreamListener
+
+    STRIP_TAGS = %w[ body font ]
+    BLOCK_TAGS = %w[ p div ]
+    HEADER_TAGS =  %w[ h1 h2 h3 h4 h5 h6 ] 
+
+    UNIFORM_HEADER_TAG = "h4"
+
+    def initialize
+      @nested_tags = []
+      @content = [""]
+    end
+
+    def result
+      # we call strip_empty_tags twice to catch empty tags nested in a tag like <p>
+      # not full-proof but good enough for now
+      x = @content.map {|line| strip_empty_tags( strip_empty_tags( line ).strip ) }.
+        select {|line| line != ""}.compact.join("\n\n")
+    end
+
+    def strip_empty_tags(line)
+      line.gsub(%r{<(\w+)[^>]*>\s*</\1>}, '')
+    end
+
+    def tag_start(name, attrs)
+      @nested_tags.push name
+      case name 
+      when 'a'
+        # effectively strips out all style tags
+        @content[-1] << "<a href='#{attrs['href']}'>"
+      when 'img'
+        if attrs['alt']
+          text = (attrs['alt'].strip == '') ? 'image ' : "image:#{attrs['alt']} "
+          @content[-1] << text
+        end
+      when *HEADER_TAGS
+        @content << "<#{UNIFORM_HEADER_TAG}>" 
+      when 'br' #skip
+        @content << "<br/>"
+      when 'blockquote'
+        @content << "<blockquote>"
+      when 'ul', 'ol', 'dl'
+        @content << "<#{name}>"
+      when 'li', 'dt', 'dd'
+        @content[-1] << "  <#{name}>"
+      when 'strong', 'em'
+        @content[-1] << "<#{name}>"
+      when *BLOCK_TAGS
+        @content << "<p>"
+      when 'pre'
+        @content << "<pre>"
+      end
+    end
+
+    def tag_end(name)
+      @nested_tags.pop
+      case name
+      when 'a'
+        @content[-1] << "</a>" 
+      when *HEADER_TAGS
+        @content[-1] << "</#{UNIFORM_HEADER_TAG}>" 
+      when 'blockquote'
+        @content << '</blockquote>'
+      when 'ul', 'ol', 'dl'
+        @content[-1] << "</#{name}>"
+      when 'li', 'dt', 'dd'
+        @content[-1] << "  </#{name}>"
+      when 'strong', 'em'
+        @content[-1] << "</#{name}>"
+      when *BLOCK_TAGS
+        @content[-1] << "</p>"
+      when 'pre'
+        @content[-1] << "</pre>"
+      end
+    end
+
+    def text(text)
+      return if text =~ /\a\s*\Z/
+
+      # probably slow, but ok for now
+      @content[-1] << text
+    end
+
+    def start_of_block?
+      BLOCK_TAGS.include? @nested_tags[-1] 
+    end
+
+    def path
+      @nested_tags.join('/')
+    end
+  end
+
+  def word_count(string)
+    string.gsub(%{</?[^>]+>}, '').split(/\s+/).size
+  end
+end
+

data/lib/feedbase.rb

@@ -0,0 +1,7 @@
+require 'feedbase/feed'
+require 'feedbase/feed_parser'
+require 'feedbase/html_simplifier'
+require 'feedbase/fetch_feed'
+
+
+

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification 
+name: feedbase
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Daniel Choi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-07-31 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: ""
+email: 
+- dhchoi@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- NOTES
+- README.markdown
+- Rakefile
+- db/create.sql
+- db/setup.sh
+- feedbase.gemspec
+- lib/feedbase.rb
+- lib/feedbase/feed.rb
+- lib/feedbase/feed_parser.rb
+- lib/feedbase/fetch_feed.rb
+- lib/feedbase/html_simplifier.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: feedbase
+rubygems_version: 1.6.1
+signing_key: 
+specification_version: 3
+summary: A feed aggregator database
+test_files: []
+