siuying-fullfeed 0.4.6 → 0.4.8

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2009 siu [dot] ying [at] gmail [dot] com
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -1,35 +1,59 @@
-require "rubygems"
-require "pathname"
-require "rake"
-require "rake/testtask"
+require 'rake/clean'
+require 'rake/testtask'
+require 'fileutils'
 
-# Gem
 require "rake/gempackagetask"
 
-NAME = "fullfeed"
-SUMMARY = "Fullfeed RSS creator"
-GEM_VERSION = "0.4.6"
-
-spec = Gem::Specification.new do |s|
-  s.name          = NAME
-  s.summary       = s.description = SUMMARY
-  s.author        = "siuying"
-  s.email         = "siu.ying@gmail.com"
-  s.version       = GEM_VERSION
-  s.platform      = Gem::Platform::RUBY
-  s.require_path  = 'lib'
-  s.files         =  %w(README Rakefile) + Dir.glob("{examples,lib,test}/**/*")
-  
-  #   s.executables = ["rackup"]
-  s.add_dependency('ruby-cache', '>= 0.3.0')
-  s.add_dependency('hpricot',    '>= 0.6.1')
+task :default => :package
+
+# PACKAGING ============================================================
+
+# Load the gemspec using the same limitations as github
+def spec
+  @spec ||=
+    begin
+      require 'rubygems/specification'
+      data = File.read('fullfeed.gemspec')
+      spec = nil
+      Thread.new { spec = eval("$SAFE = 3\n#{data}") }.join
+      spec
+    end
 end
 
 Rake::GemPackageTask.new(spec) do |pkg|
   pkg.gem_spec = spec
 end
 
-desc "Install the FullFeed as a gem"
+desc "Install the Fullfeed as a gem"
 task :install => [:repackage] do
   sh %{gem install pkg/#{spec.name}-#{spec.version}}
-end
+end
+
+# Gemspec Helpers ====================================================
+def source_version
+  line = File.read('lib/fullfeed.rb')[/^\s*VERSION = .*/]
+  line.match(/.*VERSION = '(.*)'/)[1]
+end
+
+task 'fullfeed.gemspec' => FileList['lib/**','bin/**','examples/**','Rakefile','LICENSE','README'] do |f|
+  # read spec file and split out manifest section
+  spec = File.read(f.name)
+  head, manifest, tail = spec.split("  # = MANIFEST =\n")
+  # replace version and date
+  head.sub!(/\.version = '.*'/, ".version = '#{source_version}'")
+  head.sub!(/\.date = '.*'/, ".date = '#{Date.today.to_s}'")
+  # determine file list from git ls-files
+  files = `git ls-files`.
+    split("\n").
+    sort.
+    reject{ |file| file =~ /^\./ }.
+    reject{ |file| file =~ /^.+\/\./ }.
+    reject { |file| file =~ /^doc/ }.
+    map{ |file| "    #{file}" }.
+    join("\n")
+  # piece file back together and write...
+  manifest = "  s.files = %w[\n#{files}\n  ]\n"
+  spec = [head,manifest,tail].join("  # = MANIFEST =\n")
+  File.open(f.name, 'w') { |io| io.write(spec) }
+  puts "updated #{f.name}"
+end

data/examples/applenews_hk.rb

@@ -0,0 +1,22 @@
+# Extractor Example
+#
+# create full text RSS feed from Yahoo! News HK
+
+require "rubygems"
+require "fullfeed"
+require "#{File.dirname(__FILE__)}/extractors/apple_news_extractor"
+
+# convert encoding filer, convert feed and/or fulltext item to UTF-8
+# first parameter specify the source RSS Feed encoding
+# second parameter specify the fulltext item HTML page encoding
+filter = Fullfeed::Filters::ConvertEncodingFilter.new("UTF-8", "Big5")
+
+# create full text RSS feed from Yahoo! News HK
+# At most fetch 5 pages, wait 1 seconds before each try
+feed = Fullfeed::Feed.new("http://rss.appleactionews.com/rss.xml",
+        :limit => 5,
+        :agent => :open_uri,
+        :filters => filter,
+        :wait => 1)
+result = feed.fetch
+puts result

data/examples/extractors/apple_news_extractor.rb

@@ -0,0 +1,14 @@
+require "fullfeed"
+
+module Fullfeed
+  module Extractor
+    class AppleNewsExtractor < XpathExtractor
+      # register this extractor to the system
+      register
+
+      def initialize
+        super(%r{www.appleactionews.com}, ".article")
+      end
+    end
+  end
+end

data/examples/extractors/yahoo_news_hong_kong_extractor.rb

@@ -0,0 +1,14 @@
+require "fullfeed"
+
+module Fullfeed
+  module Extractor
+    class YahooNewsHongKongExtractor < XpathExtractor
+      # register this extractor to the system
+      register
+
+      def initialize
+        super(%r{http://hk\.rd\.yahoo.com/news/rss/\*http://.+\.html}, ".livewords")
+      end
+    end
+  end
+end

data/examples/server/controller/feed_controller.rb

@@ -0,0 +1,41 @@
+module Fullfeedr
+  class FeedController
+    CONF = [
+            {
+                    'name' => "appleactionnews",
+                    'url' => "http://rss.appleactionews.com/rss.xml",
+                    'filters' => [Fullfeed::Filters::ConvertEncodingFilter.new("UTF-8", "Big5"), Fullfeed::Filters::ExcessSpaceFilter.new]},
+            {
+                    'name' => "ynews-hk",
+                    'url' => "http://hk.news.yahoo.com/rss/hongkong/rss.xml"}
+    ].freeze
+
+    def initialize
+      @feeds = {}
+      CONF.each do |conf|
+        name = conf['name']
+        url = conf['url']
+        filters = conf['filters'] || []
+
+        @feeds[name] = Fullfeed::Feed.new(url,
+          :limit => 20,
+          :wait => 1,
+          :filters => filters,
+          :agent => :open_uri,
+          :store => :db)
+      end
+    end
+
+    def fetch(name)
+      if @feeds[name]
+        @feeds[name].fetch.to_s
+      else
+        raise ArgumentError, "not a registered name!"
+      end
+    end
+
+    def list
+      CONF
+    end
+  end
+end

data/examples/server/server.rb

@@ -0,0 +1,28 @@
+# Web Example
+#
+# Start a web server that host the Yahoo! Hong Kong news with full text feed
+# Require sinatra gem
+
+require "rubygems"
+gem('fullfeed', '>= 0.4.3')
+
+require 'fullfeed'
+require 'sinatra'
+require 'erb'
+
+require "#{File.dirname(__FILE__)}/controller/feed_controller"
+
+# setup
+Fullfeed::Store::DbStore.setup("sqlite3:ynews.sqlite3", true)
+server = Fullfeedr::FeedController.new
+
+# list feeds
+get '/' do
+  @list = server.list
+  erb :index
+end
+
+# fetch pages
+get '/:name' do
+  server.fetch(params[:name])
+end

data/examples/server/views/index.erb

@@ -0,0 +1,19 @@
+<html>
+    <head>
+      <title>Fullfeedr!</title> 
+      <link rel="stylesheet" href="http://www.w3.org/StyleSheets/Core/Modernist" type="text/css"/>
+    </head>
+    <body>
+        <h1>Welcome to Fullfeedr!</h1>
+        <p>We host following feed here: </p>
+        <ol>
+<% for item in @list %>
+          <li>
+            <a href="/<%= item['name'] %>">/<%= item['name'] %></a>            
+            (source: <a href="<%= item['url'] %>"><%= item['url'] %></a>)
+          </li>
+<% end %>
+        </ol>
+        <p>(Please wait while the fulltext feed is being downloaded)</p>
+    </body>
+</html>

data/examples/sinatra/web.rb

@@ -0,0 +1,29 @@
+# Web Example
+#
+# Start a web server that host the Yahoo! Hong Kong news with full text feed
+# Require sinatra gem
+
+require 'rubygems'
+require 'sinatra'
+require "#{File.dirname(__FILE__)}/../../lib/fullfeed"
+require "#{File.dirname(__FILE__)}/../extractors/yahoo_news_hong_kong_extractor"
+
+Fullfeed::Store::DbStore.setup("sqlite3:ynews.sqlite3")
+
+#DataMapper.auto_migrate!
+
+# Create Yahoo! News HK full-text feed
+feed = Fullfeed::Feed.new("http://hk.news.yahoo.com/rss/hongkong/rss.xml",
+        :limit => 20,
+        :wait => 1,
+        :agent => :open_uri,
+        :store => :db)
+
+# pre fetch the request
+feed.logger.info "Pre-Fetching RSS, could take some time ..."
+feed.fetch
+
+get '/' do
+  # fetch updated item
+  feed.fetch.to_s
+end

data/examples/ynews_hk.rb

@@ -0,0 +1,12 @@
+# Extractor Example
+#
+# create full text RSS feed from Yahoo! News HK
+
+require "rubygems"
+require "fullfeed"
+require "#{File.dirname(__FILE__)}/extractors/yahoo_news_hong_kong_extractor"
+
+feed = Fullfeed::Feed.new("http://hk.news.yahoo.com/rss/hongkong/rss.xml")
+puts feed.fetch
+
+

data/examples/ynews_hk_db.rb

@@ -0,0 +1,32 @@
+# Extractor Example, using Datamapper
+#
+# Create full text RSS feed from Yahoo! News HK, store RSS in datamapper.
+# Later invocation will not cause older items being download again.
+# Use sqlite3 as backend, use proper adapter for your needs!
+#
+# Uncomment the line under "Migrate database" when first run this app (it setup database)
+#
+
+require "rubygems"
+require "fullfeed"
+
+require "#{File.dirname(__FILE__)}/extractors/yahoo_news_hong_kong_extractor"
+
+# setup datamaper
+Fullfeed::Store::DbStore.setup("sqlite3:ynews.sqlite3")
+
+# Migrate database, use only once
+#DataMapper.auto_migrate!
+
+# create full text RSS feed from Yahoo! News HK
+# At most fetch 20 pages, wait 1 seconds before each try
+feed = Fullfeed::Feed.new("http://hk.news.yahoo.com/rss/hongkong/rss.xml",
+        :limit => 20,
+        :store => :db,
+        :wait => 1)
+result = feed.fetch
+puts result
+
+File.open("yahoo.rss", "w") do |file|
+  file.write(result)
+end

data/fullfeed.gemspec

@@ -0,0 +1,58 @@
+Gem::Specification.new do |s|
+  s.specification_version = 2 if s.respond_to? :specification_version=
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  
+  s.name = "fullfeed"
+  s.summary = s.description = "Create full text RSS feed from RSS"
+
+  s.author = "siuying"
+  s.email = "siu.ying@gmail.com"
+
+  s.version = '0.4.8'
+  s.date = '2009-06-19'
+  s.platform = Gem::Platform::RUBY
+  s.require_path = 'lib'
+
+  # = MANIFEST =
+  s.files = %w[
+    LICENSE
+    README
+    Rakefile
+    examples/applenews_hk.rb
+    examples/extractors/apple_news_extractor.rb
+    examples/extractors/yahoo_news_hong_kong_extractor.rb
+    examples/server/controller/feed_controller.rb
+    examples/server/server.rb
+    examples/server/views/index.erb
+    examples/sinatra/web.rb
+    examples/ynews_hk.rb
+    examples/ynews_hk_db.rb
+    fullfeed.gemspec
+    lib/fullfeed.rb
+    lib/fullfeed/agent/agent_factory.rb
+    lib/fullfeed/agent/appengine_agent.rb
+    lib/fullfeed/agent/base.rb
+    lib/fullfeed/agent/mechanize_agent.rb
+    lib/fullfeed/agent/open_uri_agent.rb
+    lib/fullfeed/extractor/base_extractor.rb
+    lib/fullfeed/extractor/extractor_factory.rb
+    lib/fullfeed/extractor/text_extractor.rb
+    lib/fullfeed/extractor/xpath_extractor.rb
+    lib/fullfeed/feed.rb
+    lib/fullfeed/filters/base_filter.rb
+    lib/fullfeed/filters/convert_encoding_filter.rb
+    lib/fullfeed/filters/excess_space_filter.rb
+    lib/fullfeed/filters/uppercase_filter.rb
+    lib/fullfeed/store/base.rb
+    lib/fullfeed/store/db_store.rb
+    lib/fullfeed/store/memory_store.rb
+    lib/fullfeed/store/store_factory.rb
+    test/load_files.rb
+    test/test_agent.rb
+    test/test_store.rb
+  ]
+  # = MANIFEST =
+
+  s.rubygems_version = '1.1.1'
+
+end

data/lib/fullfeed/agent/agent_factory.rb

@@ -0,0 +1,53 @@
+require 'singleton'
+
+module Fullfeed
+  module Agent
+    class AgentFactory
+      include Singleton
+
+      def initialize
+        @agents = {}
+      end
+
+      def register(clazz)
+        name = to_symbol_name(clazz.name)
+        @agents[name.to_sym] = clazz
+      end
+
+      # get HTTP agent by symbol
+      # Accetable agents:
+      #  # :open_uri  - simplistic HTTP client
+      #  # :mechanize - full feature HTTP client with cookies support
+      def agent(name = :open_uri)
+        agent_class = @agents[name]
+        if agent_class
+          agent = agent_class.new
+
+          if agent.is_a?(BaseAgent)
+            return agent
+          end
+        end
+
+        raise ArgumentError, "unknown agent name :#{name}, accepatable: #{@agents.keys.inspect}"
+      end
+
+      def self.agent(name = :open_uri)
+        instance.agent(name)
+      end
+
+      private
+      #input:  a full class name
+      #output: the class name lowercased, underscore separated,
+      #and removed "_agent" at last part
+      #e.g. "Fullfeed::Agent::MechanizeAgent" => "mechanize" 
+      def to_symbol_name(class_name)
+        class_name.
+            split("::").
+            last.
+            gsub(/(.)([A-Z])/, '\1_\2').
+            downcase.
+            gsub(/_agent$/, '')
+      end
+    end
+  end
+end

data/lib/fullfeed/agent/appengine_agent.rb

@@ -0,0 +1,21 @@
+gem('appengine-apis')
+require 'appengine-apis/urlfetch'
+
+module Fullfeed
+  module Agent
+    class AppengineAgent < BaseAgent
+      include AppEngine::URLFetch
+      register
+
+      def initialize
+        @logger = AppEngine::Logger.new
+      end
+
+      def get(url)
+        @logger.info "download link: #{url}"
+        result = fetch(url)
+        result.urlfetch_body rescue result.body
+      end
+    end
+  end
+end

data/lib/fullfeed/agent/base.rb

@@ -0,0 +1,14 @@
+module Fullfeed
+  module Agent
+    # All agent should implement one method: get
+    class BaseAgent
+      def self.register
+        Fullfeed::Agent::AgentFactory.instance.register(self)
+      end
+
+      def get(url)
+        raise "Must override get(url)"
+      end
+    end
+  end
+end

data/lib/fullfeed/agent/mechanize_agent.rb

@@ -0,0 +1,21 @@
+require 'mechanize'
+
+module Fullfeed
+  module Agent
+    #Use Mechanize as the agent
+    #Support cookies ... etc
+    class MechanizeAgent < BaseAgent
+      register
+
+      def initialize
+        @agent = WWW::Mechanize.new
+        @agent.user_agent_alias = "Mac FireFox"   
+      end
+
+      def get(url)
+        page = @agent.get(url)
+        page.content
+      end
+    end
+  end
+end

data/lib/fullfeed/agent/open_uri_agent.rb

@@ -0,0 +1,15 @@
+require 'open-uri'
+
+module Fullfeed
+  module Agent
+    #Use open-uri as the agent
+    #Simplistic but work
+    class OpenUriAgent < BaseAgent
+      register
+
+      def get(url)
+        open(url, "User-Agent" => "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; zh-TW; rv:1.9.0.10) Gecko/2009042315 Firefox/3.0.10").read        
+      end
+    end
+  end
+end

data/lib/fullfeed/extractor/base_extractor.rb

@@ -0,0 +1,37 @@
+require 'singleton'
+
+module Fullfeed
+  module Extractor
+    module ExtractorHelper
+      #If the url matched the RegExp(s).
+      #  # url - a string
+      #  # regexps - an Array of RegExp, or a RegExp
+      def is_matched?(url, regexps)
+        regexps = [regexps] unless regexps.is_a? Array
+        regexps.each do |rexexp|
+          return true if url =~ rexexp
+        end
+        return false
+      end
+    end
+
+    class BaseExtractor
+      include Singleton
+      include ExtractorHelper
+      
+      #If this extractor accept this url, if true, use it to parse the page
+      def accept(url)
+        false
+      end
+      
+      # extract text from html document, return the content
+      def extract(doc)
+        nil
+      end
+
+      def self.register
+        ExtractorFactory.instance.register(self)
+      end
+    end
+  end
+end

data/lib/fullfeed/extractor/extractor_factory.rb

@@ -0,0 +1,32 @@
+require 'singleton'
+
+module Fullfeed
+  module Extractor
+    class ExtractorFactory
+      include Singleton
+
+      def initialize
+        @extractors = []
+      end
+
+      def register(extractor_class)
+        @extractors << extractor_class
+      end
+
+      def unregister(extractor)
+        @extractors.delete(extractor)
+      end
+
+      def extractor(url)
+        extractors = @extractors.select() {|e| e.instance.accept(url) }
+        return extractors.first.instance if extractors.size > 0
+        
+        # if no extractors accept the above URL, use default TextExtractor
+        default = TextExtractor.instance
+        return default if default.accept(url)
+        return nil
+      end
+    end
+
+  end
+end

data/lib/fullfeed/extractor/text_extractor.rb

@@ -0,0 +1,18 @@
+module Fullfeed
+  module Extractor
+    # extract all text from html. this is being use if no other extractor is suitable
+    class TextExtractor < BaseExtractor
+      PATTERN = [/^http\:.+$/, /^https\:.+$/]
+
+      def accept(url)
+        is_matched?(url, PATTERN)
+      end
+
+      # extract a html document, return the content text
+      def extract(doc)
+        hdoc = Hpricot(doc)        
+        text = (hdoc/"//body").inner_text rescue nil
+      end
+    end
+  end
+end

data/lib/fullfeed/extractor/xpath_extractor.rb

@@ -0,0 +1,23 @@
+module Fullfeed
+  module Extractor
+    class XpathExtractor < BaseExtractor
+      attr_reader :xpath, :pattern
+
+      def initialize(pattern = nil, xpath = nil)
+        @pattern = pattern
+        @xpath = xpath
+      end
+
+      def accept(url)
+        is_matched?(url, @pattern)
+      end
+
+      # return content of Yahoo News HK page
+      def extract(doc)
+        hdoc = Hpricot(doc)
+        text = (hdoc.search(@xpath)).inner_html rescue nil
+      end
+
+    end
+  end
+end

data/lib/fullfeed/feed.rb

@@ -0,0 +1,105 @@
+require 'rubygems'
+gem('hpricot',    '>= 0.6.1')
+require 'hpricot'
+
+module Fullfeed
+  class Feed  
+    attr_reader :url, :encoding, :xml, :item_limit, :store
+    attr_accessor :logger
+
+    def initialize(url, options = {})
+      @url            = url
+      @wait           = options[:wait]      || 1
+      @item_limit     = options[:limit]     || 50
+      @agent_name     = options[:agent]     || :open_uri
+      @store_name     = options[:store]     || :memory
+
+
+      validate_params
+
+      @filters  = Fullfeed::Filters::FilterChain.new(options[:filters] || [])
+      @logger   = Logger.new(STDOUT)
+      @agent    = Fullfeed::Agent::AgentFactory.agent(@agent_name)
+      @store    = Fullfeed::Store::StoreFactory.store(@url, @item_limit, @store_name)
+    end
+
+
+    #Fetch the RSS feed.
+    #
+    #For each item in the feed, extract the content of the link and replace the description with it.
+    #Extraction is based on registered Extractor, check the extractor classes for more information.
+    def fetch
+      @logger.info "Fetch RSS URL: #{@url}"
+      doc = @agent.get(@url).to_s
+      doc = @filters.before_doc(doc)
+      @xml = Hpricot.XML(doc)
+      items = (@xml/"//item")
+
+      @logger.info "Process elements of RSS (count=#{items.size}, limit=#{@item_limit})"
+      items.to_a.first(@item_limit).each do |item|
+        process_item(item)
+      end
+
+      @filters.after_doc(@xml)
+    end
+
+    private
+    def validate_params
+      if @wait <= 0
+        raise ArgumentError, "invalid wait `#{@wait}'"
+      end
+      if @item_limit <= 0
+        raise ArgumentError, "invalid limit `#{@item_limit}'"
+      end
+    end
+
+    def process_item(item)
+      link = (item/"link").first.inner_text   rescue nil
+      desc = (item/"description").first       rescue nil
+      guid = (item/"guid").first.inner_text   rescue link
+
+      if link && desc
+        begin
+          @logger.debug "  Extract item (#{guid}) link: #{link}"
+          desc.swap("<description>#{Hpricot::Tag::CData.new(extract_cached(guid, link)).to_html}</description>")
+        rescue StandardError => e
+          @logger.error "Error fetching/replacing content: #{e.inspect}"
+
+        end        
+      else
+        @logger.warn "No link or desc node found in item: #{item}"
+
+      end
+    end
+
+    # read cache or fetch result
+    def extract_cached(guid, link)
+      @store[guid] ||= extract(link)
+    end
+
+
+    #Use ExtractorFactor to find a suitable Extractor, if found, extract supplied link to the URL.
+    #If not found, use TextExtractor which extract all text from the page.
+    def extract(link)
+      extractor = Extractor::ExtractorFactory.instance.extractor(link)
+
+      begin
+        unless extractor.nil?
+          @logger.debug "  Download link: #{link}"
+          doc = @agent.get(link).to_s
+          doc = @filters.before_item(doc)
+          doc = extractor.extract(doc).strip
+          doc = @filters.after_item(doc)
+          return doc
+        else
+          return nil
+        end
+
+      ensure
+        @logger.debug "  Wait #{@wait} seconds before next URL"
+        sleep(@wait) if @wait > 0
+
+      end
+    end
+  end
+end

data/lib/fullfeed/filters/base_filter.rb

@@ -0,0 +1,61 @@
+module Fullfeed
+  module Filters
+    class FilterChain
+      def initialize(filters)
+        filters = [filters] unless filters.is_a? Array
+        @filters = filters
+      end
+
+      def before_doc(doc)
+        run_filters(@filters, :before_doc, doc)
+      end
+
+      def after_doc(doc)
+        run_filters(@filters, :after_doc, doc)
+      end
+
+      def before_item(item)
+        run_filters(@filters, :before_item, item)
+      end
+      
+      def after_item(item)
+        run_filters(@filters, :after_item, item)
+      end
+
+      private
+      def run_filters(filters, method, target)
+        filters.each do |f|
+          target = f.send(method.to_sym, target)
+        end
+        target
+      end
+    end
+
+
+    class BaseFilter
+      # run before rss is processed
+      # doc is html text, should also return html text
+      def before_doc(doc)
+        doc
+      end
+      
+      # run after rss is processed
+      # doc is a Hpricot document, should also return a document
+      def after_doc(doc)
+        doc
+      end
+
+      # run before the item is processed
+      # item is HTML text, should also return html text
+      def before_item(item)
+        item
+      end
+
+      # run after the item is processed
+      # item is HTML text, should also return html text
+      def after_item(item)
+        item
+      end
+    end
+  end
+end

data/lib/fullfeed/filters/convert_encoding_filter.rb

@@ -0,0 +1,23 @@
+require 'iconv'
+
+module Fullfeed
+  module Filters
+    # convert feed to UTF-8 encoding
+    class ConvertEncodingFilter < BaseFilter
+      def initialize(feed_encoding, item_encoding)
+        @feed_encoding = feed_encoding
+        @item_encoding = item_encoding
+      end
+
+      # run before rss is processed
+      def before_doc(feed)
+        Iconv.conv("UTF-8//IGNORE", @feed_encoding, feed)
+      end
+
+      # run after process the item node
+      def after_item(item)
+        Iconv.conv("UTF-8//IGNORE", @item_encoding, item)
+      end
+    end
+  end
+end

data/lib/fullfeed/filters/excess_space_filter.rb

@@ -0,0 +1,16 @@
+require 'iconv'
+
+module Fullfeed
+  module Filters
+    # remove spaces between two chinese text, such as appledaily action news pages
+    class ExcessSpaceFilter < BaseFilter
+      def initialize
+      end
+
+      # run after process the item node
+      def after_item(item)
+        item.gsub(/([^a-zA-Z0+9]) /, '\1')
+      end
+    end
+  end
+end

data/lib/fullfeed/filters/uppercase_filter.rb

@@ -0,0 +1,21 @@
+require 'iconv'
+
+module Fullfeed
+  module Filters
+    # convert feed to UTF-8 encoding
+    class UppercaseFilter < BaseFilter
+      def initialize
+      end
+
+      # run before rss is processed
+      def before_doc(feed)
+        feed.upcase
+      end
+
+      # run after process the item node
+      def after_item(item)
+        item.upcase
+      end
+    end
+  end
+end

data/lib/fullfeed/store/base.rb

@@ -0,0 +1,21 @@
+module Fullfeed
+  module Store
+    class BaseStore
+      def initialize(url, cache_size)
+      end
+      
+      def self.register
+        StoreFactory.instance.register(self)
+      end
+
+      def []=(args)
+        raise "Must override []="
+      end
+
+      def [](args)
+        raise "Must override []"
+      end
+
+    end
+  end
+end

data/lib/fullfeed/store/db_store.rb

@@ -0,0 +1,51 @@
+require 'rubygems'
+gem('datamapper', '>=0.9.7')
+
+require 'dm-core'
+require 'dm-timestamps'
+
+module Fullfeed
+  module Store
+    class DbStore < BaseStore
+      register
+      
+      def self.setup(url, automigrate = false)
+        DataMapper.setup(:default, url)
+        begin
+          Item.first
+        rescue
+          DataMapper.auto_migrate!
+        end
+      end
+
+      #Initialize a datamapper store
+      def initialize(url, cache_size)
+        @url = url
+      end
+
+      #Save or update existing item by key
+      def []=(key, value)
+        item = Item.first_or_create(:feed_url => @url, :guid => key)
+        item.content = value
+        item.save
+        value
+      end
+
+      #Retrieve an item by key
+      def [](key)
+        item = Item.first(:feed_url => @url, :guid => key)       
+        item.content rescue nil
+      end
+
+    end
+
+    class Item
+      include DataMapper::Resource
+      property  :id,        Serial
+      property  :feed_url,  String
+      property  :guid,      String
+      property  :content,   String
+    end
+  end
+end
+

data/lib/fullfeed/store/memory_store.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+gem('ruby-cache', '>= 0.3.0')
+
+require 'cache'
+
+module Fullfeed
+  module Store
+    #Cache in memory, based on Ruby Cache gem
+    class MemoryStore < BaseStore
+      register
+      
+      def initialize(url, cache_size)
+        @cache = Cache.new({:max_num => cache_size})
+      end
+
+      def []=(key, value)
+        @cache[key] = value
+      end
+
+      def [](key)
+        @cache[key]
+      end
+
+    end
+  end
+end

data/lib/fullfeed/store/store_factory.rb

@@ -0,0 +1,57 @@
+module Fullfeed
+  module Store
+    class StoreFactory
+      include Singleton
+
+      def initialize
+        @stores = {}
+      end
+
+      #register a Store to the StoreFactory
+      def register(clazz)
+        name = to_symbol_name(clazz.name)
+        @stores[name.to_sym] = clazz
+      end
+
+      # get class extends BaseCache from a symbol
+      # Accetable name:
+      #  # :memory - store result in memory
+      #  # :db     - store result in database (require DataMapper)
+      def store(url, cache_size, name = :memory)
+        store_class = @stores[name]
+        
+        if store_class
+          if !cache_size || cache_size <= 0
+            raise ArgumentError, "invalid store size: #{cache_size}"
+          end
+
+          store = store_class.new(url, cache_size)
+          if store.is_a?(BaseStore)
+            return store
+          end
+        end
+
+        raise ArgumentError, "unknown store name :#{name}, accepatable: #{@stores.keys.inspect}"
+      end
+
+      #see instance method store
+      def self.store(url, cache_size, name = :memory)
+        instance.store(url, cache_size, name)
+      end
+
+      private
+      #input:  a full class name
+      #output: the class name lowercased, underscore separated,
+      #and removed "_store" at last part
+      #e.g. "Fullfeed::Store::MemoryStore" => "memory"
+      def to_symbol_name(class_name)
+        class_name.
+            split("::").
+            last.
+            gsub(/(.)([A-Z])/, '\1_\2').
+            downcase.
+            gsub(/_store$/, '')
+      end
+    end
+  end
+end

data/lib/fullfeed.rb

@@ -0,0 +1,50 @@
+path = File.expand_path(File.dirname(__FILE__))
+$:.unshift(path) unless $:.include?(path)
+
+module FullFeed
+  VERSION = '0.4.7'
+end
+
+require 'logger'
+
+require "fullfeed/agent/base"
+require "fullfeed/agent/agent_factory"
+require "fullfeed/agent/open_uri_agent"
+
+begin
+  # optionally require mechanize
+  gem('mechanize')
+  require "fullfeed/agent/mechanize_agent"
+rescue Gem::LoadError
+end
+begin
+  # optionally require appengine-api
+  gem('appengine-apis')
+  require "fullfeed/agent/appengine_agent"
+rescue Gem::LoadError
+rescue NameError
+end
+
+require "fullfeed/extractor/extractor_factory"
+require "fullfeed/extractor/base_extractor"
+require "fullfeed/extractor/text_extractor"
+require "fullfeed/extractor/xpath_extractor"
+
+
+require "fullfeed/filters/base_filter"
+require "fullfeed/filters/convert_encoding_filter"
+require "fullfeed/filters/uppercase_filter"
+require "fullfeed/filters/excess_space_filter"
+
+require "fullfeed/store/base"
+require "fullfeed/store/store_factory"
+require "fullfeed/store/memory_store"
+
+# only load DbStore if datamapper is installed
+begin
+  gem('datamapper', '>= 0.9.7')
+  require "fullfeed/store/db_store"
+rescue Gem::LoadError
+end
+
+require "fullfeed/feed"

data/test/load_files.rb

@@ -0,0 +1,7 @@
+module TestFiles
+    Dir.chdir(File.dirname(__FILE__)) do
+        Dir['files/*.{html,xhtml,xml}'].each do |fname|
+            const_set fname[%r!/(\w+)\.\w+$!, 1].upcase, IO.read(fname)
+        end
+    end
+end

data/test/test_agent.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+require "#{File.dirname(__FILE__)}/../lib/fullfeed"
+require 'load_files'
+
+class TestAgent < Test::Unit::TestCase
+  def test_agent_factory
+    open_uri_agent = Fullfeed::Agent::AgentFactory.instance.agent(:open_uri)
+    assert_not_nil open_uri_agent
+  end
+
+  def test_openuri_agent
+    agent = Fullfeed::Agent::AgentFactory.instance.agent(:open_uri)
+    doc = agent.get('http://www.google.com/')
+    assert_not_nil(doc.to_s)
+  end
+
+  def test_mechanize_agent
+    agent = Fullfeed::Agent::AgentFactory.instance.agent(:mechanize)
+    doc = agent.get('http://www.google.com/')
+    assert_not_nil(doc.to_s)
+  end
+  
+end

data/test/test_store.rb

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'test/unit'
+require "#{File.dirname(__FILE__)}/../lib/fullfeed"
+require 'load_files'
+
+class TestDbStore < Test::Unit::TestCase
+  Fullfeed::Store::DbStore.setup("sqlite3:ynews.sqlite3")
+
+  def test_db_store
+    store = Fullfeed::Store::StoreFactory.store('http://test', 100, :db)
+    base_test_store(store)
+  end
+
+  def test_memory_store
+    store = Fullfeed::Store::StoreFactory.store('http://test', 100, :memory)
+    base_test_store(store)
+  end
+  private
+  def base_test_store(store)
+
+    value = rand().to_s
+    store['/100'] = value
+    assert_equal store['/100'], value
+
+    value = "中文測試"
+    store['/200'] = value
+    assert_equal store['/200'], value
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: siuying-fullfeed
 version: !ruby/object:Gem::Version 
-  version: 0.4.6
+  version: 0.4.8
 platform: ruby
 authors: 
 - siuying
@@ -9,30 +9,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-05-16 00:00:00 -07:00
+date: 2009-06-19 00:00:00 -07:00
 default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: ruby-cache
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.3.0
-    version: 
-- !ruby/object:Gem::Dependency 
-  name: hpricot
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        version: 0.6.1
-    version: 
-description: Fullfeed RSS creator
+dependencies: []
+
+description: Create full text RSS feed from RSS
 email: siu.ying@gmail.com
 executables: []
 
@@ -41,8 +22,41 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- LICENSE
 - README
 - Rakefile
+- examples/applenews_hk.rb
+- examples/extractors/apple_news_extractor.rb
+- examples/extractors/yahoo_news_hong_kong_extractor.rb
+- examples/server/controller/feed_controller.rb
+- examples/server/server.rb
+- examples/server/views/index.erb
+- examples/sinatra/web.rb
+- examples/ynews_hk.rb
+- examples/ynews_hk_db.rb
+- fullfeed.gemspec
+- lib/fullfeed.rb
+- lib/fullfeed/agent/agent_factory.rb
+- lib/fullfeed/agent/appengine_agent.rb
+- lib/fullfeed/agent/base.rb
+- lib/fullfeed/agent/mechanize_agent.rb
+- lib/fullfeed/agent/open_uri_agent.rb
+- lib/fullfeed/extractor/base_extractor.rb
+- lib/fullfeed/extractor/extractor_factory.rb
+- lib/fullfeed/extractor/text_extractor.rb
+- lib/fullfeed/extractor/xpath_extractor.rb
+- lib/fullfeed/feed.rb
+- lib/fullfeed/filters/base_filter.rb
+- lib/fullfeed/filters/convert_encoding_filter.rb
+- lib/fullfeed/filters/excess_space_filter.rb
+- lib/fullfeed/filters/uppercase_filter.rb
+- lib/fullfeed/store/base.rb
+- lib/fullfeed/store/db_store.rb
+- lib/fullfeed/store/memory_store.rb
+- lib/fullfeed/store/store_factory.rb
+- test/load_files.rb
+- test/test_agent.rb
+- test/test_store.rb
 has_rdoc: false
 homepage: 
 post_install_message: 
@@ -68,6 +82,6 @@ rubyforge_project:
 rubygems_version: 1.2.0
 signing_key: 
 specification_version: 2
-summary: Fullfeed RSS creator
+summary: Create full text RSS feed from RSS
 test_files: []