gozap_rss 0.0.4 → 0.0.6

data/lib/gozap_rss/chouti_rss.rb

@@ -2,6 +2,26 @@
 
 module GozapRss
 
+  class Error < StandardError; end
+
+  class GozapHttpTimeOutError < Error
+    def initialize(url)
+      super("http get #{url} get timeout")
+    end
+  end
+
+  class GozapHttpStatusError < Error
+    def initialize(url, code, msg)
+      super("http get #{url} return response_code is #{code} and error is #{msg}")
+    end
+  end
+
+  class GozapHttpReceiveError < Error
+    def initialize(url, msg)
+      super("http get #{url} error is #{msg}")
+    end
+  end
+
 
   class ChoutiRssBase
     def self.logger
@@ -41,12 +61,13 @@ module GozapRss
 
   class ChoutiRss < ChoutiRssBase
 
-    attr_reader :rss_items
+    attr_reader :items
 
     def initialize uri
       @http_headers_option = {"User-Agent" => "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7"}
       @url = uri
-      @rss_items = []
+      @items = []
+      @ttl = 120
       content = get_feed_content uri
       parse_rss(content)
     end
@@ -60,14 +81,11 @@ module GozapRss
         rss = RSS::Parser.parse(content, false)
         @title = rss.channel.title.to_s.html_format
         @description = rss.channel.description.to_s.html_format
-        @pub_date = rss.channel.pubDate
-        @ttl = rss.channel.ttl.to_i == 0 ? 10 * 60 : rss.channel.ttl.to_i * 60
-        @rss_items = []
+        @ttl =  rss.channel.ttl.to_i * 60 if (rss.channel.respond_to?(:ttl) && rss.channel.ttl.to_i > 0)
         rss.items.each do |item|
           rss_item = ChoutiRssItem.new(item)
-          @rss_items << rss_item if rss_item
+          @items << rss_item if rss_item
         end
-        @rss_items.sort! { |a, b| b.pub_date <=>a.pub_date }
       rescue Exception => e
         logger_exception e
       end
@@ -79,26 +97,35 @@ module GozapRss
     def get_feed_content uri
 
       content = ""
+      @retry = 3
       begin
         response = Typhoeus::Request.get(uri,
                                          :headers['User-Agent'] => "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7"\
                                                     "(KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7",
-                                         :timeout => 120000,
+                                         :timeout => 30000,
                                          :max_redirects => 3,
                                          :follow_location => true
         )
 
         if response.success? || (response.code < 400 && response.code >= 301)
           content = response.body
-          isutf8 = Kconv.isutf8(content)
-          content = Iconv.iconv("UTF-8//IGNORE", "GB2312//IGNORE", content)[0] unless isutf8
+          #isutf8 = Kconv.isutf8(content)
+          #content = content.encode("UTF-8", "GB2312")    unless isutf8
         elsif response.timed_out?
-          log_failed response
+          raise GozapHttpTimeOutError.new(uri)
         elsif response.code == 0
-          log_failed(response)
+          raise GozapHttpReceiveError.new(uri,response.curl_error_message)
         else
-          log_failed(response)
+          rails GozapHttpStatusError.new(uri, response.code, response.curl_error_message)
         end
+      rescue GozapHttpTimeOutError => e
+        logger_exception e
+        logger.info "#{@retry} #{uri}"
+        retry if  (@retry -= 1) > 0
+      rescue GozapHttpReceiveError => e
+        logger_exception e
+        logger.info "retry #{@retry} #{uri}"
+        retry if  (@retry -= 1) > 0
       rescue Exception => e
         logger_exception e
       end
@@ -109,16 +136,13 @@ module GozapRss
 
 
   class ChoutiRssItem < ChoutiRssBase
-    attr_reader :url_md5
 
     def initialize item
       @title = item.title.to_s.html_format
-      @pub_date = item.pubDate || item.lastBuildDate
       @description = item.description.to_s.html_format
       @url = item.link.to_s.strip
-      @url_md5 = Digest::MD5.hexdigest(@url)
       unless validate
-        logger.error "parser item error -- title=>#{@title}, pub_date=>#{@pub_date} description=>#{@description}, url=>#{@url}"
+        logger.error "parser item error -- title=>#{@title},  url=>#{@url}"
         return nil
       end
       self

data/lib/gozap_rss/gozap_ext.rb

@@ -1,6 +1,7 @@
 class String
   def html_format
-      str = Sanitize.clean(self);
-      str.gsub(/[\s]+?/, "")
+      #str = Sanitize.clean(self);
+      #str.gsub(/[\s]+?/, "")
+      self
   end
 end

data/lib/gozap_rss/version.rb

@@ -1,3 +1,3 @@
 module GozapRss
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

data/lib/gozap_rss.rb

@@ -1,7 +1,6 @@
 require 'rubygems'
 require 'logger'
 require 'json'
-require "mysql2"
 require 'kconv'
 require 'iconv'
 require 'rss/1.0'