gozap_rss 0.0.2 → 0.0.3

data/Gemfile

@@ -6,7 +6,7 @@ source "http://rubygems.org"
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 gem "logger"
-
+gem "typhoeus"
 gem "sanitize", ">=2.0.3"
 
 

data/Gemfile.lock

@@ -8,6 +8,7 @@ GEM
       git (>= 1.2.5)
       rake
     logger (1.2.8)
+    mime-types (1.17.2)
     multi_json (1.0.4)
     nokogiri (1.5.0)
     rake (0.9.2.2)
@@ -25,6 +26,8 @@ GEM
       multi_json (~> 1.0.3)
       simplecov-html (~> 0.5.3)
     simplecov-html (0.5.3)
+    typhoeus (0.3.3)
+      mime-types
 
 PLATFORMS
   ruby
@@ -36,3 +39,4 @@ DEPENDENCIES
   rspec (~> 2.7.0)
   sanitize (>= 2.0.3)
   simplecov
+  typhoeus

data/lib/gozap_rss/chouti_rss.rb

@@ -3,7 +3,6 @@
 module GozapRss
 
 
-
   class ChoutiRssBase
     def self.logger
       @logger || GozapRss.logger || Logger.new(STDOUT)
@@ -22,13 +21,18 @@ module GozapRss
       self.class.logger
     end
 
+    def log_failed(response)
+      msg = "#{response.code} URL: #{response.request.url}  PARAMS: #{response.request.params.to_s} in #{response.time}s FAILED :  #{response.curl_error_message}
+      BODY: #{response.body}"
+      logger.error(msg)
+    end
 
     def logger_exception e
       self.class.logger_exception e
     end
 
 
-    attr_reader :url, :description, :title, :pub_date
+    attr_reader :url, :description, :title, :pub_date, :ttl
     attr_accessor :http_headers_option
 
 
@@ -40,7 +44,7 @@ module GozapRss
     attr_reader :rss_items
 
     def initialize uri
-      @http_headers_option = {"User-Agent"=>"Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7"}
+      @http_headers_option = {"User-Agent" => "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7"}
       @url = uri
       @rss_items = []
       content = get_feed_content uri
@@ -48,7 +52,6 @@ module GozapRss
     end
 
 
-
     private
 
     def parse_rss content
@@ -58,12 +61,13 @@ module GozapRss
         @title = rss.channel.title.to_s.html_format
         @description = rss.channel.description.to_s.html_format
         @pub_date = rss.channel.pubDate
+        @ttl = rss.channel.ttl.to_i == 0 ? 10 * 60 : rss.channel.ttl.to_i * 60
         @rss_items = []
         rss.items.each do |item|
           rss_item = ChoutiRssItem.new(item)
           @rss_items << rss_item if rss_item
         end
-        @rss_items.sort!{|a,b|b.pub_date <=>a.pub_date}
+        @rss_items.sort! { |a, b| b.pub_date <=>a.pub_date }
       rescue Exception => e
         logger_exception e
       end
@@ -73,15 +77,29 @@ module GozapRss
 
     #because some site feed refuse rss robot, so i set the http headers User-Agent to disguise as a browser
     def get_feed_content uri
+
       content = ""
       begin
-        open(uri, @http_headers_option) do |s|
-          content = s.read
+        response = Typhoeus::Request.get(uri,
+                                         :headers['User-Agent'] => "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7"\
+                                                    "(KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7",
+                                         :timeout => 120000,
+                                         :max_redirects => 3,
+                                         :follow_location => true
+        )
+
+        if response.success? || (response.code < 400 && response.code >= 301)
+          content = response.body
+          isutf8 = Kconv.isutf8(content)
+          content = Iconv.iconv("UTF-8//IGNORE", "GB2312//IGNORE", content)[0] unless isutf8
+        elsif response.timed_out?
+          log_failed response
+        elsif response.code == 0
+          log_failed(response)
+        else
+          log_failed(response)
         end
-
-        isutf8 = Kconv.isutf8(content)
-        content = Iconv.iconv("UTF-8//IGNORE", "GB2312//IGNORE", content)[0] unless isutf8
-      rescue Exception=>e
+      rescue Exception => e
         logger_exception e
       end
       return content
@@ -95,10 +113,10 @@ module GozapRss
 
     def initialize item
       @title = item.title.to_s.html_format
-      @pub_date = item.pubDate ||  item.lastBuildDate
-      @description  = item.description.to_s.html_format
-      @url  = item.link.to_s.strip
-      @url_md5 =  Digest::MD5.hexdigest(@url)
+      @pub_date = item.pubDate || item.lastBuildDate
+      @description = item.description.to_s.html_format
+      @url = item.link.to_s.strip
+      @url_md5 = Digest::MD5.hexdigest(@url)
       unless validate
         logger.error "parser item error -- title=>#{@title}, pub_date=>#{@pub_date} description=>#{@description}, url=>#{@url}"
         return nil
@@ -108,8 +126,8 @@ module GozapRss
 
     private
     def validate
-      !(@url.nil? || @description.nil? ||  @title.nil? ||
-        @url.empty? || @description.empty? || @title.empty?)
+      !(@url.nil? || @description.nil? || @title.nil? ||
+          @url.empty? || @description.empty? || @title.empty?)
     end
 
   end

data/lib/gozap_rss/version.rb

@@ -1,3 +1,3 @@
 module GozapRss
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/gozap_rss.rb

@@ -9,6 +9,7 @@ require 'rss/2.0'
 require 'open-uri'
 require "digest/md5"
 require "sanitize"
+require "typhoeus"
 
 
 

metadata

@@ -2,7 +2,7 @@
 name: gozap_rss
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors: 
 - "\xE7\x8E\x8B\xE6\x98\x8E\xE5\x8D\x8E"
@@ -24,19 +24,30 @@ dependencies:
   prerelease: false
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: sanitize
+  name: typhoeus
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 2.0.3
+        version: "0"
   type: :runtime
   prerelease: false
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: sanitize
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.3
+  type: :runtime
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -44,10 +55,10 @@ dependencies:
         version: 2.7.0
   type: :development
   prerelease: false
-  version_requirements: *id003
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: bundler
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -55,10 +66,10 @@ dependencies:
         version: 1.0.0
   type: :development
   prerelease: false
-  version_requirements: *id004
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: jeweler
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -66,10 +77,10 @@ dependencies:
         version: 1.6.4
   type: :development
   prerelease: false
-  version_requirements: *id005
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: simplecov
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -77,7 +88,7 @@ dependencies:
         version: "0"
   type: :development
   prerelease: false
-  version_requirements: *id006
+  version_requirements: *id007
 description: "\xE6\x8A\x93\xE5\x8F\x96RSS\xE6\x9C\x8D\xE5\x8A\xA1\xE7\x9A\x84\xE7\xAE\x80\xE5\x8D\x95\xE5\xBA\x94\xE7\x94\xA8"
 email: wangmh.bit@gmail.com
 executables: []
@@ -112,7 +123,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -1037996241
+      hash: -27366127
       segments: 
       - 0
       version: "0"