grab_epg 0.2.1 → 0.2.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjJjOTExYzRhNzM5NjA1NjNhNjU1NTU2NjVmOWM3NTEyNDQ2MDZkMw==
+    MTI5MTMwMTY1NTRmZjk5NGIwZGM4MTk3NTljNGFjMzAwOWY4NTdiNA==
   data.tar.gz: !binary |-
-    ODlhNjQ2NDI0ZmQ1ZDU5M2M4YTA4NTg0ZWQ0ZmU0NWQwMDhjOTlhYg==
+    NzNjZTc3YzY1OWZkYTZjOGUzNTVjNzVmZjgzYjg3NjQyZWUzNGFmMg==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    NDg4MmNiYjYwMDk1NzMzMWIzMzJkNjQ0OGU2MjVhYTE5Y2RlZGE4NmM3YWE3
-    M2UxNmY2M2ZmZjIzZTY0MDQ0MTBhMTQ4Y2JmMTJhNTg1YTZmZjk5MWVlNjk4
-    YTdhODQ0NzU0MmU0OTk5MDczMmEzNGYyZTNkNDQzMWVjNjEyMDQ=
+    ZGMwNThmMzZlY2FmZmI1ZmQzNjY5ODdkYTI4MTk5MWI2NWZiODBlZjQ1YzNk
+    MzllOGE1YmNkZjRiNjc3MDlhY2FjZjMyNjFiNTcxYjFlZTlmYzgwNmVlMmQx
+    NzUxODIwMGE1MjgxZWM0NWY3ZDlmNWE0YmYyN2U0NTY1YjU3NmQ=
   data.tar.gz: !binary |-
-    MWUxNTFkNjRmZDE0Y2VhYjRhMWRmNjcyMmNlZDc5NWVmYWEzYTg5ZDhkOGY5
-    YTQ3OTBhMmVmMzY2ZTY5NjIwMjQ0MzM3NDAwNjQyZTdmNDQ0NTc0NGQzZTUx
-    OTdmMjM1ZTQwNDhjN2VjYTQ5MWJkMWQ0MTFmOWU5NDMzZjllZGE=
+    MTA3MmE5MGZkMzU5YzliYjljNTQ1NzljNWViYTQ5YWU5ZmNlZGQ5OWJmZTUz
+    YmZiN2QyNjUzOTk0OGQwMzM0NmZjOTEwOTI2MzJkZjAxMDg5YzdlNzUxNjM3
+    Y2VkNmQzMGUyMjQ0Nzc5MTZkMGE5NjY4Y2IwZTY2ZGI5Y2MyOTA=

data/.grabepg.gemspec

@@ -10,6 +10,6 @@ Gem::Specification.new do |gem|
   gem.files         = `git ls-files`.split($\)
   gem.name          = "grab_epg"
   gem.require_paths = ["lib"]
-  gem.version       = "0.2.1"
+  gem.version       = "0.2.3"
   gem.homepage      = "https://github.com/hahazql/grab_epg"
 end

data/lib/grab_tvmao.rb

@@ -264,7 +264,7 @@ module GrabTvmao
       proxy = proxylist[@proxyindex+1]
     end
     begin
-      doc = Nokogiri::HTML(open(url,:proxy=>"http://#{proxy}")) unless proxy.nil?||proxy.empty?
+      doc = Nokogiri::HTML(open(url,:proxy=>"#{proxy}")) unless proxy.nil?||proxy.empty?
       doc = Nokogiri::HTML(open(url)) if proxy.nil?||proxy.empty?
       @no_firest = 0
     rescue => err
@@ -274,15 +274,12 @@ module GrabTvmao
       end
 
       @no_firest += 1
-      p "*************************Proxy:#{proxy}, url:#{url}"
+      p "*************************Proxy:#{proxy}, url:#{url} Error:#{err.to_s}"
       #proxylist.delete(proxy)    #删除出错的代理  但如果是此网页错误则会引起BUG待修复
       get_doc_with_proxy(proxylist,url) if @no_firest<4
       raise RuntimeError,"Error: #{err.to_s}" unless @no_firest<4
     end
     @proxyindex += 1
-    unless doc
-      p "*************************Proxy:#{proxy}, url:#{url}"
-    end
     doc
   end
 

data/lib/grabepg/grab_base.rb

@@ -119,8 +119,28 @@ module Grabepg
     end
 
 
+    def err_doc_proxy(proxy,proxylist,url="",err="")
+      if proxy.empty?||proxy.nil?
+        proxylist.delete_at[@proxyindex]
+      end
+
 
+      unless @no_firest
+        @no_firest = 0
+      end
 
+      @no_firest += 1
+      p "*************************Proxy:#{proxy}, url:#{url} Error:#{err}"
+      #proxylist.delete(proxy)    #删除出错的代理  但如果是此网页错误则会引起BUG待修复
+      @proxyindex += 1
+      @proxyindex=@proxyindex%@size
+      doc=get_doc_with_proxy(proxylist,url) if @no_firest<4
+      unless @no_firest<4
+        @no_firest=0
+        raise RuntimeError,"Error: #{err}"
+      end
+      doc
+    end
 
 
     #使用代理获取url的html的doc值
@@ -129,6 +149,7 @@ module Grabepg
         unless @proxyindex
           @proxyindex = 0
         end
+        @size = proxylist.size
         @proxyindex=@proxyindex%proxylist.size
         if(proxylist[@proxyindex])
           proxy = proxylist[@proxyindex]
@@ -136,35 +157,23 @@ module Grabepg
           proxy = proxylist[@proxyindex+1]
         end
         begin
-          doc = Nokogiri::HTML(open(url,:proxy=>"http://#{proxy}")) unless proxy.nil?||proxy.empty?
-          unless doc
-            raise RuntimeError,"DOC is nil"
+          doc = Nokogiri::HTML(open(url,:proxy=>"#{proxy}")) unless proxy.nil?||proxy.empty?
+          if doc.nil?
+            doc=err_doc_proxy(proxy,proxylist,url,"doc nil")
+            @no_firest=0
           end
           @no_firest = 0
         rescue => err
-          if proxy.empty?||proxy.nil?
-            proxylist.delete_at[@proxyindex]
-          end
-
-
-          unless @no_firest
-            @no_firest = 0
-          end
-
-          @no_firest += 1
-          p "*************************Proxy:#{proxy}, url:#{url}"
-          #proxylist.delete(proxy)    #删除出错的代理  但如果是此网页错误则会引起BUG待修复
+          p "IN Rescue"
+          doc=err_doc_proxy(proxy,proxylist,url,err.to_s)
+          @no_firest=0
+          p "Get DOC"
           @proxyindex += 1
-          get_doc_with_proxy(proxylist,url) if @no_firest<4
-          unless @no_firest<4
-            @no_firest=0
-          	raise RuntimeError,"Error: #{err.to_s}"
-          end
+          @proxyindex=@proxyindex%@size
+          return doc
         end
         @proxyindex += 1
-        unless doc
-          p "*************************Proxy:#{proxy}, url:#{url}"
-        end
+        @proxyindex=@proxyindex%@size
       else
         begin
           doc = Nokogiri::HTML(open(url)) if proxy.nil?||proxy.empty?

data/lib/grabepg/grab_tvsou.rb

@@ -78,6 +78,7 @@ module Grabepg
 
 
       doc = @grabbase.get_doc_with_proxy(@proxy_list,@home_page)
+      begin
       doc.css("li").each do |li|
         case ChannelTypeMap[li.get_attribute("class")]
           when "央视"
@@ -90,6 +91,15 @@ module Grabepg
 
         end
       end
+      @error_num=0
+      rescue
+        unless @error_num
+          @error_num = 0
+        end
+        @error_num+=1
+        raise err.to_s  if @error_num==5
+        dispose_home_page
+      end
       return @channels
     end
 
@@ -123,7 +133,17 @@ module Grabepg
    def dispose_schedule_page(url,start_time,use_time)
      url = @site +"/"+url
      urls = url.split("?")
+     begin
      doc = @grabbase.get_doc_with_proxy(@proxy_list,url)
+     @error_num = 0
+     rescue => err
+       unless @error_num
+         @error_num = 0
+       end
+       @error_num+=1
+       raise err.to_s  if @error_num==5
+       dispose_schedule_page(url,start_time,use_time)
+     end
      _url = doc.css("div[class='week']")[0].css('a')[0].get_attribute("href")
      _url = urls[0]+_url
      urls = dispose_href_schedule_data(_url,start_time,use_time)
@@ -135,7 +155,9 @@ module Grabepg
        if url
          doc = @grabbase.get_doc_with_proxy(@proxy_list,url[:url])
          schedules = []
-         doc.css('div[class="time"]')[0].css("li[class='gray']").each do |schedule|
+         div = doc.css('div[class="time"]')[0]
+         if div
+         div.css("li[class='gray']").each do |schedule|
            begin
              _dispose = schedule.content
              _dispose_show =schedule.css("span")[0].text
@@ -153,6 +175,9 @@ module Grabepg
            end
          end
          ret.merge!({url[:date]=>schedules})
+         else
+           p "Error In this url: #{url} couldn't get doc.css('div[class=time]')[0]"
+         end
       end
      end
      return ret
@@ -165,11 +190,21 @@ module Grabepg
     #解析节目详情页面
     def dispose_show_info(url)
       doc = @grabbase.get_doc_with_proxy(@proxy_list,url)
+      begin
       show_name = doc.css('div[class="tv_info_top"]')[0].content
       _doc=doc.css("div[class='tv_info']")
       img_url = _doc.css("img")[0].get_attribute("src").gsub(" ","")
       show_info = _doc.css("p")[0].content.gsub("[全文]","")
+      @error_num = 0
       {show_name:show_name,img_url:img_url,show_info:show_info}
+      rescue => err
+        unless @error_num
+          @error_num = 0
+        end
+        @error_num+=1
+        raise err.to_s  if @error_num==5
+        dispose_show_info(url)
+      end
     end
 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grab_epg
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.3
 platform: ruby
 authors:
 - hahazql
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-03 00:00:00.000000000 Z
+date: 2013-06-04 00:00:00.000000000 Z
 dependencies: []
 description: ! '"用于抓取EPG信息"'
 email: