grab_epg 0.1.6 → 0.1.8

data/lib/grabepg/grab_base.rb

@@ -0,0 +1,173 @@
+#encoding:utf-8
+require 'nokogiri'
+
+module Grabepg
+
+  class GrabBase
+    # To change this template use File | Settings | File Templates.
+
+
+    def conversion_what_day(whatday)
+      ret = "星期"
+      case whatday.to_i
+        when 1
+          ret += "一"
+        when 2
+          ret += "二"
+        when 3
+          ret += "三"
+        when 4
+          ret += "四"
+        when 5
+          ret += "五"
+        when 6
+          ret += "六"
+        when 7
+          ret += "七"
+      end
+      ret
+    end
+
+
+    #获取指定访问速度的代理服务器
+    #time为最慢速度的时间 int型 代表秒
+    def self.get_topfast_list(use_time)
+      fast_list = []
+      time_use = 0
+      ips_ports = get_proxy_list()
+      ips_ports = get_proxylist_dianxin if ips_ports.size==0
+      ips_ports.each do |ip_port|
+        time_start = Time.now.to_i
+        begin
+          timeout(use_time) do
+            doc = Nokogiri::HTML(open("http://www.baidu.com",:proxy=> "http://#{ip_port}"))
+          end
+          time_end = Time.now.to_i
+          time_use = time_end - time_start
+          p  "http://#{ip_port}   use_time:#{time_use}"
+        rescue Exception =>e
+          case e
+            when Errno::ETIMEDOUT
+              p "Use http://#{ip_port} timeout"
+            when Timeout::Error
+              p "Use http://#{ip_port} timeout"
+            when Errno::ECONNREFUSED
+              p "Use http://#{ip_port} Error connection"
+            else
+              p "Use http://#{ip_port} Error:#{e.to_s}"
+          end
+          time_use = -1
+        end
+        if(time_use > 0 &&time_use < 8)
+          fast_list << ip_port
+        end
+      end
+      fast_list
+    end
+
+    #获取代理列表
+    def self.get_proxy_list()
+      list = gg('http://www.proxycn.cn/html_proxy/30fastproxy-1.html')
+      if list.count ==0
+        list = gg('http://www.proxycn.cn/html_proxy/http-1.html')
+      end
+      ips_ports = []
+      regex_port = /(?<=<TD class="list">)[0-9]*?(?=<\/TD>)/
+      regex_ip   = /(?<=a href\=whois.php\?whois\=)[0-9,.]*/
+      list.each do |proxy_txt|
+        port = proxy_txt[regex_port]
+        ip = proxy_txt[regex_ip]
+        if(ip != ""&& !port.to_s.eql?('3128'))
+          port_ip = ip.to_s + ":" + port.to_s
+          ips_ports << port_ip
+        end
+      end
+      p "Count: #{ips_ports.count}"
+      ips_ports
+    end
+
+    def self.gg(url)
+      regex_list = /<TD class="list">.*<\/TD>/
+      href =URI.parse(url)
+      contxt = ""
+      href.open{ |f|
+        f.each_line {|line| contxt =contxt + line + "\n"}
+      }
+      list = contxt.scan(regex_list)
+    end
+
+
+
+    def self.get_proxylist_dianxin()
+      list = gg("http://www.proxycn.cn/countryDX.php")
+      if list.count ==0
+        list = gg('http://www.proxycn.cn/html_proxy/http-1.html')
+      end
+      ips_ports = []
+      regex_port = /(?<=<TD class="list">)[0-9]*?(?=<\/TD>)/
+      regex_ip   = /(?<=a href\=whois.php\?whois\=)[0-9,.]*/
+      list.each do |proxy_txt|
+        port = proxy_txt[regex_port]
+        ip = proxy_txt[regex_ip]
+        if(ip != ""&& !port.to_s.eql?('3128'))
+          port_ip = ip.to_s + ":" + port.to_s
+          ips_ports << port_ip
+        end
+      end
+      p "Count: #{ips_ports.count}"
+      ips_ports
+    end
+
+
+
+
+
+
+    #使用代理获取url的html的doc值
+    def get_doc_with_proxy(proxylist,url)
+      unless proxylist.nil?||proxylist.empty?
+        unless @proxyindex
+          @proxyindex = 0
+        end
+        @proxyindex=@proxyindex%proxylist.size
+        if(proxylist[@proxyindex])
+          proxy = proxylist[@proxyindex]
+        else
+          proxy = proxylist[@proxyindex+1]
+        end
+        begin
+          doc = Nokogiri::HTML(open(url,:proxy=>"http://#{proxy}")) unless proxy.nil?||proxy.empty?
+          @no_firest = 0
+        rescue => err
+          if proxy.empty?||proxy.nil?
+            proxylist.delete_at[@proxyindex]
+          end
+
+
+          unless @no_firest
+            @no_firest = 0
+          end
+
+          @no_firest += 1
+          p "*************************Proxy:#{proxy}, url:#{url}"
+          #proxylist.delete(proxy)    #删除出错的代理  但如果是此网页错误则会引起BUG待修复
+          @proxyindex += 1
+          get_doc_with_proxy(proxylist,url) if @no_firest<4
+          raise RuntimeError,"Error: #{err.to_s}" unless @no_firest<4
+        end
+        @proxyindex += 1
+        unless doc
+          p "*************************Proxy:#{proxy}, url:#{url}"
+        end
+      else
+        begin
+          doc = Nokogiri::HTML(open(url)) if proxy.nil?||proxy.empty?
+        rescue => err
+          raise RuntimeError,"Error: #{err.to_s} Method:get_doc_with_proxy"
+        end
+      end
+      doc
+    end
+
+  end
+end

data/lib/grabepg/grab_tvsou.rb

@@ -0,0 +1,170 @@
+#encoding:utf-8
+
+require File.expand_path("../grab_base.rb", __FILE__)
+module Grabepg
+  # To change this template use File | Settings | File Templates.
+
+
+  class GrabTvsou
+    include Grabepg
+    #首页
+    attr_reader :home_page
+
+    #代理列表
+    attr_reader :proxy_list
+
+    attr_reader :grabbase
+
+    #频道存储
+    attr_reader :channels
+
+    #时间表存储
+    attr_reader :schedules
+
+    #俩个节目间的最小间隔时间
+    attr_reader :default_min_interval
+
+
+    ChannelTypeMap = {"yg_ys_li"=>"央视","yg_ws_li"=>"卫视","yg_hw_li"=>"海外","yg_df_li"=>"地方"}
+
+    #type 从mobie还是网站接口抓取数据
+    def initialize(grabtype,proxy_list)
+      @home_page = get_url(grabtype)
+      @proxy_list = proxy_list
+      @grabbase = GrabBase.new
+      @channels = {}
+      @site="http://m.tvsou.com"
+    end
+
+    #获取从tvsou的什么网站上获取
+    #type: mobile,webpage
+    def get_url(type)
+      return "http://m.tvsou.com/index.asp" if type.eql?("mobile")
+    end
+
+    def get_data_year_month_day(time)
+
+      return {time:"#{time.year}-#{time.month}-#{time.day}",date:"#{@grabbase.conversion_what_day(time.wday)}(#{time.month}-#{time.day})"}
+    end
+
+    #获取时间
+    #start_time 时间起始点
+    #use_time   天数
+    def get_data(start_time,use_time)
+      time = Time.now+start_time*24*60*60
+      ret = []
+      use_time.times.each do |i|
+        _time = time + i*24*60*60
+        ret << get_data_year_month_day(_time)
+      end
+      ret
+    end
+
+
+    #对首页进行处理获取部分频道的URL和嘻嘻
+    def dispose_home_page
+
+      get_channellist = lambda { |li,type|
+        channellist = {}
+        li.css('a').each do |a|
+          channellist.merge!({a.content=>{url:a.get_attribute("href"),type:type}}) unless channellist.has_key?(a.content)
+        end
+        channellist
+      }
+
+
+      doc = @grabbase.get_doc_with_proxy(@proxy_list,@home_page)
+      doc.css("li").each do |li|
+        case ChannelTypeMap[li.get_attribute("class")]
+          when "央视"
+            @channels.merge!(get_channellist.call(li,"CCTV"))
+          when "卫视"
+            @channels.merge!(get_channellist.call(li,"WTV"))
+          when "海外"
+
+          when "地方"
+
+        end
+      end
+      return @channels
+    end
+
+
+    #获取频道列表
+    #url是获取频道列表的首页
+    #地方需要调用此函数
+    def dispose_channel_page(url,channel_type)
+
+    end
+
+
+
+    #获取频道时间表URL
+    def dispose_href_schedule_data(href,start_time,use_time)
+     hrefs=href.split("&programDT=")
+     _hrefs=hrefs[1].split("&")
+     ret = []
+     get_data(start_time,use_time).each do |time|
+       _hrefs[0]=time[:time]
+       url = hrefs[0]+"&programDT=" + time[:time]
+       1.upto(_hrefs.length-1).each do |i|
+         url += "&"+_hrefs[i]
+       end
+       ret<<{url:url,time:time[:time],date:time[:date]}
+     end
+     ret
+    end
+
+   #根据URL解析时间表页面
+   def dispose_schedule_page(url,start_time,use_time)
+     url = @site +"/"+url
+     urls = url.split("?")
+     doc = @grabbase.get_doc_with_proxy(@proxy_list,url)
+     _url = doc.css("div[class='week']")[0].css('a')[0].get_attribute("href")
+     _url = urls[0]+_url
+     urls = dispose_href_schedule_data(_url,start_time,use_time)
+     ret = {}
+     last_time = -5
+     last_schedule = {}
+     urls.each do |url|
+       p "Grab url: #{url}"
+       if url
+         doc = @grabbase.get_doc_with_proxy(@proxy_list,url[:url])
+         schedules = []
+         doc.css('div[class="time"]')[0].css("li[class='gray']").each do |schedule|
+           begin
+             _dispose = schedule.content
+             _dispose_show =schedule.css("span")[0].text
+             time = _dispose.gsub(_dispose_show,"")
+             _url = @site + schedule.css('a')[0].get_attribute("href") if schedule.css('a')[0]
+             schedules << {time:time,schedule_name:_dispose_show.delete(" 剧情"),url:_url}
+             now = time.gsub(":","").to_i
+             if((now-last_time)<5)
+               schedules.delete(last_schedule)
+             end
+             last_schedule = {time:time,schedule_name:_dispose_show.gsub(" 剧情",""),url:_url}
+             last_time = now
+           rescue => err
+             p "Schedule: #{schedule}"
+           end
+         end
+         ret.merge!({url[:date]=>schedules})
+      end
+     end
+     return ret
+   end
+
+    #解析节目详情页面
+    def dispose_show_info(url)
+      doc = @grabbase.get_doc_with_proxy(@proxy_list,url)
+      show_name = doc.css('div[class="tv_info_top"]')[0].content
+      _doc=doc.css("div[class='tv_info']")
+      img_url = _doc.css("img")[0].get_attribute("src").gsub(" ","")
+      show_info = _doc.css("p")[0].content.gsub("[全文]","")
+      {show_name:show_name,img_url:img_url,show_info:show_info}
+    end
+
+
+  end
+
+end