http_crawler 0.3.1.12 → 0.3.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a3ad2fc0f149033558a66e87a8aa5f15b39aab1ce651b09ea8e8c86f99393d2
-  data.tar.gz: fbcde9c6aed889e7dd1386499dc70efbc82e1415630a0ffb7eaf6b343061ae54
+  metadata.gz: b691ef837786382eeaae3b6368bbd6709ddc2d4fbee03fac819fa6e691f89e03
+  data.tar.gz: c5e94dd28381d78dd2798dc74f9d83498fa33e2c173b7edc62f7bd73a8692748
 SHA512:
-  metadata.gz: 1d77e4285837d83229131b57b78fa13dd42aefb6465a8346ed06eca430bacfe31593e4ff05a25bb4637ac52ba9ab4eb4dd05da514c413cd5b0232b45a5be2a14
-  data.tar.gz: cd974847c9f3f5cdc0a4576bd1932267c6c7516fff1aaddc6e1571735ae5a4e9b5f19b9acfad816ce99ab1bac9ce1cab0314412b3fbc34cfc3907cfb05a67c8c
+  metadata.gz: e05f70823be3fb9e88768f508f726c1d2769fd255df1eca0007748854deb4b123623cc660b034cee72273014effbde05a860b398b87e3fad7a2d794c63adb2e1
+  data.tar.gz: ce6b3a0f35a2e70adf9a6747a2808ebe066d35c988a186e910979353553dd882ae3599285cf73e6bffcf4f7688b37f43df27410e693e68cf7700f7eccfcb21e6

data/lib/http_crawler/client.rb

@@ -309,7 +309,12 @@ module HttpCrawler
       raise "必须定义块" unless block_given?
       n = max_error_num
       begin
-        block.call
+        r = block.call
+        if r.status.success?
+          return r
+        else
+          raise "请求失败(#{r.code}):#{r.uri.to_s}"
+        end
       rescue => error
         Rails.logger.debug error.class
         case error

data/lib/http_crawler/common.rb

@@ -1,2 +1,3 @@
 require_dependency File.dirname(__FILE__) + '/common/object.rb'
 require_dependency File.dirname(__FILE__) + '/common/string.rb'
+require_dependency File.dirname(__FILE__) + '/common/integer.rb'

data/lib/http_crawler/common/integer.rb

@@ -0,0 +1,9 @@
+class Integer
+  def to_time
+    if self >= 1000000000000 && self < 10000000000000
+      return Time.at(self / 1000.0)
+    else
+      return Time.at(self)
+    end
+  end
+end

data/lib/http_crawler/common/string.rb

@@ -3,7 +3,7 @@ class String
   # 清除包含: 空格,回车
   #
   def del_inter
-    self.gsub(/(?:\n|\t|\r| )/, "")
+    self.gsub(/(?:\n|\t|\r| |　)/, "")
   end
 
   # 转换成时间格式
@@ -20,6 +20,14 @@ class String
         "%Y-%m-%d %H:%M:%S",
         "%Y-%m-%d%H:%M",
         "%Y-%m-%d %H:%M",
+
+        "%Y-%m-%d",
+        "%Y年%m月%d日",
+        "%Y%m%d",
+
+        "%Y%m%d%H%M%S",
+        "%Y%m%d%H%M",
+
     ].each do |v|
       begin
         return Time.strptime(self, v)
@@ -28,6 +36,9 @@ class String
       end
     end
 
+    Time.at(self.to_i / 1000.0) if self.length == 13
+    Time.at(self.to_i) if self.length == 10
+
     # 最后用 Time通用类型尝试
     return Time.parse(self)
   end

data/lib/http_crawler/http/response.rb

@@ -60,7 +60,7 @@ module HTTP
     def html(data = nil)
 
       if (data.blank? && defined? @html)
-        # 如果 data 为空 并且 @json 有值，直接返回 @json
+        # 如果 data 为空 并且 @html 有值，直接返回 @html
         return @html
       end
 
@@ -117,7 +117,12 @@ module HTTP
     end
 
     # 获取正文内容
-    def content(data = readability.content)
+    def content(data = nil)
+      if(data.blank?)
+        data = readability.content
+      else
+        data = readability(data).content
+      end
       Nokogiri::HTML(data).text.del_inter
     end
 

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.1.12"
+  VERSION = "0.3.1.13"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.1.12
+  version: 0.3.1.13
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-03-23 00:00:00.000000000 Z
+date: 2019-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -134,6 +134,7 @@ files:
 - lib/http_crawler.rb
 - lib/http_crawler/client.rb
 - lib/http_crawler/common.rb
+- lib/http_crawler/common/integer.rb
 - lib/http_crawler/common/object.rb
 - lib/http_crawler/common/string.rb
 - lib/http_crawler/errors.rb