http_crawler 0.3.1.27 → 0.3.1.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3236b0e130a26ee4e07238719f7bcbe70e72ae78729bc3d5dc5149c072a1b03
-  data.tar.gz: adf5773258b3863038858e6086ecb39fce1f2d6fb09f292ea619e1552ab86011
+  metadata.gz: 809654bd670ca7a1586478e45467c162d12ab5656d5ac32f0072beea5de967ee
+  data.tar.gz: 93c0efeeb3737db3184d3ca93d0bd9ced39ad13e52f8b73ae30b874786504d9b
 SHA512:
-  metadata.gz: 5170bfaba6df0ce2e14372f3e3e83e773d112dfd9c29060f9ca3b72b40a3dada05f040d24e8408aa4ff27575c347927fc47e7e2fc6598ecbb0e8ebf65d70c13b
-  data.tar.gz: 85161b1c474689cef0fcb57118caf701c642f4896381be1bbf1f3640889f7f950c7b53124cc7229225834a8a372daec6ec30ec55d0d7366d6599d2d9df7e0162
+  metadata.gz: c9752c6d3ab7203fdb6058340dd049288217fb91a3cfa459055d369d522218a37d228a7ac47ee33f904f660dab6faeec7cba25f4a30123bd2327d5c30a8293d3
+  data.tar.gz: 8bf4fa8399ffe7fae9d7e6f256614ce23c9757772b165e319330078069dc76349cb5dd1dfffe9e44dd6ecf5ebf82572c49e4fc843314ed238cb95f8880f062f6

data/.idea/workspace.xml

@@ -95,8 +95,8 @@
       <file leaf-file-name="client.rb" pinned="false" current-in-tab="true">
         <entry file="file://$PROJECT_DIR$/lib/http_crawler/client.rb">
           <provider selected="true" editor-type-id="text-editor">
-            <state relative-caret-position="201">
-              <caret line="208" selection-start-line="208" selection-end-line="208" />
+            <state relative-caret-position="273">
+              <caret line="212" column="11" lean-forward="true" selection-start-line="212" selection-start-column="11" selection-end-line="212" selection-end-column="11" />
             </state>
           </provider>
         </entry>
@@ -165,7 +165,7 @@
   <component name="NodePackageJsonFileManager">
     <packageJsonPaths />
   </component>
-  <component name="ProjectFrameBounds" fullScreen="true">
+  <component name="ProjectFrameBounds" extendedState="6" fullScreen="true">
     <option name="y" value="23" />
     <option name="width" value="1280" />
     <option name="height" value="777" />
@@ -175,6 +175,7 @@
       <foldersAlwaysOnTop value="true" />
     </navigator>
     <panes>
+      <pane id="Scope" />
       <pane id="ProjectPane">
         <subPane>
           <expand>
@@ -197,7 +198,6 @@
           <select />
         </subPane>
       </pane>
-      <pane id="Scope" />
     </panes>
   </component>
   <component name="PropertiesComponent">
@@ -253,29 +253,29 @@
       <workItem from="1557137463254" duration="382000" />
       <workItem from="1557156104186" duration="1815000" />
       <workItem from="1557160216202" duration="138000" />
-      <workItem from="1563360666497" duration="12000" />
+      <workItem from="1563360666497" duration="431000" />
+      <workItem from="1563361538580" duration="6000" />
     </task>
     <servers />
   </component>
   <component name="TimeTrackingManager">
-    <option name="totallyTimeSpent" value="33040000" />
+    <option name="totallyTimeSpent" value="33465000" />
   </component>
   <component name="ToolWindowManager">
-    <frame x="0" y="0" width="1680" height="1050" extended-state="0" />
-    <editor active="true" />
+    <frame x="0" y="0" width="1680" height="1050" extended-state="6" />
     <layout>
-      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.12576313" />
+      <window_info active="true" content_ui="combo" id="Project" order="0" visible="true" weight="0.0964591" />
       <window_info anchor="bottom" id="TODO" order="6" />
       <window_info anchor="bottom" id="Docker" order="7" show_stripe_button="false" />
       <window_info anchor="bottom" id="Event Log" order="7" side_tool="true" />
       <window_info anchor="right" id="Database" order="3" />
       <window_info anchor="bottom" id="Database Changes" order="7" show_stripe_button="false" />
-      <window_info anchor="bottom" id="Run" order="2" />
       <window_info anchor="bottom" id="Version Control" order="7" />
+      <window_info anchor="bottom" id="Run" order="2" />
       <window_info id="Structure" order="1" side_tool="true" weight="0.25" />
-      <window_info anchor="bottom" id="Terminal" order="7" visible="true" weight="0.16410257" />
-      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
+      <window_info anchor="bottom" id="Terminal" order="7" visible="true" weight="0.11794872" />
       <window_info id="Favorites" order="2" side_tool="true" />
+      <window_info anchor="bottom" id="Debug" order="3" weight="0.4" />
       <window_info anchor="right" content_ui="combo" id="Hierarchy" order="2" weight="0.25" />
       <window_info anchor="bottom" id="Inspection" order="5" weight="0.4" />
       <window_info anchor="right" id="Commander" order="0" weight="0.4" />
@@ -589,8 +589,8 @@
     </entry>
     <entry file="file://$PROJECT_DIR$/lib/http_crawler/client.rb">
       <provider selected="true" editor-type-id="text-editor">
-        <state relative-caret-position="201">
-          <caret line="208" selection-start-line="208" selection-end-line="208" />
+        <state relative-caret-position="273">
+          <caret line="212" column="11" lean-forward="true" selection-start-line="212" selection-start-column="11" selection-end-line="212" selection-end-column="11" />
         </state>
       </provider>
     </entry>

data/lib/http_crawler/client.rb

@@ -61,7 +61,7 @@ module HttpCrawler
     attr_accessor :max_error_num
     # 最大错误重试次数
     def max_error_num
-      @max_error_num ||= 1
+      @max_error_num ||= 3
     end
 
     attr_reader :uri
@@ -334,13 +334,7 @@ module HttpCrawler
         else
           # 每次错误次数尝试 -1
           n -= 1
-          case error
-          when HTTP::TimeoutError
-            # 超时错误切换代理
-            raise error unless self.update_proxy?
-          else
-            raise error unless self.update_proxy?
-          end
+          self.update_proxy?
           retry
         end
       end

data/lib/http_crawler/http/response.rb

@@ -8,9 +8,7 @@ module HTTP
       # 数据解压
       case self.headers['Content-Encoding']
       when 'gzip' then
-        sio = StringIO.new(self.body.to_s)
-        gz = Zlib::GzipReader.new(sio)
-        @decoding_body = gz.read()
+        @decoding_body = Zlib::GzipReader.new(StringIO.new(self.body.to_s), encoding: "ASCII-8BIT").read
       when 'br'
         @decoding_body = Brotli.inflate(self.body.to_s)
         # when 'deflate'
@@ -35,12 +33,12 @@ module HTTP
 
       # 进行转码
       begin
-        @decoding_body.force_encoding(encoding).encode!('utf-8') if encoding && encoding != @decoding_body.encoding
+        @decoding_body.force_encoding(encoding).encode!('utf-8',invalid: :replace) if encoding && encoding != @decoding_body.encoding
       rescue => e
         # 转码错误后再次使用 CharDet 判断编码格式后进行转码
         cd = CharDet.detect(@decoding_body)["encoding"]
         if (cd && cd != encoding)
-          @decoding_body.force_encoding(cd).encode!('utf-8') if encoding != @decoding_body.encoding
+          @decoding_body.force_encoding(cd).encode!('utf-8',invalid: :replace) if encoding != @decoding_body.encoding
         else
           # 还是转码错误则抛出源码转字符串内容
           self.body.to_s

data/lib/http_crawler/version.rb

@@ -1,3 +1,3 @@
 module HttpCrawler
-  VERSION = "0.3.1.27"
+  VERSION = "0.3.1.28"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: http_crawler
 version: !ruby/object:Gem::Version
-  version: 0.3.1.27
+  version: 0.3.1.28
 platform: ruby
 authors:
 - jagger
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-07-17 00:00:00.000000000 Z
+date: 2019-07-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec