digger 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19e59bc2161a078d80d00adf538a7c33891a53f9beeb453748eec7e0810c5b65
-  data.tar.gz: da1d93c663b42a6e0b7be2f136bea3b0f3a86c3c36c57a72347c70e7e0538508
+  metadata.gz: ed96af1c5ae92569e1de4885958ac5852864c045f6c6337d5f17d91747d8ed80
+  data.tar.gz: 82003ae80f54cd3f9b805757e5dcb4c7894bba91c4c376cf08ebd43e6de6e80b
 SHA512:
-  metadata.gz: d13b06c8491f9cda42f8a8fd4fa9547aa6a95d075efbf52c789760adaa66d0c2bde24f810e23e4713df4362ae82cb82b0d61996386caaad2c685eb22f1a375db
-  data.tar.gz: a63a5cfe70b154b446dcbd0d2937d94a33fb033277c0d17aa1500a9ccd86dc8791e3cc40e8487ecc1c609eec304b91fb4a29b75b89a74b8efe24e29c49957d4e
+  metadata.gz: b7aad69fd46c7d1688026ece2e1efe14d7dea29b42f94656d794655e12a92677bd3e1034f0c776bf197bcd75c96bd49377df399433bd2e1c13507520af1addc5
+  data.tar.gz: 60055a69ec3ad77e80fc4f1b50bb3a6c298e2274827ae21b64d1d82dae53a1d5338ccc99bcdb09c7d2e946abd007a488ff3034a753bc54586e9b651aeb3c5ce7

data/lib/digger/index.rb

@@ -17,6 +17,35 @@ module Digger
       pattern.gsub('*').each_with_index { |_, i| arg[i] }
     end
 
+    def self.slow_down(entities, conf = {}, &block)
+      raise NoBlockError, 'No block given' unless block
+
+      config = {
+        sleep_range_seconds: 4...10, # 随机等待时间范围
+        fail_max_cnt: 10, # 最多失败次数
+        fail_unit_seconds: 10 * 60, # 失败等待时间
+        when_fail: ->(ent, e, failed_cnt) {}
+      }.merge(conf)
+      failed_cnt = 0
+      cursor = 0
+      result = []
+      while cursor < entities.length
+        begin
+          result << block.call(entities[cursor])
+        rescue StandardError => e
+          failed_cnt += 1
+          config[:when_fail].call(entities[cursor], e, failed_cnt)
+          break if failed_cnt >= config[:fail_max_cnt]
+
+          sleep(failed_cnt * config[:fail_unit_seconds])
+        else
+          cursor += 1
+          sleep(rand(config[:sleep_range_seconds]))
+        end
+      end
+      result
+    end
+
     def self.batch(entities, cocurrence = 1, &block)
       raise NoBlockError, 'No block given' unless block
 

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = '0.1.8'.freeze
+  VERSION = '0.1.9'.freeze
 end

data/spec/index_spec.rb

@@ -9,4 +9,20 @@ describe Digger::Index do
     end
     expect(pt.join).to eq(list.map { |num| "##{num}" }.join)
   end
+
+  it 'slow down' do
+    list = [1, 2, 3, 4]
+    conf = {
+      sleep_range_seconds: 1...2,
+      fail_unit_seconds: 1,
+      fail_max_cnt: 2,
+      when_fail: ->(_, e, nth) { puts "#{nth}: #{e.message}" }
+    }
+    pt = Digger::Index.slow_down(list, conf) do |num|
+      raise 'error' if num == 3
+      num
+    end
+    p pt
+    expect(pt.size).to eq(2)
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - binz