embulk-input-jira 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a7ada5966316509edccb0d858aee0e74fc6907dc
-  data.tar.gz: 5e80e6f96afe4aef6994579abb49cd466e097a4b
+  metadata.gz: f18cc941dfa5acf3d4042b5bf94be8f56f839874
+  data.tar.gz: fe29da077a7423972da281afd02d9818ddbb1214
 SHA512:
-  metadata.gz: cb335f39601ae83421e5a5b3ce632d3be06c35592d21b290e68f74fe5f14c01c9a92947b0b28ed33bea4e8b6128d7e4746f17fdbbabc6d6513b4c27572f03640
-  data.tar.gz: 1508035dad8ad5c76c138bda382686f5bf90cc470e4359f01f637df93ea3cae4dfa21645e40a7ec98c1e755723bc3be058b7f76ae3a903925610c798d8b8cf75
+  metadata.gz: c65a216e19e5117bafc0b72a460c8705e19249aecfb5869cf92d2ee5f120d438f1dedcc34a947b92388b1c236542da39fbfcdc66d44a1c335662c4837dbac0a4
+  data.tar.gz: 1791e18b0d566961f64dfa20c854dfba3aa712732cf28974577e3ac903c6fcd9be287eab04279eefc8e3bc452e0ad73dae7a3da4de36de123a9c90422c9bbee1

data/.gitignore

@@ -5,3 +5,4 @@
 /Gemfile.lock
 /.ruby-version
 /coverage/
+/vendor/*

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.5 - 2018-11-27
+
+* [fixed] Fix infinitive 401 errors and dynamically adjust parallel threads [#52](https://github.com/treasure-data/embulk-input-jira/pull/52)
+
 ## 0.2.4 - 2017-11-17
 
 * [fixed] Fixed checking credentials by `Jiralicious User` [#51](https://github.com/treasure-data/embulk-input-jira/pull/51)

data/embulk-input-jira.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-jira"
-  spec.version       = "0.2.4"
+  spec.version       = "0.2.5"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Jira input plugin for Embulk"
   spec.description   = "Loads records from Jira."
@@ -14,6 +14,7 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'jiralicious', ['~> 0.5.0']
   spec.add_dependency 'parallel', ['~> 1.6.0']
+  spec.add_dependency 'ruby-limiter', ['~> 1.0']
   spec.add_dependency 'perfect_retry', ['~> 0.3']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['< 11.0']

data/lib/embulk/input/jira.rb

@@ -103,7 +103,7 @@ module Embulk
         last_page = (total_count.to_f / PER_PAGE).ceil
 
         0.step(total_count, PER_PAGE).with_index(1) do |start_at, page|
-          logger.debug "Fetching #{page} / #{last_page} page"
+          logger.info "Fetching #{page} / #{last_page} page"
           @retryer.with_retry do
             @jira.search_issues(@jql, options.merge(start_at: start_at)).each do |issue|
               values = @attributes.map do |(attribute_name, type)|

data/lib/embulk/input/jira_api/client.rb

@@ -1,5 +1,6 @@
 require "jiralicious"
 require "parallel"
+require "limiter"
 require "embulk/input/jira_api/issue"
 require "timeout"
 
@@ -7,31 +8,68 @@ module Embulk
   module Input
     module JiraApi
       class Client
-        PARALLEL_THREAD_COUNT = 50
-        SEARCH_TIMEOUT_SECONDS = 5
-        SEARCH_ISSUES_TIMEOUT_SECONDS = 60
+        MAX_RATE_LIMIT = 50
+        MIN_RATE_LIMIT = 2
+        # Normal http request timeout is 300s
+        SEARCH_ISSUES_TIMEOUT_SECONDS = 300
         DEFAULT_SEARCH_RETRY_TIMES = 10
 
+        def initialize
+          @rate_limiter = Limiter::RateQueue.new(MAX_RATE_LIMIT, interval: 2)
+        end
+
         def self.setup(&block)
           Jiralicious.configure(&block)
           new
         end
 
         def search_issues(jql, options={})
-          timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do
-            issues_raw = search(jql, options).issues_raw
-
-            # TODO: below code has race-conditon.
-            Parallel.map(issues_raw, in_threads: PARALLEL_THREAD_COUNT) do |issue_raw|
-              # https://github.com/dorack/jiralicious/blob/v0.4.0/lib/jiralicious/search_result.rb#L32-34
-              issue = Jiralicious::Issue.find(issue_raw["key"])
-              JiraApi::Issue.new(issue)
+          issues_raw = search(jql, options).issues_raw
+          # Maximum number of issues to retrieve is 50
+          rate_limit = MAX_RATE_LIMIT
+          success_items = []
+          fail_items = []
+          error_object = nil
+          timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS * MAX_RATE_LIMIT ) do
+            retry_count = 0
+            semaphore = Mutex.new
+            @rate_limiter = Limiter::RateQueue.new(rate_limit, interval: 2)
+            error_object = nil
+            while issues_raw.length > 0 && retry_count <= DEFAULT_SEARCH_RETRY_TIMES do
+              Parallel.each(issues_raw, in_threads: rate_limit) do |issue_raw|
+                # https://github.com/dorack/jiralicious/blob/v0.4.0/lib/jiralicious/search_result.rb#L32-34
+                begin
+                  issue = find_issue(issue_raw["key"])
+                  semaphore.synchronize {
+                    success_items.push(JiraApi::Issue.new(issue))
+                  }
+                rescue MultiJson::ParseError => e
+                  html = e.message
+                  title = html[%r|<title>(.*?)</title>|, 1]
+                  # 401 due to high number of concurrent requests with current account
+                  # The number of concurrent requests is not fixed by every account
+                  # Hence catch the error item and retry later
+                  raise title if title != "Unauthorized (401)"
+                  semaphore.synchronize {
+                    fail_items.push(issue_raw)
+                    error_object = e
+                  }
+                end
+              end
+              retry_count += 1
+              rate_limit = calculate_rate_limit(rate_limit, issues_raw.length, fail_items.length, retry_count)
+              issues_raw = fail_items
+              fail_items = []
+              raise error_object if retry_count > DEFAULT_SEARCH_RETRY_TIMES && !error_object.nil?
+              # Sleep after some seconds for JIRA API perhaps under the overload
+              sleep retry_count if fail_items.length > 0
             end
+            success_items
           end
         end
 
         def search(jql, options={})
-          timeout_and_retry(SEARCH_TIMEOUT_SECONDS) do
+          timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do
             Jiralicious.search(jql, options)
           end
         end
@@ -53,10 +91,19 @@ module Embulk
           raise ConfigError.new("Can not authorize with your credential.") if title == 'Unauthorized (401)'
         end
 
+        # Calculate rate limit based on previous run result
+        # Return 2 MIN_RATE_LIMIT in case turning from the 5th times or success_items is less than 2
+        # Otherwise return the min number between fail_items, success_items and current_limit
+        def calculate_rate_limit(current_limit, all_items, fail_items, times)
+          success_items = all_items - fail_items
+          return MIN_RATE_LIMIT if times >= DEFAULT_SEARCH_RETRY_TIMES/2 || success_items < MIN_RATE_LIMIT
+          return [fail_items, success_items, current_limit].min
+        end
+
         private
 
         def timeout_and_retry(wait, retry_times = DEFAULT_SEARCH_RETRY_TIMES, &block)
-          count = 1
+          count = 0
           begin
             Timeout.timeout(wait) do
               yield
@@ -72,29 +119,25 @@ module Embulk
             # And (b) `search_issues` method has race-condition bug. If it occurred, MultiJson::ParseError raised too.
             html = e.message
             title = html[%r|<title>(.*?)</title>|, 1] #=> e.g. "Unauthorized (401)"
-            if title
-              # (a)
-              case title
-              when "Atlassian Cloud Notifications - Page Unavailable"
-                # a.k.a. HTTP 503
-                raise title
-              when "Unauthorized (401)"
-                Embulk.logger.warn "JIRA returns error: #{title}. Will go to retry"
-                count += 1
-                retry
-              end
-            else
-              # (b)
-              count += 1
-              retry
-            end
+            raise title if title == "Atlassian Cloud Notifications - Page Unavailable"
+            count += 1
+            raise title.nil? ? "Unknown Error" : title if count > retry_times
+            Embulk.logger.warn "JIRA returns error: #{title == 'Unauthorized (401)' ? title + " due to overloading API requests. Retrying on failed items only" : title}."
+            sleep count
+            retry
           rescue Timeout::Error => e
             count += 1
-            sleep count # retry after some seconds for JIRA API perhaps under the overload
             raise e if count > retry_times
+            Embulk.logger.warn "Time out error."
+            sleep count # retry after some seconds for JIRA API perhaps under the overload
             retry
           end
         end
+
+        def find_issue(issue_key)
+          @rate_limiter.shift
+          Jiralicious::Issue.find(issue_key)
+        end
       end
     end
   end

data/spec/embulk/input/jira_api/client_spec.rb

@@ -30,7 +30,7 @@ describe Embulk::Input::JiraApi::Client do
       end
 
       it "retry DEFAULT_SEARCH_RETRY_TIMES times then raise error" do
-        expect(Timeout).to receive(:timeout).exactly(Embulk::Input::JiraApi::Client::DEFAULT_SEARCH_RETRY_TIMES)
+        expect(Timeout).to receive(:timeout).exactly(Embulk::Input::JiraApi::Client::DEFAULT_SEARCH_RETRY_TIMES + 1)
         expect { subject }.to raise_error
       end
     end
@@ -38,6 +38,9 @@ describe Embulk::Input::JiraApi::Client do
 
   describe "#search_issues" do
     let(:jql) { "project=FOO" }
+    let(:jira_api) { Embulk::Input::JiraApi::Client.new }
+    let(:title_401) { "Unauthorized (401)"}
+    let(:multi_json_error) {MultiJson::ParseError.build(StandardError.new("<title>#{title_401}</title>"), {})}
     let(:results) do
       [
         {
@@ -67,15 +70,39 @@ describe Embulk::Input::JiraApi::Client do
       ]
     end
 
-    subject { Embulk::Input::JiraApi::Client.new.search_issues(jql) }
+    subject { jira_api.search_issues(jql) }
 
-    it do
+    it "Search issues successfully" do
+      allow(Jiralicious).to receive_message_chain(:search, :issues_raw).and_return(results)
+      allow(jira_api).to receive(:find_issue).and_return(results.first)
+
+      expect(subject).to be_kind_of Array
+      expect(subject.map(&:class)).to match_array [Embulk::Input::JiraApi::Issue, Embulk::Input::JiraApi::Issue]
+    end
+
+    it "Search issues successfully when first item success - 401 - second items success" do
       allow(Jiralicious).to receive_message_chain(:search, :issues_raw).and_return(results)
-      allow(Jiralicious::Issue).to receive(:find).and_return(results.first)
+      allow(jira_api).to receive(:find_issue).and_return(results.first).and_raise(multi_json_error).and_return(results.first)
 
       expect(subject).to be_kind_of Array
       expect(subject.map(&:class)).to match_array [Embulk::Input::JiraApi::Issue, Embulk::Input::JiraApi::Issue]
     end
+
+    it "Search issues successfully when 401 - first and second items success" do
+      allow(Jiralicious).to receive_message_chain(:search, :issues_raw).and_return(results)
+      allow(jira_api).to receive(:find_issue).and_raise(multi_json_error).and_return(results.first)
+
+      expect(subject).to be_kind_of Array
+      expect(subject.map(&:class)).to match_array [Embulk::Input::JiraApi::Issue, Embulk::Input::JiraApi::Issue]
+    end
+
+    it "Search issues got 401 due to high concurrent load issues" do
+      allow(Jiralicious).to receive_message_chain(:search, :issues_raw).and_return(results)
+      allow(jira_api).to receive(:find_issue).and_raise(multi_json_error)
+      allow(jira_api).to receive(:sleep)
+
+      expect { subject }.to raise_error(StandardError, title_401)
+    end
   end
 
   describe "#total_count" do
@@ -120,7 +147,7 @@ describe Embulk::Input::JiraApi::Client do
     it "Always timeout, raise error after N times retry" do
       allow(Timeout).to receive(:timeout) { raise Timeout::Error }
 
-      expect(Timeout).to receive(:timeout).with(wait).exactly(retry_times).times
+      expect(Timeout).to receive(:timeout).with(wait).exactly(retry_times + 1).times
       expect { subject }.to raise_error(Timeout::Error)
     end
 
@@ -146,4 +173,52 @@ describe Embulk::Input::JiraApi::Client do
       end
     end
   end
+
+  describe "#calculate_rate_limit" do
+    let(:jira_api) { Embulk::Input::JiraApi::Client.new }
+    it "current_limit = 50, all_items = 50, fail_items=50, times=1" do
+      current_limit = 50
+      all_items = 50
+      fail_items = 50
+      times = 1
+      expected_result = Embulk::Input::JiraApi::Client::MIN_RATE_LIMIT
+      expect(jira_api.calculate_rate_limit(current_limit, all_items, fail_items, times)).to eq expected_result
+    end
+
+    it "current_limit = 50, all_items = 50, fail_items=20, times=1" do
+      current_limit = 50
+      all_items = 50
+      fail_items = 20
+      times = 1
+      expected_result = 20
+      expect(jira_api.calculate_rate_limit(current_limit, all_items, fail_items, times)).to eq expected_result
+    end
+
+    it "current_limit = MIN_RATE_LIMIT, all_items = 50, fail_items=20, times=2" do
+      current_limit = Embulk::Input::JiraApi::Client::MIN_RATE_LIMIT
+      all_items = 50
+      fail_items = 20
+      times = 2
+      expected_result = Embulk::Input::JiraApi::Client::MIN_RATE_LIMIT
+      expect(jira_api.calculate_rate_limit(current_limit, all_items, fail_items, times)).to eq expected_result
+    end
+
+    it "current_limit = 10, all_items = 30, fail_items=25, times=2" do
+      current_limit = 10
+      all_items = 30
+      fail_items = 25
+      times = 2
+      expected_result = 5
+      expect(jira_api.calculate_rate_limit(current_limit, all_items, fail_items, times)).to eq expected_result
+    end
+
+    it "current_limit = 50, all_items = 50, fail_items=20, times=DEFAULT_SEARCH_RETRY_TIMES/2" do
+      current_limit = 50
+      all_items = 50
+      fail_items = 20
+      times = Embulk::Input::JiraApi::Client::DEFAULT_SEARCH_RETRY_TIMES/2
+      expected_result = Embulk::Input::JiraApi::Client::MIN_RATE_LIMIT
+      expect(jira_api.calculate_rate_limit(current_limit, all_items, fail_items, times)).to eq expected_result
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jira
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - uu59
@@ -9,162 +9,176 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-17 00:00:00.000000000 Z
+date: 2018-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: jiralicious
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.0
+  name: jiralicious
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
+  name: parallel
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: ruby-limiter
   prerelease: false
   type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: perfect_retry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
+  name: perfect_retry
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.3'
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: '11.0'
+  name: rake
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "<"
       - !ruby/object:Gem::Version
         version: '11.0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.0
+  name: rspec
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.2.0
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: embulk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
+  name: embulk
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: simplecov
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: pry
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: codeclimate-test-reporter
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: everyleaf-embulk_helper
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+  name: everyleaf-embulk_helper
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  prerelease: false
-  type: :development
 description: Loads records from Jira.
 email:
 - k@uu59.org