embulk-input-jira 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c300bb312a6bbf2bfa30a0ed5531e7c3a70a7c4f
-  data.tar.gz: 1cd0319e3794bbde87d9fb82b1d4c9833bb05678
+  metadata.gz: c37d6bfba4c1f808f3911a222698a62e87b1a6ec
+  data.tar.gz: 4348bce6dd5c2c293e090396f8375c4a520830cd
 SHA512:
-  metadata.gz: 73cb22cdbf30f9a43b4d3c3a42d3487064581cb56a6af3fc7c4b3dbd4cd700c5c89979126cbc712da38b1ea588bb6ff0d5ad61675c840e62f8694bb67c73d37f
-  data.tar.gz: 5ad6cf1fbcef9d78c22a1b21f03df1818d856f0a3803cf339aaea0ccf4fbb9bb640c8b30100b5993b3c963eea68ab99d5d2af3ba7e6e8a19f31972ddd6aa2dc3
+  metadata.gz: e383256cdde3bb1623a7777ff4eb811842a8fb1a37a26ea8319a6c9e8f3d7ab5f9ec572b2ee55e7e648c4d923ede50655ea36b14ab3c411488b29bd6b5951923
+  data.tar.gz: 91d53c2b6ebf59b2ee8dd0888da96360c592b2b56be0349caae291dc8874fac2ed61bfab7778e1a666a410b250a4f99c4db9191bf907161eb912fcae495d3d70

data/CHANGELOG.md

@@ -1,3 +1,9 @@
-## 0.0.1 (2015-05-28)
+## 0.0.2 - 2015-06-04
+* [maintenance] Add release:prepare task for development [#21](https://github.com/treasure-data/embulk-input-jira/pull/21) 
+* [enhancement] Add timeout and retry for JIRA API search [#20](https://github.com/treasure-data/embulk-input-jira/pull/20) 
+* [enhancement] Display progress [#19](https://github.com/treasure-data/embulk-input-jira/pull/19) 
+* [maintenance] Remove needless guessed config [#18](https://github.com/treasure-data/embulk-input-jira/pull/18) 
+
+## 0.0.1 - 2015-05-28
 
 The first release!!

data/Rakefile

@@ -1,5 +1,9 @@
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
+require "json"
+require "pathname"
+require "open-uri"
+
 
 task default: :spec
 
@@ -7,3 +11,51 @@ desc "Run all examples"
 RSpec::Core::RakeTask.new(:spec) do |t|
   t.rspec_opts = %w[--color]
 end
+
+namespace :release do
+  desc "Add header of now version release to ChangeLog and bump up version"
+  task :prepare do
+    root_dir = Pathname.new(File.expand_path("../", __FILE__))
+    version_file = root_dir.join("VERSION")
+    changelog_file = root_dir.join("CHANGELOG.md")
+
+    system("git fetch origin")
+
+    # detect merged PR
+    old_version = version_file.read.strip
+    pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
+
+    if !$?.success? || pr_numbers.empty?
+      puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
+      exit(false)
+    end
+
+    # Generate new version
+    major, minor, patch = old_version.split(".").map(&:to_i)
+    new_version = "#{major}.#{minor}.#{patch + 1}"
+
+    # Update ChangeLog
+    pr_descriptions = pr_numbers.map do |number|
+      body = open("https://api.github.com/repos/treasure-data/embulk-input-jira/issues/#{number.gsub("#", "")}").read
+      payload = JSON.parse(body)
+      "* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-jira/pull/#{number.gsub('#', '')}) "
+    end.join("\n")
+
+    new_changelog = <<-HEADER
+## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
+#{pr_descriptions}
+
+#{changelog_file.read.chomp}
+HEADER
+
+    File.open(changelog_file, "w") {|f| f.write(new_changelog) }
+
+    # Update version.rb
+    File.open(version_file, "w") {|f| f.write new_version }
+
+    # Update Gemfile.lock
+    system("bundle install")
+
+    puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
+  end
+end

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/embulk/input/jira.rb

@@ -1,5 +1,7 @@
 require "embulk/input/jira-input-plugin-utils"
 require "jira/api"
+require "logger"
+require "time"
 
 module Embulk
   module Input
@@ -67,11 +69,6 @@ module Embulk
         columns = JiraInputPluginUtils.guess_columns(records)
 
         guessed_config = {
-          "username" => username,
-          "password" => password,
-          "uri" => uri,
-          "api_version" => api_version,
-          "auth_type" => auth_type,
           "columns" => columns,
         }
 
@@ -92,8 +89,10 @@ module Embulk
 
       def run
         total_count = @jira.total_count(@jql)
+        last_page = (total_count.to_f / PER_PAGE).ceil
 
-        0.step(total_count, PER_PAGE) do |start_at|
+        0.step(total_count, PER_PAGE).with_index(1) do |start_at, page|
+          logger.debug "Fetching #{page} / #{last_page} page"
           @jira.search_issues(@jql, start_at: start_at).each do |issue|
             values = @attributes.map do |(attribute_name, type)|
               JiraInputPluginUtils.cast(issue[attribute_name], type)
@@ -108,6 +107,21 @@ module Embulk
         commit_report = {}
         return commit_report
       end
+
+      def self.logger
+        @logger ||=
+          begin
+            logger = Logger.new($stdout)
+            logger.formatter = proc do |severity, datetime, progname, msg|
+              "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%L %z")} [#{severity}] #{msg}\n"
+            end
+            logger
+          end
+      end
+
+      def logger
+        self.class.logger
+      end
     end
   end
 end

data/lib/jira/api.rb

@@ -1,25 +1,50 @@
 require "jiralicious"
 require "jira/issue"
+require "timeout"
 
 module Jira
   class Api
+    SEARCH_TIMEOUT_SECONDS = 5
+    SEARCH_ISSUES_TIMEOUT_SECONDS = 60
+    DEFAULT_SEARCH_RETRY_TIMES = 10
+
     def self.setup(&block)
       Jiralicious.configure(&block)
       new
     end
 
     def search_issues(jql, options={})
-      search(jql, options).issues.map do |issue|
-        ::Jira::Issue.new(issue)
+      timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do
+        search(jql, options).issues.map do |issue|
+          ::Jira::Issue.new(issue)
+        end
       end
     end
 
     def search(jql, options={})
-      Jiralicious.search(jql, options)
+      timeout_and_retry(SEARCH_TIMEOUT_SECONDS) do
+        Jiralicious.search(jql, options)
+      end
     end
 
     def total_count(jql)
       search(jql, max_results: 1).num_results
     end
+
+    private
+
+    def timeout_and_retry(wait, retry_times = DEFAULT_SEARCH_RETRY_TIMES, &block)
+      count = 1
+      begin
+        Timeout.timeout(wait) do
+          yield
+        end
+      rescue Timeout::Error => e
+        count += 1
+        sleep count # retry after some seconds for JIRA API perhaps under the overload
+        raise e if count > retry_times
+        retry
+      end
+    end
   end
 end

data/spec/embulk/input/jira_spec.rb

@@ -116,11 +116,6 @@ describe Embulk::Input::JiraInputPlugin do
 
     let(:guessed_config) do
       {
-        "username" => username,
-        "password" => password,
-        "uri" => uri,
-        "api_version" => "latest",
-        "auth_type" => "basic",
         "columns" => [
           {name: "id", type: :long},
           {name: "key", type: :string},
@@ -169,7 +164,11 @@ describe Embulk::Input::JiraInputPlugin do
 
   describe "#run" do
     subject do
-      Embulk::Input::JiraInputPlugin.new(task, nil, nil, page_builder).run
+      result = nil
+      capture_output(:out) do
+        result = Embulk::Input::JiraInputPlugin.new(task, nil, nil, page_builder).run
+      end
+      result
     end
 
     let(:jira_api) { Jira::Api.new }
@@ -244,4 +243,21 @@ describe Embulk::Input::JiraInputPlugin do
       expect(subject).to eq File.read(version_file_path).strip
     end
   end
+
+  describe ".logger" do
+    let(:logger) { Embulk::Input::JiraInputPlugin.logger }
+
+    subject { logger }
+
+    it { is_expected.to be_a(Logger) }
+  end
+
+  describe "#logger" do
+    let(:instance) { Embulk::Input::JiraInputPlugin.new({}, nil, nil, nil) }
+    let(:logger) { instance.logger }
+
+    subject { logger }
+
+    it { is_expected.to be_a(Logger) }
+  end
 end

data/spec/jira_api_spec.rb

@@ -18,12 +18,25 @@ describe Jira::Api do
 
   describe "#search" do
     let(:jql) { "project=FOO" }
+    let(:api) { Jira::Api.new }
 
-    subject { Jira::Api.new.search(jql) }
+    subject { api.search(jql) }
 
     it do
       allow(Jiralicious).to receive(:search).with(jql)
     end
+
+    describe "retry and timeout" do
+      before do
+        allow(Timeout).to receive(:timeout) { raise Timeout::Error }
+        allow(api).to receive(:sleep)
+      end
+
+      it "retry DEFAULT_SEARCH_RETRY_TIMES times then raise error" do
+        expect(Timeout).to receive(:timeout).exactly(Jira::Api::DEFAULT_SEARCH_RETRY_TIMES)
+        expect { subject }.to raise_error
+      end
+    end
   end
 
   describe "#search_issues" do
@@ -89,4 +102,28 @@ describe Jira::Api do
       expect(subject).to eq results_count
     end
   end
+
+  describe "#timeout_and_retry" do
+    let(:wait) { 1 }
+    let(:retry_times) { 3 }
+    let(:jira_api) { Jira::Api.new }
+    let(:block) { proc{ "it works" } }
+
+    subject { jira_api.send(:timeout_and_retry, wait, retry_times, &block) }
+
+    before do
+      allow(jira_api).to receive(:sleep)
+    end
+
+    it "return given block result if timeout is not occured" do
+      expect(subject).to eq block.call
+    end
+
+    it "Always timeout, raise error after N times retry" do
+      allow(Timeout).to receive(:timeout) { raise Timeout::Error }
+
+      expect(Timeout).to receive(:timeout).with(wait).exactly(retry_times).times
+      expect { subject }.to raise_error(Timeout::Error)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jira
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-28 00:00:00.000000000 Z
+date: 2015-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement