embulk-input-jira 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 14d965de93b1ed1a36384160c678009c61e87f12
-  data.tar.gz: cc230ada4b25382dc40ce20912f9e87181e8ca72
+  metadata.gz: 7fb29dc47926d5c193b6b8b08f4a824cd2a83872
+  data.tar.gz: 2ebccda2aa31109d07021be6a1bdcdd052376f64
 SHA512:
-  metadata.gz: 207f74f545d0d951ef5f8644f3557d3a60d6e1599e184eb88a6bcc6d491e38c55482350c6301e73651c8faf08dc572ad53f82dc82e53ada8fe683472fb81e491
-  data.tar.gz: 9cd3d4d3b4ac8909945d6de6e67b24c309ddcf00a457410d9abc6f564bdacbf7153947fc79fd4f341808faa5876d25de6a1d745c4d7e57e2e34b6f4c9f3f0ed2
+  metadata.gz: fc399cfc63fd0adc78b46f4824a0ee12d45d0acfbb7dc249d43888c63b5e9c466a6ab08c5bdedcfcf9727448605084ff3646d0d31c4732f61723c002e18a22dc
+  data.tar.gz: 90475bce6b6d0572d51e52cea7c36345bc5c2a52b93c4dc7da488bd32eca991eb4745de7853a244930843b9d3f4034aecfb808e4633c57aa8ba36759cd36e64d

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.0.6 - 2015-06-24
+* [maintenance] String keys to symbols [#35](https://github.com/treasure-data/embulk-input-jira/pull/35)
+* [enhancement] Faster fetching with parallel API request [#34](https://github.com/treasure-data/embulk-input-jira/pull/34)
+
 ## 0.0.5 - 2015-06-24
 
 **Embulk 0.6.12+ is required since this version**

data/embulk-input-jira.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-jira"
-  spec.version       = "0.0.5"
+  spec.version       = "0.0.6"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Jira input plugin for Embulk"
   spec.description   = "Loads records from Jira."
@@ -13,6 +13,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'jiralicious', ['~> 0.5.0']
+  spec.add_dependency 'parallel', ['~> 1.6.0']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
   spec.add_development_dependency 'rspec', "~> 3.2.0"

data/lib/embulk/input/jira.rb

@@ -12,21 +12,21 @@ module Embulk
 
       def self.transaction(config, &control)
         task = {
-          "username" => config.param("username", :string),
-          "password" => config.param("password", :string),
-          "uri" => config.param("uri", :string),
-          "jql" => config.param("jql", :string),
+          username: config.param(:username, :string),
+          password: config.param(:password, :string),
+          uri: config.param(:uri, :string),
+          jql: config.param(:jql, :string),
         }
 
         attributes = {}
-        columns = config.param("columns", :array).map do |column|
+        columns = config.param(:columns, :array).map do |column|
           name = column["name"]
           type = column["type"].to_sym
           attributes[name] = type
           Column.new(nil, name, type, column["format"])
         end
 
-        task["attributes"] = attributes
+        task[:attributes] = attributes
 
         resume(task, columns, 1, &control)
       end
@@ -39,22 +39,20 @@ module Embulk
       end
 
       def self.guess(config)
-        # TODO: api_version should be 2 (the latest version)
-        # auth_type should be specified from config. (The future task)
-
-        username = config.param("username", :string)
-        password = config.param("password", :string)
-        uri = config.param("uri", :string)
-        api_version = "latest"
-        auth_type = "basic"
-        jql = config.param("jql", :string)
+        username = config.param(:username, :string)
+        password = config.param(:password, :string)
+        uri = config.param(:uri, :string)
+        jql = config.param(:jql, :string)
 
         jira = JiraApi::Client.setup do |jira_config|
+          # TODO: api_version should be 2 (the latest version)
+          # auth_type should be specified from config. (The future task)
+
           jira_config.username = username
           jira_config.password = password
           jira_config.uri = uri
-          jira_config.api_version = api_version
-          jira_config.auth_type = auth_type
+          jira_config.api_version = "latest"
+          jira_config.auth_type = "basic"
         end
 
         # TODO: we use 0..10 issues to guess config?
@@ -72,15 +70,15 @@ module Embulk
       end
 
       def init
-        @attributes = task["attributes"]
+        @attributes = task[:attributes]
         @jira = JiraApi::Client.setup do |config|
-          config.username = task["username"]
-          config.password = task["password"]
-          config.uri = task["uri"]
+          config.username = task[:username]
+          config.password = task[:password]
+          config.uri = task[:uri]
           config.api_version = "latest"
-          config.auth_type = :basic
+          config.auth_type = "basic"
         end
-        @jql = task["jql"]
+        @jql = task[:jql]
       end
 
       def run

data/lib/embulk/input/jira_api/client.rb

@@ -1,4 +1,5 @@
 require "jiralicious"
+require "parallel"
 require "embulk/input/jira_api/issue"
 require "timeout"
 
@@ -6,6 +7,7 @@ module Embulk
   module Input
     module JiraApi
       class Client
+        PARALLEL_THREAD_COUNT = 50
         SEARCH_TIMEOUT_SECONDS = 5
         SEARCH_ISSUES_TIMEOUT_SECONDS = 60
         DEFAULT_SEARCH_RETRY_TIMES = 10
@@ -17,7 +19,10 @@ module Embulk
 
         def search_issues(jql, options={})
           timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do
-            search(jql, options).issues.map do |issue|
+            issues_raw = search(jql, options).issues_raw
+            Parallel.map(issues_raw, in_threads: PARALLEL_THREAD_COUNT) do |issue_raw|
+              # https://github.com/dorack/jiralicious/blob/v0.4.0/lib/jiralicious/search_result.rb#L32-34
+              issue = Jiralicious::Issue.find(issue_raw["key"])
               JiraApi::Issue.new(issue)
             end
           end

data/spec/embulk/input/jira_api/client_spec.rb

@@ -48,7 +48,7 @@ describe Embulk::Input::JiraApi::Client do
             "summary" => "issue summary",
             "project" =>
             {
-              "key" => "FOO"
+              "key" => "FO1"
             }
           }
         },
@@ -60,7 +60,7 @@ describe Embulk::Input::JiraApi::Client do
             "summary" => "jira issue",
             "project" =>
             {
-              "key" => "FOO"
+              "key" => "FO2"
             }
           }
         }
@@ -70,7 +70,8 @@ describe Embulk::Input::JiraApi::Client do
     subject { Embulk::Input::JiraApi::Client.new.search_issues(jql) }
 
     it do
-      allow(Jiralicious).to receive_message_chain(:search, :issues).and_return(results)
+      allow(Jiralicious).to receive_message_chain(:search, :issues_raw).and_return(results)
+      allow(Jiralicious::Issue).to receive(:find).and_return(results.first)
 
       expect(subject).to be_kind_of Array
       expect(subject.map(&:class)).to match_array [Embulk::Input::JiraApi::Issue, Embulk::Input::JiraApi::Issue]

data/spec/embulk/input/jira_spec.rb

@@ -15,11 +15,11 @@ describe Embulk::Input::Jira do
 
     let(:task) do
       {
-        "username" => username,
-        "password" => password,
-        "uri" => uri,
-        "jql" => jql,
-        "attributes" => {
+        username: username,
+        password: password,
+        uri: uri,
+        jql: jql,
+        attributes: {
           "project.key" => :string,
           "comment.total" => :long
         }
@@ -41,11 +41,11 @@ describe Embulk::Input::Jira do
     end
 
     before do
-      allow(config).to receive(:param).with("username", :string).and_return(username)
-      allow(config).to receive(:param).with("password", :string).and_return(password)
-      allow(config).to receive(:param).with("uri", :string).and_return(uri)
-      allow(config).to receive(:param).with("jql", :string).and_return(jql)
-      allow(config).to receive(:param).with("columns", :array).and_return(columns)
+      allow(config).to receive(:param).with(:username, :string).and_return(username)
+      allow(config).to receive(:param).with(:password, :string).and_return(password)
+      allow(config).to receive(:param).with(:uri, :string).and_return(uri)
+      allow(config).to receive(:param).with(:jql, :string).and_return(jql)
+      allow(config).to receive(:param).with(:columns, :array).and_return(columns)
     end
 
     # NOTE: I should check other factor, but i don't know it...
@@ -60,11 +60,11 @@ describe Embulk::Input::Jira do
 
     let(:task) do
       {
-        "username" => username,
-        "password" => password,
-        "uri" => uri,
-        "jql" => jql,
-        "attributes" => {
+        username: username,
+        password: password,
+        uri: uri,
+        jql: jql,
+        attributes: {
           "project.key" => :string,
           "comment.total" => :long
         }
@@ -129,10 +129,10 @@ describe Embulk::Input::Jira do
     before do
       allow(jira_api).to receive(:search_issues).with(jql, max_results: described_class::GUESS_RECORDS_COUNT).and_return(jira_issues)
 
-      allow(config).to receive(:param).with("username", :string).and_return(username)
-      allow(config).to receive(:param).with("password", :string).and_return(password)
-      allow(config).to receive(:param).with("uri", :string).and_return(uri)
-      allow(config).to receive(:param).with("jql", :string).and_return(jql)
+      allow(config).to receive(:param).with(:username, :string).and_return(username)
+      allow(config).to receive(:param).with(:password, :string).and_return(password)
+      allow(config).to receive(:param).with(:uri, :string).and_return(uri)
+      allow(config).to receive(:param).with(:jql, :string).and_return(jql)
     end
 
     it "setup Embulk::Input::JiraApi::Client" do
@@ -188,8 +188,8 @@ describe Embulk::Input::Jira do
     let(:page_builder) { Object.new } # add mock later
     let(:task) do
       {
-        "jql" => jql,
-        "attributes" => {"project.key" => "string"}
+        jql: jql,
+        attributes: {"project.key" => "string"}
       }
     end
 
@@ -243,8 +243,8 @@ describe Embulk::Input::Jira do
     let(:plugin) { described_class.new(task, nil, nil, page_builder) }
     let(:task) do
       {
-        "jql" => jql,
-        "attributes" => {"project.key" => "string"}
+        jql: jql,
+        attributes: {"project.key" => "string"}
       }
     end
     let(:page_builder) { double("page_builder") }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jira
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - uu59
@@ -25,6 +25,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.5.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  name: parallel
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: