embulk-input-jira 0.2.5 → 0.2.6

data/Gemfile

@@ -1,3 +0,0 @@
-source 'https://rubygems.org/'
-
-gemspec

data/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2015 Everyleaf Corporation
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

data/Rakefile

@@ -1,15 +0,0 @@
-require "bundler/gem_tasks"
-require 'rspec/core/rake_task'
-require "everyleaf/embulk_helper/tasks"
-
-Everyleaf::EmbulkHelper::Tasks.install({
-  gemspec: "./embulk-input-jira.gemspec",
-  github_name: "treasure-data/embulk-input-jira",
-})
-
-task default: :spec
-
-desc "Run all examples"
-RSpec::Core::RakeTask.new(:spec) do |t|
-  t.rspec_opts = %w[--color]
-end

data/embulk-input-jira.gemspec

@@ -1,27 +0,0 @@
-Gem::Specification.new do |spec|
-  spec.name          = "embulk-input-jira"
-  spec.version       = "0.2.5"
-  spec.authors       = ["uu59", "yoshihara"]
-  spec.summary       = "Jira input plugin for Embulk"
-  spec.description   = "Loads records from Jira."
-  spec.email         = ["k@uu59.org", "h.yoshihara@everyleaf.com"]
-  spec.licenses      = ["Apache2"]
-  spec.homepage      = "https://github.com/treasure-data/embulk-input-jira"
-
-  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency 'jiralicious', ['~> 0.5.0']
-  spec.add_dependency 'parallel', ['~> 1.6.0']
-  spec.add_dependency 'ruby-limiter', ['~> 1.0']
-  spec.add_dependency 'perfect_retry', ['~> 0.3']
-  spec.add_development_dependency 'bundler', ['~> 1.0']
-  spec.add_development_dependency 'rake', ['< 11.0']
-  spec.add_development_dependency 'rspec', "~> 3.2.0"
-  spec.add_development_dependency 'embulk', ["~> 0.8.7"]
-  spec.add_development_dependency 'simplecov'
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'codeclimate-test-reporter'
-  spec.add_development_dependency 'everyleaf-embulk_helper'
-end

data/gemfiles/embulk-0.8.0-latest

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "~> 0.8.0"

data/gemfiles/embulk-0.8.7

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.8.7"

data/gemfiles/embulk-0.8.8

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "0.8.8"

data/gemfiles/embulk-latest

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "> 0.8.7"

data/gemfiles/template.erb

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "<%= version %>"

data/lib/embulk/input/jira_api.rb

@@ -1,9 +0,0 @@
-require "embulk/input/jira_api/client"
-require "embulk/input/jira_api/issue"
-
-module Embulk
-  module Input
-    module JiraApi
-    end
-  end
-end

data/lib/embulk/input/jira_api/client.rb

@@ -1,144 +0,0 @@
-require "jiralicious"
-require "parallel"
-require "limiter"
-require "embulk/input/jira_api/issue"
-require "timeout"
-
-module Embulk
-  module Input
-    module JiraApi
-      class Client
-        MAX_RATE_LIMIT = 50
-        MIN_RATE_LIMIT = 2
-        # Normal http request timeout is 300s
-        SEARCH_ISSUES_TIMEOUT_SECONDS = 300
-        DEFAULT_SEARCH_RETRY_TIMES = 10
-
-        def initialize
-          @rate_limiter = Limiter::RateQueue.new(MAX_RATE_LIMIT, interval: 2)
-        end
-
-        def self.setup(&block)
-          Jiralicious.configure(&block)
-          new
-        end
-
-        def search_issues(jql, options={})
-          issues_raw = search(jql, options).issues_raw
-          # Maximum number of issues to retrieve is 50
-          rate_limit = MAX_RATE_LIMIT
-          success_items = []
-          fail_items = []
-          error_object = nil
-          timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS * MAX_RATE_LIMIT ) do
-            retry_count = 0
-            semaphore = Mutex.new
-            @rate_limiter = Limiter::RateQueue.new(rate_limit, interval: 2)
-            error_object = nil
-            while issues_raw.length > 0 && retry_count <= DEFAULT_SEARCH_RETRY_TIMES do
-              Parallel.each(issues_raw, in_threads: rate_limit) do |issue_raw|
-                # https://github.com/dorack/jiralicious/blob/v0.4.0/lib/jiralicious/search_result.rb#L32-34
-                begin
-                  issue = find_issue(issue_raw["key"])
-                  semaphore.synchronize {
-                    success_items.push(JiraApi::Issue.new(issue))
-                  }
-                rescue MultiJson::ParseError => e
-                  html = e.message
-                  title = html[%r|<title>(.*?)</title>|, 1]
-                  # 401 due to high number of concurrent requests with current account
-                  # The number of concurrent requests is not fixed by every account
-                  # Hence catch the error item and retry later
-                  raise title if title != "Unauthorized (401)"
-                  semaphore.synchronize {
-                    fail_items.push(issue_raw)
-                    error_object = e
-                  }
-                end
-              end
-              retry_count += 1
-              rate_limit = calculate_rate_limit(rate_limit, issues_raw.length, fail_items.length, retry_count)
-              issues_raw = fail_items
-              fail_items = []
-              raise error_object if retry_count > DEFAULT_SEARCH_RETRY_TIMES && !error_object.nil?
-              # Sleep after some seconds for JIRA API perhaps under the overload
-              sleep retry_count if fail_items.length > 0
-            end
-            success_items
-          end
-        end
-
-        def search(jql, options={})
-          timeout_and_retry(SEARCH_ISSUES_TIMEOUT_SECONDS) do
-            Jiralicious.search(jql, options)
-          end
-        end
-
-        def total_count(jql)
-          search(jql, max_results: 1).num_results
-        end
-
-        def check_user_credential(username)
-          Jiralicious::User.search(username)
-        rescue Jiralicious::JqlError, Jiralicious::AuthenticationError, Jiralicious::NotLoggedIn, Jiralicious::InvalidLogin => e
-          raise Embulk::ConfigError.new(e.message)
-        rescue ::SocketError => e
-          # wrong `uri` option given
-          raise Embulk::ConfigError.new(e.message)
-        rescue MultiJson::ParseError => e
-          html = e.message
-          title = html[%r|<title>(.*?)</title>|, 1] #=> e.g. "Unauthorized (401)"
-          raise ConfigError.new("Can not authorize with your credential.") if title == 'Unauthorized (401)'
-        end
-
-        # Calculate rate limit based on previous run result
-        # Return 2 MIN_RATE_LIMIT in case turning from the 5th times or success_items is less than 2
-        # Otherwise return the min number between fail_items, success_items and current_limit
-        def calculate_rate_limit(current_limit, all_items, fail_items, times)
-          success_items = all_items - fail_items
-          return MIN_RATE_LIMIT if times >= DEFAULT_SEARCH_RETRY_TIMES/2 || success_items < MIN_RATE_LIMIT
-          return [fail_items, success_items, current_limit].min
-        end
-
-        private
-
-        def timeout_and_retry(wait, retry_times = DEFAULT_SEARCH_RETRY_TIMES, &block)
-          count = 0
-          begin
-            Timeout.timeout(wait) do
-              yield
-            end
-          rescue Jiralicious::JqlError, Jiralicious::AuthenticationError, Jiralicious::NotLoggedIn, Jiralicious::InvalidLogin => e
-            raise Embulk::ConfigError.new(e.message)
-          rescue ::SocketError => e
-            # wrong `uri` option given
-            raise Embulk::ConfigError.new(e.message)
-          rescue MultiJson::ParseError => e
-            # same as this Mailchimp plugin issue: https://github.com/treasure-data/embulk-output-mailchimp/issues/10
-            # (a) JIRA returns error as HTML, but HTTParty try to parse it as JSON.
-            # And (b) `search_issues` method has race-condition bug. If it occurred, MultiJson::ParseError raised too.
-            html = e.message
-            title = html[%r|<title>(.*?)</title>|, 1] #=> e.g. "Unauthorized (401)"
-            raise title if title == "Atlassian Cloud Notifications - Page Unavailable"
-            count += 1
-            raise title.nil? ? "Unknown Error" : title if count > retry_times
-            Embulk.logger.warn "JIRA returns error: #{title == 'Unauthorized (401)' ? title + " due to overloading API requests. Retrying on failed items only" : title}."
-            sleep count
-            retry
-          rescue Timeout::Error => e
-            count += 1
-            raise e if count > retry_times
-            Embulk.logger.warn "Time out error."
-            sleep count # retry after some seconds for JIRA API perhaps under the overload
-            retry
-          end
-        end
-
-        def find_issue(issue_key)
-          @rate_limiter.shift
-          Jiralicious::Issue.find(issue_key)
-        end
-      end
-    end
-  end
-end

data/lib/embulk/input/jira_api/issue.rb

@@ -1,133 +0,0 @@
-module Embulk
-  module Input
-    module JiraApi
-      class Issue
-        attr_reader :id, :key, :fields
-
-        def initialize(attributes)
-          @id = attributes.fetch("id")
-
-          # https://github.com/dorack/jiralicious/blob/404b7b6d5b7020f42064cf8d7a745ab02057e728/lib/jiralicious/issue.rb#L11-L12
-          @key = attributes.fetch("jira_key")
-          @fields = attributes.fetch("fields")
-        end
-
-        def [](attribute)
-          case attribute
-          when "id"
-            return id
-          when "key"
-            return key
-          end
-
-          attribute_keys = attribute.split('.')
-
-          fetch(fields, attribute_keys)
-        end
-
-        def to_record
-          @record = {}
-
-          @record["id"] = id
-          @record["key"] = key
-
-          generate_record(fields, "")
-
-          @record
-        end
-
-        private
-
-        def fetch(fields, keys)
-          return fields if fields.nil? || (fields.is_a?(Array) && fields.empty?)
-
-          if keys.empty?
-            case fields
-            when Array
-              values = fields.map do |field|
-                if field.is_a?(String)
-                  field.to_s
-                else
-                  field.to_json
-                end
-              end
-
-              return values.join(",")
-            when Hash
-              return fields.to_json
-            else
-              return fields
-            end
-          end
-
-          target_key = keys.shift
-          if fields.is_a?(Array)
-            values = fields.map do |field|
-              if field.is_a?(Hash)
-                field[target_key]
-              else
-                field.to_json
-              end
-            end
-
-            fetch(values, keys)
-          else
-            fetch(fields[target_key], keys)
-          end
-        end
-
-        def generate_record(value, prefix_key)
-          case value
-          when Hash
-            # NOTE: If you want to flatten JSON completely, please
-            # remove this if...end and #add_heuristic_value.
-            if prefix_key.count(".") > 1
-              add_heuristic_value(value, prefix_key)
-              return
-            end
-
-            value.each_pair do |_key, _value|
-              generate_record(_value, record_key(prefix_key, _key))
-            end
-          when Array
-            if value.empty? || value.any? {|v| !v.is_a?(Hash) }
-              @record[prefix_key] = "\"#{value.map(&:to_s).join(',')}\""
-              return
-            end
-
-            # gathering values from each Hash
-            keys = value.map(&:keys).inject([]) {|sum, key| sum + key }.uniq
-            values = value.inject({}) do |sum, elem|
-              keys.each {|key| sum[key] = (sum[key] || []) << elem[key] }
-              sum
-            end
-
-            generate_record(values, prefix_key)
-          else
-            @record[prefix_key] = value
-          end
-        end
-
-        def record_key(prefix, key)
-          return key if prefix.empty?
-
-          "#{prefix}.#{key}"
-        end
-
-        def add_heuristic_value(hash, prefix_key)
-          heuristic_values = hash.select do |key, value|
-            ["name", "key", "id"].include?(key) && !value.nil?
-          end
-
-          if heuristic_values.empty?
-            @record[prefix_key] = hash.to_json
-          else
-            heuristic_values.each do |key, value|
-              @record[record_key(prefix_key, key)] = value
-            end
-          end
-        end
-      end
-    end
-  end
-end

data/lib/embulk/input/jira_input_plugin_utils.rb

@@ -1,58 +0,0 @@
-# This module contains methods for Plugin.
-
-module Embulk::Guess
-  module SchemaGuess
-    class << self
-
-      # NOTE: Original #from_hash_records uses keys of the first record only,
-      #       but JSON from JIRA API has fields which of value is sometimes nil,
-      #       sometimes JSON,
-      #       so the first record doesn't have all key for guess always.
-      # original Embulk::Guess::SchemaGuess is https://github.com/embulk/embulk/blob/57b42c31d1d539177e1e818f294550cde5b69e1f/lib/embulk/guess/schema_guess.rb#L16-L24
-      def from_hash_records(array_of_hash)
-        array_of_hash = Array(array_of_hash)
-        if array_of_hash.empty?
-          raise "SchemaGuess Can't guess schema from no records"
-        end
-        column_names = array_of_hash.map(&:keys).inject([]) {|r, a| r + a }.uniq.sort
-        samples = array_of_hash.to_a.map {|hash| column_names.map {|name| hash[name] } }
-        from_array_records(column_names, samples)
-      end
-    end
-  end
-end
-
-module Embulk
-  module Input
-    module JiraInputPluginUtils
-      # Guess::SchemaGuess.from_hash_records returns Columns
-      # containing 'index' key, but it is needless.
-      def self.guess_columns(records)
-        schema = Guess::SchemaGuess.from_hash_records(records)
-
-        schema.map do |c|
-          column = {name: c.name, type: c.type}
-          column[:format] = c.format if c.format
-          column
-        end
-      end
-
-      def self.cast(value, type)
-        return value if value.nil?
-
-        case type.to_sym
-        when :long
-          Integer(value)
-        when :double
-          Float(value)
-        when :timestamp
-          Time.parse(value)
-        when :boolean
-          !!value
-        else
-          value.to_s
-        end
-      end
-    end
-  end
-end

data/spec/embulk/input/jira-input-plugin-utils_spec.rb

@@ -1,89 +0,0 @@
-require "spec_helper"
-
-describe Embulk::Input::JiraInputPluginUtils do
-  describe ".guess_columns" do
-    subject do
-      Embulk::Input::JiraInputPluginUtils.guess_columns(records)
-    end
-
-    let(:records) do
-      [
-        {"project.key" => "FOO-1", "comment.total" => 3, "created" => "2015-03-01T00:12:00"},
-        {"project.id" => "1", "project.key" => "FOO", "comment.total" => 3, "created" => "2015-03-01T00:12:00"}
-      ]
-    end
-
-    let(:expected) do
-      [
-        {name: "comment.total", type: :long},
-        {name: "created", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S"},
-        {name: "project.id", type: :long},
-        {name: "project.key", type: :string},
-      ]
-    end
-
-    it "returns Array containing columns without 'index' key from each record" do
-      expect(subject).to eq expected
-    end
-  end
-
-  describe ".cast" do
-    subject do
-      Embulk::Input::JiraInputPluginUtils.cast(value, type)
-    end
-
-    context "when value is nil" do
-      let(:value) { nil }
-      let(:type) { :string }
-
-      it "returns nil" do
-        expect(subject).to be_nil
-      end
-    end
-
-    context "when value is not nil" do
-      let(:value) { 123 }
-
-      context "and type is :string" do
-        let(:type) { :string }
-
-        it "returns '123'" do
-          expect(subject).to eq "123"
-        end
-      end
-
-      context "and type is :long" do
-        let(:type) { :long }
-
-        it "returns 123" do
-          expect(subject).to eq 123
-        end
-      end
-
-      context "and type is :double" do
-        let(:type) { :double }
-
-        it "returns 123.0" do
-          expect(subject).to eq 123.0
-        end
-      end
-
-      context "and type is :timestamp" do
-        let(:value) { "2015-03-01T00:12:00" }
-        let(:type) { :timestamp }
-
-        it "returns Time object" do
-          expect(subject).to eq Time.new(2015, 3, 1, 0, 12, 0)
-        end
-      end
-
-      context "and type is :boolean" do
-        let(:type) { :boolean }
-
-        it "returns true" do
-          expect(subject).to be true
-        end
-      end
-    end
-  end
-end