embulk-input-jira 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e99c37b9c22a10ca74920b940eb2acf874d0676
-  data.tar.gz: b85987d31a8870a6136d14dd75f6b65605160211
+  metadata.gz: c3c5410e3708624d8c297422cbe1e1551964fd41
+  data.tar.gz: b29d96f3a03d59786269c4e1799888761f3afb95
 SHA512:
-  metadata.gz: 010bff47e3e9c3a48c24d1da78ef835450f350d7b102969e5ab89c4847425e368719009bc70511f54d2f46347f70e9235883e3e383e3c54651fbfb5a86d193be
-  data.tar.gz: 6a99ada3fcfc2fc9d0b51484a40e85d6d51797a75964251714d9ebcc8d4f3b24f6f3e6f7af56097e329fd8647f9f7d9397587d827dfc10e1a273e658c9b32005
+  metadata.gz: 76b46642effb1df89e29d8a2cd7fd60ac2321dd2f0f5736fff448836d11f427239d8c829ffdd3635965a82b3e3d95b5492a4e82180cb52de3a19f92e2201e40c
+  data.tar.gz: 1a292c77fa20cef49500dadc2dde6929f6aa7bdf0be8f732e3e32049664c6eeb1815293443aa006b5546681fbde876817f667f90ff3f24dc5d7ee485a2d69ca8

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.0.4 - 2015-06-22
+* [enhancement] Add more schema generated by guess command [#27](https://github.com/treasure-data/embulk-input-jira/pull/27)
+
 ## 0.0.3 - 2015-06-12
 * [enhancement] Limit records for guess and preview [#25](https://github.com/treasure-data/embulk-input-jira/pull/25)
 * [maintenance] Define version number as constant [#24](https://github.com/treasure-data/embulk-input-jira/pull/24)

data/lib/embulk/input/jira/issue.rb

@@ -20,46 +20,112 @@ module Embulk
             return key
           end
 
-          chunk = fields
-          attribute.split('.').each do |key|
-            chunk = chunk[key]
-            return chunk if chunk.nil?
-          end
+          attribute_keys = attribute.split('.')
 
-          if chunk.is_a?(Array) || chunk.is_a?(Hash)
-            chunk.to_json.to_s
-          else
-            chunk
-          end
+          fetch(fields, attribute_keys)
         end
 
         def to_record
-          record = {}
+          @record = {}
 
-          record["id"] = id
-          record["key"] = key
+          @record["id"] = id
+          @record["key"] = key
 
-          fields.each_pair do |key, value|
-            record_key = key
-            record_value = value.to_json.to_s
+          generate_record(fields, "")
+
+          @record
+        end
+
+        private
+
+        def fetch(fields, keys)
+          return fields if fields.nil? || (fields.is_a?(Array) && fields.empty?)
+
+          if keys.empty?
+            case fields
+            when Array
+              values = fields.map do |field|
+                if field.is_a?(String)
+                  field.to_s
+                else
+                  field.to_json
+                end
+              end
 
-            case value
-            when String
-              record_value = value
+              return values.join(",")
             when Hash
-              if value.keys.include?("name")
-                record_key += ".name"
-                record_value = value["name"]
-              elsif value.keys.include?("id")
-                record_key += ".id"
-                record_value = value["id"]
+              return fields.to_json
+            else
+              return fields
+            end
+          end
+
+          target_key = keys.shift
+          if fields.is_a?(Array)
+            values = fields.map do |field|
+              if field.is_a?(Hash)
+                field[target_key]
+              else
+                field.to_json
               end
             end
 
-            record[record_key] = record_value
+            fetch(values, keys)
+          else
+            fetch(fields[target_key], keys)
           end
+        end
+
+        def generate_record(value, prefix_key)
+          case value
+          when Hash
+            # NOTE: If you want to flatten JSON completely, please
+            # remove this if...end and #add_heuristic_value.
+            if prefix_key.count(".") > 1
+              add_heuristic_value(value, prefix_key)
+              return
+            end
 
-          record
+            value.each_pair do |_key, _value|
+              generate_record(_value, record_key(prefix_key, _key))
+            end
+          when Array
+            if value.empty? || value.any? {|v| !v.is_a?(Hash) }
+              @record[prefix_key] = "\"#{value.map(&:to_s).join(',')}\""
+              return
+            end
+
+            # gathering values from each Hash
+            keys = value.map(&:keys).inject([]) {|sum, key| sum + key }.uniq
+            values = value.inject({}) do |sum, elem|
+              keys.each {|key| sum[key] = (sum[key] || []) << elem[key] }
+              sum
+            end
+
+            generate_record(values, prefix_key)
+          else
+            @record[prefix_key] = value
+          end
+        end
+
+        def record_key(prefix, key)
+          return key if prefix.empty?
+
+          "#{prefix}.#{key}"
+        end
+
+        def add_heuristic_value(hash, prefix_key)
+          heuristic_values = hash.select do |key, value|
+            ["name", "key", "id"].include?(key) && !value.nil?
+          end
+
+          if heuristic_values.empty?
+            @record[prefix_key] = hash.to_json
+          else
+            heuristic_values.each do |key, value|
+              @record[record_key(prefix_key, key)] = value
+            end
+          end
         end
       end
     end

data/lib/embulk/input/jira/version.rb

@@ -1,7 +1,7 @@
 module Embulk
   module Input
     module Jira
-      VERSION = "0.0.3".freeze
+      VERSION = "0.0.4".freeze
     end
   end
 end

data/lib/embulk/input/jira_input_plugin_utils.rb

@@ -1,5 +1,27 @@
 # This module contains methods for Plugin.
 
+module Embulk::Guess
+  module SchemaGuess
+    class << self
+
+      # NOTE: Original #from_hash_records uses keys of the first record only,
+      #       but JSON from JIRA API has fields which of value is sometimes nil,
+      #       sometimes JSON,
+      #       so the first record doesn't have all key for guess always.
+      # original Embulk::Guess::SchemaGuess is https://github.com/embulk/embulk/blob/57b42c31d1d539177e1e818f294550cde5b69e1f/lib/embulk/guess/schema_guess.rb#L16-L24
+      def from_hash_records(array_of_hash)
+        array_of_hash = Array(array_of_hash)
+        if array_of_hash.empty?
+          raise "SchemaGuess Can't guess schema from no records"
+        end
+        column_names = array_of_hash.map(&:keys).inject([]) {|r, a| r + a }.uniq.sort
+        samples = array_of_hash.to_a.map {|hash| column_names.map {|name| hash[name] } }
+        from_array_records(column_names, samples)
+      end
+    end
+  end
+end
+
 module Embulk
   module Input
     module JiraInputPluginUtils

data/spec/embulk/input/jira/issue_spec.rb

@@ -49,85 +49,139 @@ describe Embulk::Input::Jira::Issue do
       {
         "id" => "1",
         "jira_key" => "FOO-1",
-        "fields" => {
-          "summary" => "jira issue",
-          "project" => project_attribute,
-          "labels" =>
-          [
-            "Feature",
-            "WantTo"
-          ],
-          "priority" => {
-            "iconUrl" => "https://jira-api/images/icon.png",
-            "name" => "Must",
-            "id" => "1"
-          },
-          "customfield_1" => nil,
-        }
+        "fields" => fields_attributes
       }
     end
 
-    let(:project_attribute) do
-      {
-        "key" => "FOO",
-      }
+    context "when target attribute_name is especially key" do
+      let(:fields_attributes) { {} }
+
+      context 'id' do
+        let(:attribute_name) { 'id' }
+
+        it "returns issue id" do
+          expect(subject).to eq "1"
+        end
+      end
+
+      context 'key' do
+        let(:attribute_name) { 'key' }
+
+        it "returns issue key" do
+          expect(subject).to eq "FOO-1"
+        end
+      end
     end
 
-    context 'id' do
-      let(:attribute_name) { 'id' }
+    context "when fields_attributes is empty Hash" do
+      let(:fields_attributes) { {} }
+
+      let(:attribute_name) { 'key1' }
+
+      it "returns nil" do
+        expect(subject).to eq nil
+      end
+    end
+
+    context "when fields_attributes is empty Hash including Array" do
+      let(:fields_attributes) { { 'key1' => [] } }
+
+      let(:attribute_name) { 'key1' }
 
-      it "returns issue id" do
-        expect(subject).to eq "1"
+      it "returns empty Array" do
+        expect(subject).to eq []
       end
     end
 
-    context 'key' do
+    context "when fields_attributes is Hash" do
+      let(:fields_attributes) do
+        {'key' => 'value'}
+      end
+
       let(:attribute_name) { 'key' }
 
-      it "returns issue key" do
+      it "returns 'value'" do
         expect(subject).to eq "FOO-1"
       end
     end
 
-    context 'summary' do
-      let(:attribute_name) { 'summary' }
+    context "when fields_attributes is `{'key1' => {'key2' => 'value2', 'key3' => 'value3'}}`" do
+      let(:fields_attributes) do
+        {'key1' => {'key2' => 'value2', 'key3' => 'value3'}}
+      end
 
-      it "returns issue summary" do
-        expect(subject).to eq 'jira issue'
+      context "when attribute_name is 'key1'" do
+        let(:attribute_name) { 'key1' }
+
+        it "returns key1's JSON" do
+          expect(subject).to eq({'key2' => 'value2', 'key3' => 'value3'}.to_json)
+        end
+      end
+
+      context "when attribute_name is 'key1.key2'" do
+        let(:attribute_name) { 'key1.key2' }
+
+        it "returns 'value2'" do
+          expect(subject).to eq 'value2'
+        end
       end
     end
 
-    context 'project.key' do
-      let(:attribute_name) { 'project.key' }
+    context "when fields_attributes is `{'key1' => [{'key2' => 'value2-1'}, {'key2' => 'value2-2'}]}`" do
+      let(:fields_attributes) do
+        {'key1' => [{'key2' => 'value2-1'}, {'key2' => 'value2-2'}]}
+      end
+
+      context "when attribute_name is 'key1'" do
+        let(:attribute_name) { 'key1' }
 
-      context "when project is not nil" do
-        it "returns issue's project key" do
-          expect(subject).to eq 'FOO'
+        it "returns JSON array" do
+          expect(subject).to eq '{"key2":"value2-1"},{"key2":"value2-2"}'
         end
       end
 
-      context "when project is nil" do
-        let(:project_attribute) { nil }
+      context "when attribute_name is 'key1.key2'" do
+        let(:attribute_name) { 'key1.key2' }
 
-        it "returns nil" do
-          expect(subject).to be_nil
+        it "returns CSV values assigned by 'key2' key" do
+          expect(subject).to eq 'value2-1,value2-2'
         end
       end
     end
 
-    context 'labels' do
-      let(:attribute_name) { 'labels' }
+    context "when fields_attributes is `{'key1' => [{'key2' => 'value2-1'}, nil]}`" do
+      let(:fields_attributes) do
+        {'key1' => [{'key2' => 'value2-1'}, nil]}
+      end
+
+      context "when attribute_name is 'key1'" do
+        let(:attribute_name) { 'key1' }
+
+        it "returns CSV value including null" do
+          expect(subject).to eq '{"key2":"value2-1"},null'
+        end
+      end
+
+      context "when attribute_name is 'key1.key2'" do
+        let(:attribute_name) { 'key1.key2' }
 
-      it "returns issue's labels JSON string" do
-        expect(subject).to eq '["Feature","WantTo"]'
+        it "returns JSON array including null" do
+          expect(subject).to eq 'value2-1,null'
+        end
       end
     end
 
-    context 'priority' do
-      let(:attribute_name) { 'priority' }
+    context "when fields_attributes is `{'key1' => ['element1', 'element2', 'element3']}`" do
+      let(:fields_attributes) do
+        {
+          'key1' => ['element1', 'element2', 'element3'],
+        }
+      end
+
+      let(:attribute_name) { 'key1' }
 
-      it "returns issue's priority JSON string" do
-        expect(subject).to eq '{"iconUrl":"https://jira-api/images/icon.png","name":"Must","id":"1"}'
+      it "returns CSV values assigned by 'key1'" do
+        expect(subject).to eq 'element1,element2,element3'
       end
     end
   end
@@ -137,44 +191,204 @@ describe Embulk::Input::Jira::Issue do
       Embulk::Input::Jira::Issue.new(issue_attributes).to_record
     end
 
+    shared_examples 'return guessed record' do
+      it do
+        expect(subject).to eq expected_record
+      end
+    end
+
     let(:issue_attributes) do
+      {"jira_key" => "FOO-1", "id" => "1", "fields" => fields_attributes}
+    end
+
+    let(:expected_record) do
       {
-        "id" => 1,
-        "jira_key" => "FOO-1",
-        "fields" => {
-          "summary" => "jira issue",
-          "project" => {
-            "id" => "FOO",
-          },
-          "labels" =>
+        "key" => "FOO-1",
+        "id" => "1"
+      }.merge(exptected_record_from_fields)
+    end
+
+    context "when fields_attributes is `{'key' => 'value'}`" do
+      let(:fields_attributes) do
+        {'key' => 'value'}
+      end
+
+      let(:exptected_record_from_fields) do
+        {'key' => 'value'}
+      end
+
+      it_behaves_like "return guessed record"
+    end
+
+    context "when fields_attributes is `{'key1' => {'key2' => 'value2', 'key3' => 'value3'}}`" do
+      let(:fields_attributes) do
+        {
+          'key1' => {
+            'key2' => 'value2',
+            'key3' => 'value3',
+          }
+        }
+      end
+
+      let(:exptected_record_from_fields) do
+        {
+          "key1.key2" => "value2",
+          "key1.key3" => "value3"
+        }
+      end
+
+      it_behaves_like "return guessed record"
+    end
+
+    context "when fields_attributes is `{'key1' => {'key2' => {'key3' => last_child}}}`" do
+      let(:fields_attributes) do
+        {'key1' => {'key2' => {'key3' => last_child}}}
+      end
+
+      context "when last_child is String" do
+        let(:last_child) do
+          "String"
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3" => "String"}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+
+      context "when last_child has 'key' key" do
+        let(:last_child) do
+          {"key" => "BAR-1"}
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3.key" => "BAR-1"}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+
+      context "when last_child has 'id' key" do
+        let(:last_child) do
+          {"id" => "20"}
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3.id" => "20"}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+
+      context "when last_child has 'name' key" do
+        let(:last_child) do
+          {"name" => "Foo name"}
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3.name" => "Foo name"}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+
+      context "when last_child has another key except 'key', 'id', 'name'" do
+        let(:last_child) do
+          {"customfield_0001" => "value0001"}
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3" => "{\"customfield_0001\":\"value0001\"}"}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+
+      context "when last_child Hash Array" do
+        let(:last_child) do
           [
-            "Feature",
-            "WantTo"
-          ],
-          "priority" => {
-            "iconUrl" => "https://jira-api/images/icon.png",
-            "name" => "Must",
-            "id" => "1"
-          },
-          "customfield_1" => nil,
+            {"key4" => "value4", "key5" => "value5-1"},
+            {"key6" => "value6", "key5" => "value5-2"},
+          ]
+        end
+
+        let(:exptected_record_from_fields) do
+          {"key1.key2.key3" => '{"key4":["value4",null],"key5":["value5-1","value5-2"],"key6":[null,"value6"]}'}
+        end
+
+        it_behaves_like "return guessed record"
+      end
+    end
+
+    context "when fields_attributes is `{'key1' => ['element1-1', 'element1-2', 'element1-3'], 'key2' => ['element2-1', 'element2-2', 'element2-3']}`" do
+      let(:fields_attributes) do
+        {
+          'key1' => ['element1-1', 'element1-2', 'element1-3'],
+          'key2' => ['element2-1', 'element2-2', 'element2-3'],
         }
-      }
+      end
+
+      let(:exptected_record_from_fields) do
+        {
+          'key1' => '"element1-1,element1-2,element1-3"',
+          'key2' => '"element2-1,element2-2,element2-3"',
+        }
+      end
+
+      it_behaves_like "return guessed record"
     end
 
-    let(:expected) do
-      {
-        "id" => 1,
-        "key" => "FOO-1",
-        "summary" => "jira issue",
-        "project.id" => "FOO",
-        "labels" => "[\"Feature\",\"WantTo\"]",
-        "priority.name" => "Must",
-        "customfield_1" => "null"
-      }
+    context "when fields_attributes is `{'key' => []}`" do
+      let(:fields_attributes) do
+        {'key' => []}
+      end
+
+      let(:exptected_record_from_fields) do
+        {"key" => '""'}
+      end
+
+      it_behaves_like "return guessed record"
     end
 
-    it 'return guessed record' do
-      expect(subject).to eq expected
+    context "when fields_attributes is `{'key1' => { 'key2' => [{'key3' => 'value3-1'}, {'key3'=> 'value3-2'}]}}`" do
+      let(:fields_attributes) do
+        {
+          'key1' => {
+            'key2' => [
+              {'key3' => 'value3-1'}, {'key3'=> 'value3-2'},
+            ]
+          }
+        }
+      end
+
+      let(:exptected_record_from_fields) do
+        {
+          "key1.key2.key3" => '"value3-1,value3-2"',
+        }
+      end
+
+      it_behaves_like "return guessed record"
+    end
+
+    context "when fields_attributes is `{'key1' => { 'key2' => ['element2-1', 'element2-2', 'element2-3'], 'key3' => ['element3-1', 'element3-2', 'element3-3']}}`" do
+      let(:fields_attributes) do
+        {
+          'key1' => {
+            'key2' => ['element2-1', 'element2-2', 'element2-3'],
+            'key3' => ['element3-1', 'element3-2', 'element3-3']
+          }
+        }
+      end
+
+      let(:exptected_record_from_fields) do
+        {
+          "key1.key2" => '"element2-1,element2-2,element2-3"',
+          "key1.key3" => '"element3-1,element3-2,element3-3"',
+        }
+      end
+
+      it_behaves_like "return guessed record"
     end
   end
 end

data/spec/embulk/input/jira-input-plugin-utils_spec.rb

@@ -8,15 +8,17 @@ describe Embulk::Input::JiraInputPluginUtils do
 
     let(:records) do
       [
-        {"project_key" => "FOO", "comment.total" => 3, "created" => "2015-03-01T00:12:00"}
+        {"project.key" => "FOO-1", "comment.total" => 3, "created" => "2015-03-01T00:12:00"},
+        {"project.id" => "1", "project.key" => "FOO", "comment.total" => 3, "created" => "2015-03-01T00:12:00"}
       ]
     end
 
     let(:expected) do
       [
-        {name: "project_key", type: :string},
         {name: "comment.total", type: :long},
-        {name: "created", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S"}
+        {name: "created", type: :timestamp, format: "%Y-%m-%dT%H:%M:%S"},
+        {name: "project.id", type: :long},
+        {name: "project.key", type: :string},
       ]
     end
 

data/spec/embulk/input/jira_input_plugin_spec.rb

@@ -116,10 +116,12 @@ describe Embulk::Input::JiraInputPlugin do
     let(:guessed_config) do
       {
         "columns" => [
+          {name: "comment.comments", type: :string},
+          {name: "comment.total", type: :long},
           {name: "id", type: :long},
           {name: "key", type: :string},
+          {name: "project.key", type: :string},
           {name: "project.name", type: :string},
-          {name: "comment", type: :string}
         ]
       }
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jira
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-12 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement