embulk-input-marketo 0.4.0 → 0.5.0

data/lib/embulk/input/marketo_api/soap/lead.rb

@@ -53,24 +53,9 @@ module Embulk
             remaining = response.xpath('//remainingCount').text.to_i
             Embulk.logger.info "Fetched records in the range: #{records.size}"
             Embulk.logger.info "Remaining records in the range: #{remaining}"
-            records.each do |lead|
-              record = {
-                "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
-                "email" => {type: :string, value: lead.xpath('Email').text}
-              }
-              lead.xpath('leadAttributeList/attribute').each do |attr|
-                name = attr.xpath('attrName').text
-                type = attr.xpath('attrType').text
-                value = attr.xpath('attrValue').text
-                record = record.merge(
-                  name => {
-                    type: type,
-                    value: value
-                  }
-                )
-              end
 
-              block.call(record)
+            records.each do |lead|
+              process_record(lead, &block)
             end
 
             if remaining > 0
@@ -79,6 +64,26 @@ module Embulk
               nil
             end
           end
+
+          def process_record(lead, &block)
+            record = {
+              "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
+              "email" => {type: :string, value: lead.xpath('Email').text}
+            }
+            lead.xpath('leadAttributeList/attribute').each do |attr|
+              name = attr.xpath('attrName').text
+              type = attr.xpath('attrType').text
+              value = attr.xpath('attrValue').text
+              record = record.merge(
+                name => {
+                  type: type,
+                  value: value
+                }
+              )
+            end
+
+            block.call(record)
+          end
         end
       end
     end

data/test/activity_log_fixtures.rb

@@ -1,171 +1,202 @@
+require "savon_helper"
+
 module ActivityLogFixtures
-  private
+  include SavonHelper
 
-  def activity_logs_response
-    activity_logs(response)
-  end
+  private
 
-  def next_stream_activity_logs_response
-    activity_logs(next_stream_response)
+  def activity_log_xml(body)
+    <<XML
+<?xml version="1.0" encoding="UTF-8"?>
+<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
+  <SOAP-ENV:Body>
+    <ns1:successGetLeadChanges>
+      <result>
+        #{body}
+      </result>
+    </ns1:successGetLeadChanges>
+  </SOAP-ENV:Body>
+</SOAP-ENV:Envelope>
+XML
   end
 
-  def preview_activity_logs_response
-    activity_logs(preview_response)
+  def xml_ac_response_no_attributes
+    activity_log_xml <<XML
+<returnCount>2</returnCount>
+<remainingCount>0</remainingCount>
+<newStartPosition>
+  <latestCreatedAt>2015-07-14T09:13:10+09:00</latestCreatedAt>
+  <oldestCreatedAt>2015-07-14T09:13:13+09:00</oldestCreatedAt>
+  <activityCreatedAt xsi:nil="true"/>
+  <offset>offset</offset>
+</newStartPosition>
+<leadChangeRecordList>
+  <leadChangeRecord>
+    <id>1</id>
+    <activityDateTime>2015-07-14T09:00:09+09:00</activityDateTime>
+    <activityType>at1</activityType>
+    <mktgAssetName>score1</mktgAssetName>
+    <activityAttributes>
+      <attribute>
+        <attrName>Attribute Name</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>Attribute1</attrValue>
+      </attribute>
+      <attribute>
+        <attrName>Old Value</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>402</attrValue>
+      </attribute>
+    </activityAttributes>
+    <mktPersonId>100</mktPersonId>
+  </leadChangeRecord>
+  <leadChangeRecord>
+    <id>2</id>
+    <activityDateTime>2015-07-14T09:00:10+09:00</activityDateTime>
+    <activityType>at2</activityType>
+    <mktgAssetName>score2</mktgAssetName>
+    <mktPersonId>90</mktPersonId>
+  </leadChangeRecord>
+</leadChangeRecordList>
+XML
   end
 
-  def none_activity_log_response
-    activity_logs(none_response)
-  end
 
-  def activity_logs(body)
-    Struct.new(:body).new({
-      success_get_lead_changes: {
-        result: body
-      }
-    })
+  def xml_ac_response
+    activity_log_xml <<XML
+<returnCount>2</returnCount>
+<remainingCount>1</remainingCount>
+<newStartPosition>
+  <latestCreatedAt>2015-07-14T09:13:10+09:00</latestCreatedAt>
+  <oldestCreatedAt>2015-07-14T09:13:13+09:00</oldestCreatedAt>
+  <activityCreatedAt xsi:nil="true"/>
+  <offset>offset</offset>
+</newStartPosition>
+<leadChangeRecordList>
+  <leadChangeRecord>
+    <id>1</id>
+    <activityDateTime>2015-07-14T09:00:09+09:00</activityDateTime>
+    <activityType>at1</activityType>
+    <mktgAssetName>score1</mktgAssetName>
+    <activityAttributes>
+      <attribute>
+        <attrName>Attribute Name</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>Attribute1</attrValue>
+      </attribute>
+      <attribute>
+        <attrName>Old Value</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>402</attrValue>
+      </attribute>
+    </activityAttributes>
+    <mktPersonId>100</mktPersonId>
+  </leadChangeRecord>
+  <leadChangeRecord>
+    <id>2</id>
+    <activityDateTime>2015-07-14T09:00:10+09:00</activityDateTime>
+    <activityType>at2</activityType>
+    <mktgAssetName>score2</mktgAssetName>
+    <activityAttributes>
+      <attribute>
+        <attrName>Attribute Name</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>Attribute2</attrValue>
+      </attribute>
+      <attribute>
+        <attrName>Old Value</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>403</attrValue>
+      </attribute>
+    </activityAttributes>
+    <mktPersonId>90</mktPersonId>
+  </leadChangeRecord>
+</leadChangeRecordList>
+XML
   end
 
-  def response
-    {
-      return_count: "2",
-      remaining_count: "1",
-      new_start_position: {
-        latest_created_at: true,
-        oldest_created_at: "2015-07-14T00:13:13+0000",
-        activity_created_at: true,
-        offset: offset,
-      },
-      lead_change_record_list: {
-        lead_change_record: [
-          {
-            id: "1",
-            activity_date_time: "2015-07-14T00:00:09+0000",
-            activity_type: "at1",
-            mktg_asset_name: "score1",
-            activity_attributes: {
-              attribute: [
-                {
-                  attr_name: "Attribute Name",
-                  attr_type: nil,
-                  attr_value: "Attribute1",
-                },
-                {
-                  attr_name: "Old Value",
-                  attr_type: nil,
-                  attr_value: "402",
-                },
-              ],
-            },
-            mkt_person_id: "100",
-          },
-          {
-            id: "2",
-            activity_date_time: "2015-07-14T00:00:10+0000",
-            activity_type: "at2",
-            mktg_asset_name: "score2",
-            activity_attributes: {
-              attribute: [
-                {
-                  attr_name: "Attribute Name",
-                  attr_type: nil,
-                  attr_value: "Attribute2",
-                },
-                {
-                  attr_name: "Old Value",
-                  attr_type: nil,
-                  attr_value: "403",
-                },
-              ],
-            },
-            mkt_person_id: "90",
-          },
-        ]
-      }
-    }
+  def xml_ac_next_response
+    activity_log_xml <<XML
+<returnCount>1</returnCount>
+<remainingCount>0</remainingCount>
+<newStartPosition>
+</newStartPosition>
+<leadChangeRecordList>
+  <leadChangeRecord>
+    <id>3</id>
+    <activityDateTime>2015-07-14T09:00:11+09:00</activityDateTime>
+    <activityType>at3</activityType>
+    <mktgAssetName>score3</mktgAssetName>
+    <activityAttributes>
+      <attribute>
+        <attrName>Attribute Name</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>Attribute3</attrValue>
+      </attribute>
+      <attribute>
+        <attrName>Old Value</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>404</attrValue>
+      </attribute>
+    </activityAttributes>
+    <mktPersonId>100</mktPersonId>
+  </leadChangeRecord>
+</leadChangeRecordList>
+XML
   end
 
-  def offset
-    "offset"
+  def xml_ac_none_response
+    activity_log_xml <<XML
+<returnCount>0</returnCount>
+<remainingCount>0</remainingCount>
+<newStartPosition>
+</newStartPosition>
+<leadChangeRecordList>
+</leadChangeRecordList>
+XML
   end
 
-  def next_stream_response
-    {
-      return_count: 1,
-      remaining_count: 0,
-      new_start_position: {
-      },
-      lead_change_record_list: {
-        lead_change_record: [
-          {
-            id: "3",
-            activity_date_time: "2015-07-14T00:00:11+0000",
-            activity_type: "at3",
-            mktg_asset_name: "score3",
-            activity_attributes: {
-              attribute: [
-                {
-                  attr_name: "Attribute Name",
-                  attr_type: nil,
-                  attr_value: "Attribute3",
-                },
-                {
-                  attr_name: "Old Value",
-                  attr_type: nil,
-                  attr_value: "404",
-                },
-              ],
-            },
-            mkt_person_id: "100",
-          },
-        ]
-      }
-    }
+  def xml_ac_preview_response
+    activity_log_xml <<XML
+<returnCount>15</returnCount>
+<remainingCount>0</remainingCount>
+<newStartPosition>
+</newStartPosition>
+<leadChangeRecordList>
+#{xml_ac_attr(15)}
+</leadChangeRecordList>
+XML
   end
 
-  def preview_response
-    records = (1..15).map do |i|
-      {
-        id: i,
-        activity_date_time: "2015-07-14T00:00:11+0000",
-        activity_type: "at#{i}",
-        mktg_asset_name: "score#{i}",
-        activity_attributes: {
-          attribute: [
-            {
-              attr_name: "Attribute Name",
-              attr_type: nil,
-              attr_value: "Attribute#{i}",
-            },
-            {
-              attr_name: "Old Value",
-              attr_type: nil,
-              attr_value: "404",
-            },
-          ],
-        },
-        mkt_person_id: "100",
-      }
+  def xml_ac_attr(times = 15)
+    response = ""
+    (1..times).each do |n|
+    response << <<-XML
+  <leadChangeRecord>
+    <id>#{n}</id>
+    <activityDateTime>2015-07-14T00:00:11+00:00</activityDateTime>
+    <activityType>at#{n}</activityType>
+    <mktgAssetName>score#{n}</mktgAssetName>
+    <activityAttributes>
+      <attribute>
+        <attrName>Attribute Name</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>Attribute#{n}</attrValue>
+      </attribute>
+      <attribute>
+        <attrName>Old Value</attrName>
+        <attrType xsi:nil="true"/>
+        <attrValue>404</attrValue>
+      </attribute>
+    </activityAttributes>
+    <mktPersonId>100</mktPersonId>
+  </leadChangeRecord>
+    XML
     end
-
-    {
-      return_count: 15,
-      remaining_count: 0,
-      new_start_position: {},
-      lead_change_record_list: {
-        lead_change_record: records
-      }
-    }
+    response
   end
 
-  def none_response
-    {
-      result: {
-        return_count: 0,
-        remaining_count: 0,
-        new_start_position: {
-        },
-        lead_change_record_list: nil
-      }
-    }
+  def offset
+    "offset"
   end
 end

data/test/embulk/input/marketo/test_activity_log.rb

@@ -130,7 +130,7 @@ module Embulk
           def expected_guessed_columns
             [
               {name: :id, type: :long},
-              {name: :activity_date_time, type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z"},
+              {name: :activity_date_time, type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%:z"},
               {name: :activity_type, type: :string},
               {name: :mktg_asset_name, type: :string},
               {name: :mkt_person_id, type: :long},
@@ -157,17 +157,11 @@ module Embulk
 
           def test_retry
             any_instance_of(Savon::Client) do |klass|
-              stub(klass).call(:get_multiple_leads, anything) do
+              stub(klass).call(:get_lead_changes, anything) do
                 raise "foo"
               end
             end
 
-            any_instance_of(::Embulk::Input::MarketoApi::Soap::Base) do |klass|
-              task[:retry_limit].times do |n|
-                mock(klass).sleep(task[:retry_initial_wait_sec] * (2**n))
-              end
-            end
-
             mock(Embulk.logger).warn(/Retrying/).times(task[:retry_limit])
             stub(Embulk.logger).info {}
 
@@ -200,11 +194,11 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_lead_changes, message: request) do
-                activity_logs_response
+                savon_response(xml_ac_response)
               end
 
               mock(klass).call(:get_lead_changes, message: offset_request) do
-                next_stream_activity_logs_response
+                savon_response(xml_ac_next_response)
               end
             end
 
@@ -213,6 +207,7 @@ module Embulk
             mock(@page_builder).add(["3", Time.parse("2015-07-14T00:00:11+0000"), "at3", "score3", "100", "Attribute3", "404"])
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -221,12 +216,13 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_lead_changes, message: request) do
-                none_activity_log_response
+                savon_response xml_ac_none_response
               end
             end
 
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
@@ -235,22 +231,23 @@ module Embulk
 
             any_instance_of(Savon::Client) do |klass|
               mock(klass).call(:get_lead_changes, message: preview_request) do
-                preview_activity_logs_response
+                savon_response xml_ac_preview_response
               end
             end
 
             1.upto(ActivityLog::PREVIEW_COUNT) do |count|
-              mock(@page_builder).add([count, Time.parse("2015-07-14T00:00:11+0000"), "at#{count}", "score#{count}", "100", "Attribute#{count}", "404"])
+              mock(@page_builder).add([count.to_s, Time.parse("2015-07-14T00:00:11+0000"), "at#{count}", "score#{count}", "100", "Attribute#{count}", "404"])
             end
             mock(@page_builder).finish
 
+            @plugin.init
             @plugin.run
           end
 
           def test_wrong_type
             any_instance_of(Savon::Client) do |klass|
               stub(klass).call(:get_lead_changes, message: request) do
-                next_stream_activity_logs_response
+                savon_response xml_ac_next_response
               end
             end
             stub(@page_builder).add {}
@@ -263,6 +260,7 @@ module Embulk
             @plugin = ActivityLog.new(task, nil, nil, @page_builder)
 
             assert_raise(Embulk::ConfigError) do
+              @plugin.init
               @plugin.run
             end
           end
@@ -325,7 +323,7 @@ module Embulk
             encryption_key: "TOPSECRET",
             from_datetime: from_datetime,
             to_datetime: to_datetime,
-            retry_initial_wait_sec: 3,
+            retry_initial_wait_sec: 0,
             retry_limit: 2,
             columns: [
               {"name" => :id, "type" => :long},