embulk-input-marketo 0.0.1 → 0.1.0

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -0,0 +1,55 @@
+require "savon"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class Base
+          attr_reader :endpoint, :wsdl, :user_id, :encryption_key
+
+          def initialize(endpoint, wsdl, user_id, encryption_key)
+            @endpoint = endpoint
+            @wsdl = wsdl
+            @user_id = user_id
+            @encryption_key = encryption_key
+          end
+
+          private
+
+          def savon
+            headers = {
+              'ns1:AuthenticationHeader' => {
+                "mktowsUserId" => user_id,
+              }.merge(signature)
+            }
+            # NOTE: Do not memoize this to use always fresh signature (avoid 20016 error)
+            # ref. https://jira.talendforge.org/secure/attachmentzip/unzip/167201/49761%5B1%5D/Marketo%20Enterprise%20API%202%200.pdf (41 page)
+            Savon.client(
+              log: true,
+              logger: Embulk.logger,
+              wsdl: wsdl,
+              soap_header: headers,
+              endpoint: endpoint,
+              open_timeout: 90,
+              read_timeout: 300,
+              raise_errors: true,
+              namespace_identifier: :ns1,
+              env_namespace: 'SOAP-ENV'
+            )
+          end
+
+          def signature
+            timestamp = Time.now.to_s
+            encryption_string = timestamp + user_id
+            digest = OpenSSL::Digest.new('sha1')
+            hashed_signature = OpenSSL::HMAC.hexdigest(digest, encryption_key, encryption_string)
+            {
+              'requestTimestamp' => timestamp,
+              'requestSignature' => hashed_signature.to_s
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo_api/soap/lead.rb

@@ -0,0 +1,75 @@
+require "embulk/input/marketo_api/soap/base"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class Lead < Base
+          def metadata
+            # http://developers.marketo.com/documentation/soap/describemobject/
+            response = savon.call(:describe_m_object, message: {object_name: "LeadRecord"})
+            response.body[:success_describe_m_object][:result][:metadata][:field_list][:field]
+          end
+
+          def each(last_updated_at, &block)
+            # http://developers.marketo.com/documentation/soap/getmultipleleads/
+
+            last_updated_at = Time.parse(last_updated_at).iso8601
+
+            # TODO: generate request in #fetch
+            # TODO: use PREVIEW_COUNT as batch_size in preview
+            request = {
+              lead_selector: {
+                oldest_updated_at: last_updated_at,
+              },
+              attributes!: {
+                lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
+              },
+              batch_size: 1000,
+            }
+
+            stream_position = fetch(request, &block)
+
+            while stream_position
+              stream_position = fetch(request.merge(stream_position: stream_position), &block)
+            end
+          end
+
+          private
+
+          def fetch(request = {}, &block)
+            response = savon.call(:get_multiple_leads, message: request)
+
+            remaining = response.xpath('//remainingCount').text.to_i
+            Embulk.logger.info "Remaining records: #{remaining}"
+            response.xpath('//leadRecordList/leadRecord').each do |lead|
+              record = {
+                "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
+                "email" => {type: :string, value: lead.xpath('Email').text}
+              }
+              lead.xpath('leadAttributeList/attribute').each do |attr|
+                name = attr.xpath('attrName').text
+                type = attr.xpath('attrType').text
+                value = attr.xpath('attrValue').text
+                record = record.merge(
+                  name => {
+                    type: type,
+                    value: value
+                  }
+                )
+              end
+
+              block.call(record)
+            end
+
+            if remaining > 0
+              response.xpath('//newStreamPosition').text
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/test/activity_log_fixtures.rb

@@ -0,0 +1,155 @@
+module ActivityLogFixtures
+  private
+
+  def activity_logs_response
+    activity_logs(response)
+  end
+
+  def next_stream_activity_logs_response
+    activity_logs(next_stream_response)
+  end
+
+  def preview_activity_logs_response
+    activity_logs(preview_response)
+  end
+
+  def activity_logs(body)
+    Struct.new(:body).new({
+      success_get_lead_changes: {
+        result: body
+      }
+    })
+  end
+
+  def response
+    {
+      return_count: "2",
+      remaining_count: "1",
+      new_start_position: {
+        latest_created_at: true,
+        oldest_created_at: DateTime.parse("2015-07-14T00:13:13+00:00"),
+        activity_created_at: true,
+        offset: offset,
+      },
+      lead_change_record_list: {
+        lead_change_record: [
+          {
+            id: "1",
+            activity_date_time: DateTime.parse("2015-07-14T00:00:09+00:00"),
+            activity_type: "at1",
+            mktg_asset_name: "score1",
+            activity_attributes: {
+              attribute: [
+                {
+                  attr_name: "Attribute Name",
+                  attr_type: nil,
+                  attr_value: "Attribute1",
+                },
+                {
+                  attr_name: "Old Value",
+                  attr_type: nil,
+                  attr_value: "402",
+                },
+              ],
+            },
+            mkt_person_id: "100",
+          },
+          {
+            id: "2",
+            activity_date_time: DateTime.parse("2015-07-14T00:00:10+00:00"),
+            activity_type: "at2",
+            mktg_asset_name: "score2",
+            activity_attributes: {
+              attribute: [
+                {
+                  attr_name: "Attribute Name",
+                  attr_type: nil,
+                  attr_value: "Attribute2",
+                },
+                {
+                  attr_name: "Old Value",
+                  attr_type: nil,
+                  attr_value: "403",
+                },
+              ],
+            },
+            mkt_person_id: "90",
+          },
+        ]
+      }
+    }
+  end
+
+  def offset
+    "offset"
+  end
+
+  def next_stream_response
+    {
+      return_count: 1,
+      remaining_count: 0,
+      new_start_position: {
+      },
+      lead_change_record_list: {
+        lead_change_record: [
+          {
+            id: "3",
+            activity_date_time: DateTime.parse("2015-07-14T00:00:11+00:00"),
+            activity_type: "at3",
+            mktg_asset_name: "score3",
+            activity_attributes: {
+              attribute: [
+                {
+                  attr_name: "Attribute Name",
+                  attr_type: nil,
+                  attr_value: "Attribute3",
+                },
+                {
+                  attr_name: "Old Value",
+                  attr_type: nil,
+                  attr_value: "404",
+                },
+              ],
+            },
+            mkt_person_id: "100",
+          },
+        ]
+      }
+    }
+  end
+
+  def preview_response
+    records = (1..15).map do |i|
+      {
+        id: i,
+        activity_date_time: DateTime.parse("2015-07-14T00:00:11+00:00"),
+        activity_type: "at#{i}",
+        mktg_asset_name: "score#{i}",
+        activity_attributes: {
+          attribute: [
+            {
+              attr_name: "Attribute Name",
+              attr_type: nil,
+              attr_value: "Attribute#{i}",
+            },
+            {
+              attr_name: "Old Value",
+              attr_type: nil,
+              attr_value: "404",
+            },
+          ],
+        },
+        mkt_person_id: "100",
+      }
+    end
+
+    {
+      return_count: 15,
+      remaining_count: 0,
+      new_start_position: {},
+      lead_change_record_list: {
+        lead_change_record: records
+      }
+    }
+  end
+end

data/test/embulk/input/marketo/test_activity_log.rb

@@ -0,0 +1,201 @@
+require "prepare_embulk"
+require "embulk/input/marketo/activity_log"
+require "activity_log_fixtures"
+
+module Embulk
+  module Input
+    module Marketo
+      class ActivityLogTest < Test::Unit::TestCase
+        include ActivityLogFixtures
+
+        def test_target
+          assert_equal(:activity_log, ActivityLog.target)
+        end
+
+        class GuessTest < self
+          setup :setup_soap
+
+          def setup_soap
+            @soap = MarketoApi::Soap::ActivityLog.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+            stub(ActivityLog).soap_client(config) { @soap }
+          end
+
+          def test_include_metadata
+            stub(@soap).metadata(last_updated_at, batch_size: ActivityLog::PREVIEW_COUNT) { Guess::SchemaGuess.from_hash_records(records) }
+
+            assert_equal(
+              {"columns" => expected_guessed_columns},
+              Marketo::ActivityLog.guess(config)
+            )
+          end
+
+          private
+
+          def records
+            [
+              {
+                id: "12",
+                activity_date_time: "2015-06-25T00:12:00+00:00",
+                activity_type: "Visit Webpage",
+                mktg_asset_name: "webpage.example.com/person/1/edit",
+                mkt_person_id: "34",
+                "Webpage ID" => "56",
+                "Webpage URL" => "/person/1/edit",
+                "Referrer URL" => "https://webpage.example.com",
+                "Client IP Address" => "127.0.0.1",
+                "User Agent" => "UserAgent",
+                "Message Id" => "78",
+                "Created At" => "2015-07-06 19:00:02",
+                "Lead ID" => "90"
+              }
+            ]
+          end
+
+          def expected_guessed_columns
+            [
+              {name: :id, type: :long},
+              {name: :activity_date_time, type: :timestamp, format: "%Y-%m-%dT%H:%M:%S%z"},
+              {name: :activity_type, type: :string},
+              {name: :mktg_asset_name, type: :string},
+              {name: :mkt_person_id, type: :long},
+              {name: "Webpage ID", type: :long},
+              {name: "Webpage URL", type: :string},
+              {name: "Referrer URL", type: :string},
+              {name: "Client IP Address", type: :string},
+              {name: "User Agent", type: :string},
+              {name: "Message Id", type: :long},
+              {name: "Created At", type: :timestamp, format: "%Y-%m-%d %H:%M:%S"},
+              {name: "Lead ID", type: :long}
+            ]
+          end
+        end
+
+        class RunTest < self
+          def setup_soap
+            @soap = MarketoApi::Soap::ActivityLog.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+            stub(ActivityLog).soap_client(task) { @soap }
+          end
+
+          def setup_plugin
+            @page_builder = Object.new
+            @plugin = ActivityLog.new(task, nil, nil, @page_builder)
+            stub(Embulk).logger { ::Logger.new(IO::NULL) }
+          end
+
+          def setup
+            setup_soap
+            setup_plugin
+          end
+
+          def test_run_through
+            stub(@plugin).preview? { false }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: request) do
+                activity_logs_response
+              end
+
+              mock(klass).call(:get_lead_changes, message: offset_request) do
+                next_stream_activity_logs_response
+              end
+            end
+
+            mock(@page_builder).add(["1", Time.parse("2015-07-14 09:00:09 +0900"), "at1", "score1", "100", "Attribute1", "402"])
+            mock(@page_builder).add(["2", Time.parse("2015-07-14 09:00:10 +0900"), "at2", "score2", "90", "Attribute2", "403"])
+            mock(@page_builder).add(["3", Time.parse("2015-07-14 09:00:11 +0900"), "at3", "score3", "100", "Attribute3", "404"])
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          def test_preview_through
+            stub(@plugin).preview? { true }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: preview_request) do
+                preview_activity_logs_response
+              end
+            end
+
+            1.upto(ActivityLog::PREVIEW_COUNT) do |count|
+              mock(@page_builder).add([count, Time.parse("2015-07-14 09:00:11 +0900"), "at#{count}", "score#{count}", "100", "Attribute#{count}", "404"])
+            end
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          private
+
+          def request
+            {
+              start_position: {
+                oldest_created_at: Time.parse(last_updated_at).iso8601,
+              },
+              batch_size: 100
+            }
+          end
+
+          def offset_request
+            {
+              start_position: {
+                offset: "offset"
+              },
+              batch_size: 100
+            }
+          end
+        end
+
+          def preview_request
+            {
+              start_position: {
+                oldest_created_at: Time.parse(last_updated_at).iso8601,
+              },
+              batch_size: ActivityLog::PREVIEW_COUNT
+            }
+          end
+
+        private
+
+        def settings
+          {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+          }
+        end
+
+        def config
+          DataSource[settings.to_a]
+        end
+
+        def task
+          {
+            endpoint_url: "https://marketo.example.com",
+            wsdl_url: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+            columns: [
+              {"name" => :id, "type" => :long},
+              {"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
+              {"name" => :activity_type, "type" => :string},
+              {"name" => :mktg_asset_name, "type" => :string},
+              {"name" => :mkt_person_id, "type" => :long},
+              {"name" => "Attribute Name", "type" => :string},
+              {"name" => "Old Value", "type" => :string},
+            ]
+          }
+        end
+
+        def last_updated_at
+          "2015-07-01 00:00:00+00:00"
+        end
+      end
+    end
+  end
+end