embulk-input-marketo 0.0.1 → 0.1.0

data/test/embulk/input/marketo/test_base.rb

@@ -0,0 +1,63 @@
+require "prepare_embulk"
+require "embulk/input/marketo/base"
+
+module Embulk
+  module Input
+    module Marketo
+      class BaseTest < Test::Unit::TestCase
+        def test_target
+          assert_raise(NotImplementedError) do
+            Base.target
+          end
+        end
+
+        def test_transaction
+          control = proc {} # dummy
+          columns = task[:columns].map do |col|
+            Column.new(nil, col["name"], col["type"].to_sym)
+          end
+
+          mock(Base).resume(task, columns, 1, &control)
+          Base.transaction(config, &control)
+        end
+
+        private
+
+        def config
+          DataSource[settings.to_a]
+        end
+
+        def settings
+          {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ]
+          }
+        end
+
+        def last_updated_at
+          "2015-07-01 00:00:00+00:00"
+        end
+
+        def task
+          {
+            endpoint_url: "https://marketo.example.com",
+            wsdl_url: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ]
+          }
+        end
+
+      end
+    end
+  end
+end

data/test/embulk/input/marketo/test_lead.rb

@@ -0,0 +1,172 @@
+require "prepare_embulk"
+require "lead_fixtures"
+require "embulk/input/marketo/lead"
+
+module Embulk
+  module Input
+    module Marketo
+      class LeadTest < Test::Unit::TestCase
+        include LeadFixtures
+
+        def test_target
+          assert_equal(:lead, Lead.target)
+        end
+
+        def setup_soap
+          @soap = MarketoApi::Soap::Lead.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+          stub(Lead).soap_client(task) { @soap }
+        end
+
+        def setup_plugin
+          @page_builder = Object.new
+          @plugin = Lead.new(task, nil, nil, @page_builder)
+          stub(Embulk).logger { ::Logger.new(File::NULL) }
+        end
+
+        class RunTest < self
+          def setup
+            setup_soap
+            setup_plugin
+          end
+
+          def test_run_through
+            stub(@plugin).preview? { false }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_multiple_leads, message: request) do
+                leads_response
+              end
+
+              mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
+                next_stream_leads_response
+              end
+            end
+
+            mock(@page_builder).add(["manyo"])
+            mock(@page_builder).add(["everyleaf"])
+            mock(@page_builder).add(["ten-thousand-leaf"])
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          def test_preview_through
+            stub(@plugin).preview? { true }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_multiple_leads, message: request) do
+                preview_leads_response
+              end
+            end
+
+            Lead::PREVIEW_COUNT.times do |count|
+              mock(@page_builder).add(["manyo#{count}"])
+            end
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
+          private
+
+          def request
+            {
+              lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
+              attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
+              batch_size: 1000
+            }
+          end
+        end
+
+        class GuessTest < self
+          setup :setup_soap
+
+          def setup_soap
+            @soap = MarketoApi::Soap::Lead.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+            stub(Lead).soap_client(config) { @soap }
+          end
+
+          def test_include_metadata
+            stub(@soap).metadata { metadata }
+
+            assert_equal(
+              {"columns" => expected_guessed_columns},
+              Lead.guess(config)
+            )
+          end
+        end
+
+        def test_generate_columns
+          assert_equal(expected_guessed_columns, Lead.generate_columns(metadata))
+        end
+
+        private
+
+        def config
+          DataSource[settings.to_a]
+        end
+
+        def settings
+          {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ]
+          }
+        end
+
+        def last_updated_at
+          "2015-07-01 00:00:00+00:00"
+        end
+
+        def task
+          {
+            endpoint_url: "https://marketo.example.com",
+            wsdl_url: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+            last_updated_at: last_updated_at,
+            columns: [
+              {"name" => "Name", "type" => "string"},
+            ]
+          }
+        end
+
+        def metadata
+          [
+            {
+              name: "FieldName",
+              description: nil,
+              display_name: "The Name of Field",
+              source_object: "Lead",
+              data_type: "datetime",
+              size: nil,
+              is_readonly: false,
+              is_update_blocked: false,
+              is_name: nil,
+              is_primary_key: false,
+              is_custom: true,
+              is_dynamic: true,
+              dynamic_field_ref: "leadAttributeList",
+              updated_at: DateTime.parse("2000-01-01 22:22:22")
+            }
+          ]
+        end
+
+        def expected_guessed_columns
+          [
+            {name: "id", type: "long"},
+            {name: "email", type: "string"},
+            {name: "FieldName", type: "string"},
+          ]
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/marketo_api/soap/test_activity_log.rb

@@ -0,0 +1,109 @@
+require "embulk/input/marketo_api/soap/activity_log"
+require "activity_log_fixtures"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class ActivityLogTest < Test::Unit::TestCase
+          include ActivityLogFixtures
+
+          def setup
+            stub(Embulk).logger { ::Logger.new(IO::NULL) }
+          end
+
+          def test_each
+            request = {
+              start_position: {
+                oldest_created_at: Time.parse(last_updated_at).iso8601,
+              },
+              batch_size: 100
+            }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: request) do
+                next_stream_activity_logs_response
+              end
+            end
+
+            proc = proc{ "" }
+            activity_log_count = next_stream_activity_logs_response[:body][:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record].size
+
+            mock(proc).call(anything).times(activity_log_count)
+
+            soap.each(last_updated_at, &proc)
+          end
+
+          class TestMetadata < self
+            def setup
+              super
+
+              @savon = soap.__send__(:savon)
+              stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
+            end
+
+            def test_savon_call
+              mock(@savon).call(:get_lead_changes, message: request) {
+                next_stream_activity_logs_response
+              }
+              soap.metadata(last_updated_at)
+            end
+
+            def test_return_schema
+              stub(@savon).call(:get_lead_changes, message: request) {
+                next_stream_activity_logs_response
+              }
+              assert_equal(schema, soap.metadata(last_updated_at))
+            end
+
+            private
+
+            def request
+              {
+                start_position: {
+                  oldest_created_at: Time.parse(last_updated_at).iso8601,
+                },
+                batch_size: 100
+              }
+            end
+
+            def schema
+              metadata = [
+                {index: 0, name: "id", type: :long},
+                {index: 1, name: "activity_date_time", type: :timestamp, format: "%Y-%m-%d %H:%M:%S %z"},
+                {index: 2, name: "activity_type", type: :string},
+                {index: 3, name: "mktg_asset_name", type: :string},
+                {index: 4, name: "mkt_person_id", type: :long},
+                {index: 5, name: "Attribute Name", type: :string},
+                {index: 6, name: "Old Value", type: :long},
+              ]
+
+              metadata.map do |column|
+                Column.new(column)
+              end
+            end
+          end
+
+          private
+
+          def last_updated_at
+            "2015-07-06"
+          end
+
+          def soap
+            @soap ||= ActivityLog.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+          end
+
+          def settings
+            {
+              endpoint: "https://marketo.example.com",
+              wsdl: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/marketo_api/soap/test_base.rb

@@ -0,0 +1,43 @@
+require "embulk/input/marketo_api/soap/base"
+require "lead_fixtures"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class BaseTest < Test::Unit::TestCase
+          include LeadFixtures
+
+          class TestSignature < self
+            def setup
+              @signature = soap.__send__(:signature)
+            end
+
+            def test_sigature_keys
+              assert_equal(%w(requestTimestamp requestSignature).sort, @signature.keys.sort)
+            end
+
+            def test_is_hash
+              assert_equal(Hash, @signature.class)
+            end
+          end
+
+          private
+
+          def soap
+            @soap ||= Base.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+          end
+
+          def settings
+            {
+              endpoint: "https://marketo.example.com",
+              wsdl: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/marketo_api/soap/test_lead.rb

@@ -0,0 +1,110 @@
+require "embulk/input/marketo_api/soap/lead"
+require "lead_fixtures"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class LeadTest < Test::Unit::TestCase
+          include LeadFixtures
+
+          def test_each
+            stub(Embulk).logger { ::Logger.new(IO::NULL) }
+            last_updated_at = "2015-07-06"
+
+            request = {
+              lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
+              attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
+              batch_size: 1000
+            }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_multiple_leads, message: request) do
+                next_stream_leads_response
+              end
+            end
+
+            proc = proc{ "" }
+            leads_count = next_stream_leads_response.xpath('//leadRecord').length
+            mock(proc).call(anything).times(leads_count)
+
+            soap.each(last_updated_at, &proc)
+          end
+
+          class TestMetadata < self
+            def setup
+              @savon = soap.__send__(:savon)
+              stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
+            end
+
+            def test_savon_call
+              mock(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
+                Struct.new(:body).new(body)
+              }
+              soap.metadata
+            end
+
+            def test_return_fields
+              stub(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
+                Struct.new(:body).new(body)
+              }
+              assert_equal(fields, soap.metadata)
+            end
+
+            private
+
+            def body
+              {
+                success_describe_m_object: {
+                  result: {
+                    metadata: {
+                      field_list: {
+                        field: fields
+                      }
+                    }
+                  }
+                }
+              }
+            end
+
+            def fields
+              [
+                {
+                  name: "FieldName",
+                  description: nil,
+                  display_name: "The Name of Field",
+                  source_object: "Lead",
+                  data_type: "datetime",
+                  size: nil,
+                  is_readonly: false,
+                  is_update_blocked: false,
+                  is_name: nil,
+                  is_primary_key: false,
+                  is_custom: true,
+                  is_dynamic: true,
+                  dynamic_field_ref: "leadAttributeList",
+                  updated_at: DateTime.parse("2000-01-01 22:22:22")
+                }
+              ]
+            end
+          end
+
+          private
+
+          def soap
+            @soap ||= Lead.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+          end
+
+          def settings
+            {
+              endpoint: "https://marketo.example.com",
+              wsdl: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+            }
+          end
+        end
+      end
+    end
+  end
+end