embulk-input-marketo 0.0.1 → 0.1.0

data/test/embulk/input/test_marketo_api.rb

@@ -0,0 +1,28 @@
+require "embulk/input/marketo_api"
+
+module Embulk
+  module Input
+    class MarketoApiTest < Test::Unit::TestCase
+      class SoapClientTest < self
+        data do
+          {
+            lead: [:lead, MarketoApi::Soap::Lead],
+            activity_log: [:activity_log, MarketoApi::Soap::ActivityLog],
+          }
+        end
+
+        def test_valid_target(data)
+          target, expected = data
+          actual = MarketoApi.soap_client({}, target)
+          assert_equal(expected, actual.class)
+        end
+
+        def test_unknown_target
+          assert_raise do
+            MarketoApi.soap_client({}, "unknown")
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-06 00:00:00.000000000 Z
+date: 2015-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -146,11 +146,21 @@ files:
 - README.md
 - Rakefile
 - embulk-input-marketo.gemspec
-- lib/embulk/input/marketo.rb
+- lib/embulk/input/marketo/activity_log.rb
+- lib/embulk/input/marketo/base.rb
+- lib/embulk/input/marketo/lead.rb
 - lib/embulk/input/marketo_api.rb
-- lib/embulk/input/marketo_api/soap.rb
-- test/embulk/input/marketo_api/test_soap.rb
-- test/embulk/input/test_marketo.rb
+- lib/embulk/input/marketo_api/soap/activity_log.rb
+- lib/embulk/input/marketo_api/soap/base.rb
+- lib/embulk/input/marketo_api/soap/lead.rb
+- test/activity_log_fixtures.rb
+- test/embulk/input/marketo/test_activity_log.rb
+- test/embulk/input/marketo/test_base.rb
+- test/embulk/input/marketo/test_lead.rb
+- test/embulk/input/marketo_api/soap/test_activity_log.rb
+- test/embulk/input/marketo_api/soap/test_base.rb
+- test/embulk/input/marketo_api/soap/test_lead.rb
+- test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
 - test/prepare_embulk.rb
 - test/run-test.rb
@@ -179,8 +189,14 @@ signing_key:
 specification_version: 4
 summary: Marketo input plugin for Embulk
 test_files:
-- test/embulk/input/marketo_api/test_soap.rb
-- test/embulk/input/test_marketo.rb
+- test/activity_log_fixtures.rb
+- test/embulk/input/marketo/test_activity_log.rb
+- test/embulk/input/marketo/test_base.rb
+- test/embulk/input/marketo/test_lead.rb
+- test/embulk/input/marketo_api/soap/test_activity_log.rb
+- test/embulk/input/marketo_api/soap/test_base.rb
+- test/embulk/input/marketo_api/soap/test_lead.rb
+- test/embulk/input/test_marketo_api.rb
 - test/lead_fixtures.rb
 - test/prepare_embulk.rb
 - test/run-test.rb

data/lib/embulk/input/marketo.rb

@@ -1,138 +0,0 @@
-require "embulk/input/marketo_api"
-
-module Embulk
-  module Input
-    class Marketo < InputPlugin
-      PREVIEW_COUNT = 15
-
-      Plugin.register_input("marketo", self)
-
-      def self.transaction(config, &control)
-        endpoint_url = config.param(:endpoint, :string)
-
-        task = {
-          endpoint_url: endpoint_url,
-          wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-          user_id: config.param(:user_id, :string),
-          encryption_key: config.param(:encryption_key, :string),
-          last_updated_at: config.param(:last_updated_at, :string),
-          columns: config.param(:columns, :array)
-        }
-
-        columns = []
-
-        task[:columns].each do |column|
-          name = column["name"]
-          type = column["type"].to_sym
-
-          columns << Column.new(nil, name, type, column["format"])
-        end
-
-        resume(task, columns, 1, &control)
-      end
-
-      def self.resume(task, columns, count, &control)
-        commit_reports = yield(task, columns, count)
-
-        next_config_diff = {}
-        return next_config_diff
-      end
-
-      def self.guess(config)
-        client = soap_client(config)
-        metadata = client.lead_metadata
-
-        return {"columns" => generate_columns(metadata)}
-      end
-
-      def self.soap_client(config)
-        @soap ||=
-          begin
-            endpoint_url = config.param(:endpoint, :string),
-            soap_config = {
-              endpoint_url: endpoint_url,
-              wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-              user_id: config.param(:user_id, :string),
-              encryption_key: config.param(:encryption_key, :string),
-            }
-
-            MarketoApi.soap_client(soap_config)
-          end
-      end
-
-      def self.generate_columns(metadata)
-        columns = [
-          {name: "id", type: "long"},
-          {name: "email", type: "string"},
-        ]
-
-        metadata.each do |field|
-          type =
-            case field[:data_type]
-            when "integer"
-              "long"
-            when "dateTime", "date"
-              "timestamp"
-            when "string", "text", "phone", "currency"
-              "string"
-            when "boolean"
-              "boolean"
-            when "float"
-              "double"
-            else
-              "string"
-            end
-
-          columns << {name: field[:name], type: type}
-        end
-
-        columns
-      end
-
-      def init
-        @last_updated_at = task[:last_updated_at]
-        @columns = task[:columns]
-        @soap = MarketoApi.soap_client(task)
-      end
-
-      def run
-        # TODO: preview
-        count = 0
-        @soap.each_lead(@last_updated_at) do |lead|
-          values = @columns.map do |column|
-            name = column["name"].to_s
-            (lead[name] || {})[:value]
-          end
-
-          page_builder.add(values)
-
-          count += 1
-          break if preview? && count >= PREVIEW_COUNT
-        end
-
-        page_builder.finish
-
-        commit_report = {}
-        return commit_report
-      end
-
-      def self.logger
-        Embulk.logger
-      end
-
-      def logger
-        self.class.logger
-      end
-
-      private
-
-      def preview?
-        begin
-          org.embulk.spi.Exec.isPreview()
-        rescue java.lang.NullPointerException => e
-          false
-        end
-      end
-    end
-  end
-end

data/lib/embulk/input/marketo_api/soap.rb

@@ -1,112 +0,0 @@
-require "savon"
-
-module Embulk
-  module Input
-    module MarketoApi
-      class Soap
-        attr_reader :endpoint, :wsdl, :user_id, :encryption_key
-
-        def initialize(endpoint, wsdl, user_id, encryption_key)
-          @endpoint = endpoint
-          @wsdl = wsdl
-          @user_id = user_id
-          @encryption_key = encryption_key
-        end
-
-        def lead_metadata
-          # http://developers.marketo.com/documentation/soap/describemobject/
-          response = savon.call(:describe_m_object, message: {object_name: "LeadRecord"})
-          response.body[:success_describe_m_object][:result][:metadata][:field_list][:field]
-        end
-
-        def each_lead(last_updated_at, &block)
-          # http://developers.marketo.com/documentation/soap/getmultipleleads/
-
-          last_updated_at = Time.parse(last_updated_at).iso8601
-          request = {
-            lead_selector: {
-              oldest_updated_at: last_updated_at,
-            },
-            attributes!: {
-              lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
-            },
-            batch_size: 1000,
-          }
-
-          stream_position = fetch_leads(request, &block)
-
-          while stream_position
-            stream_position = fetch_leads(request.merge(stream_position: stream_position), &block)
-          end
-        end
-
-        private
-
-        def fetch_leads(request = {}, &block)
-          response = savon.call(:get_multiple_leads, message: request)
-
-          remaining = response.xpath('//remainingCount').text.to_i
-          Embulk.logger.info "Remaining records: #{remaining}"
-          response.xpath('//leadRecordList/leadRecord').each do |lead|
-            record = {
-              "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
-              "email" => {type: :string, value: lead.xpath('Email').text}
-            }
-            lead.xpath('leadAttributeList/attribute').each do |attr|
-              name = attr.xpath('attrName').text
-              type = attr.xpath('attrType').text
-              value = attr.xpath('attrValue').text
-              record = record.merge(
-                name => {
-                  type: type,
-                  value: value
-                }
-              )
-            end
-
-            block.call(record)
-          end
-
-          if remaining > 0
-            response.xpath('//newStreamPosition').text
-          else
-            nil
-          end
-        end
-
-        def savon
-          headers = {
-            'ns1:AuthenticationHeader' => {
-              "mktowsUserId" => user_id,
-            }.merge(signature)
-          }
-          # NOTE: Do not memoize this to use always fresh signature (avoid 20016 error)
-          # ref. https://jira.talendforge.org/secure/attachmentzip/unzip/167201/49761%5B1%5D/Marketo%20Enterprise%20API%202%200.pdf (41 page)
-          Savon.client(
-            log: true,
-            logger: Embulk.logger,
-            wsdl: wsdl,
-            soap_header: headers,
-            endpoint: endpoint,
-            open_timeout: 90,
-            read_timeout: 300,
-            raise_errors: true,
-            namespace_identifier: :ns1,
-            env_namespace: 'SOAP-ENV'
-          )
-        end
-
-        def signature
-          timestamp = Time.now.to_s
-          encryption_string = timestamp + user_id
-          digest = OpenSSL::Digest.new('sha1')
-          hashed_signature = OpenSSL::HMAC.hexdigest(digest, encryption_key, encryption_string)
-          {
-            'requestTimestamp' => timestamp,
-            'requestSignature' => hashed_signature.to_s
-          }
-        end
-      end
-    end
-  end
-end

data/test/embulk/input/marketo_api/test_soap.rb

@@ -1,123 +0,0 @@
-require "embulk/input/marketo_api/soap"
-require "lead_fixtures"
-
-module Embulk
-  module Input
-    module MarketoApi
-      class SoapTest < Test::Unit::TestCase
-        include LeadFixtures
-
-        class TestSignature < self
-          def setup
-            @signature = soap.__send__(:signature)
-          end
-
-          def test_sigature_keys
-            assert_equal(%w(requestTimestamp requestSignature).sort, @signature.keys.sort)
-          end
-
-          def test_is_hash
-            assert_equal(Hash, @signature.class)
-          end
-        end
-
-        def test_each_lead
-          stub(Embulk).logger { ::Logger.new(IO::NULL) }
-          last_updated_at = "2015-07-06"
-
-          request = {
-            lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
-            attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
-            batch_size: 1000
-          }
-
-          any_instance_of(Savon::Client) do |klass|
-            mock(klass).call(:get_multiple_leads, message: request) do
-              next_stream_leads_response
-            end
-          end
-
-          proc = proc{ "" }
-          leads_count = next_stream_leads_response.xpath('//leadRecord').length
-          mock(proc).call(anything).times(leads_count)
-
-          soap.each_lead(last_updated_at, &proc)
-        end
-
-        class TestLeadMetadata < self
-          def setup
-            @savon = soap.__send__(:savon)
-            stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
-          end
-
-          def test_savon_call
-            mock(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
-              Struct.new(:body).new(body)
-            }
-            soap.lead_metadata
-          end
-
-          def test_return_fields
-            stub(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
-              Struct.new(:body).new(body)
-            }
-            assert_equal(fields, soap.lead_metadata)
-          end
-
-          private
-
-          def body
-            {
-              success_describe_m_object: {
-                result: {
-                  metadata: {
-                    field_list: {
-                      field: fields
-                    }
-                  }
-                }
-              }
-            }
-          end
-
-          def fields
-            [
-              {
-                name: "FieldName",
-                description: nil,
-                display_name: "The Name of Field",
-                source_object: "Lead",
-                data_type: "datetime",
-                size: nil,
-                is_readonly: false,
-                is_update_blocked: false,
-                is_name: nil,
-                is_primary_key: false,
-                is_custom: true,
-                is_dynamic: true,
-                dynamic_field_ref: "leadAttributeList",
-                updated_at: DateTime.parse("2000-01-01 22:22:22")
-              }
-            ]
-          end
-        end
-
-        private
-
-        def soap
-          @soap ||= Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
-        end
-
-        def settings
-          {
-            endpoint: "https://marketo.example.com",
-            wsdl: "https://marketo.example.com/?wsdl",
-            user_id: "user_id",
-            encryption_key: "TOPSECRET",
-          }
-        end
-      end
-    end
-  end
-end
-