embulk-input-marketo 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 297928aea65e91bd2295774e2c041b97a9c84906
+  data.tar.gz: 693b68594689b659b7e027d9cfacdf2c515b0f69
+SHA512:
+  metadata.gz: 4d1d412ee4f8c711a16d26b952eadc2599fb2b04e17ad12d1c9646477965fb6e1436518d9f0e568af28300f4f816008994ea90ea168ef2860defd51e4416ea31
+  data.tar.gz: 41f2a41de9ed400df4bdd2a3fe042afc75c6047483f6c7bcddaff96b380f99a83213306699810cdf1ec97b84ecc57fe2cba1b405b997ac0e8f781ab56fb56a1c

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock

data/.ruby-version

@@ -0,0 +1 @@
+jruby-1.7.20

data/.travis.yml

@@ -0,0 +1,12 @@
+language: ruby
+rvm:
+  - jruby-19mode
+addons:
+  code_climate:
+    repo_token:
+      secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
+notifications:
+  slack:
+    secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
+jdk:
+  - oraclejdk8

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1 - 2015-07-06
+
+The first release!!

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org/'
+
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2015 Everyleaf Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,48 @@
+[![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
+
+[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
+
+[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
+
+# Marketo input plugin for Embulk
+
+embulk-input-marketo is the Embulk input plugin for [Marketo](http://www.marketo.com/).
+This plugin uses Marketo SOAP API.
+
+## Overview
+
+Required Embulk version >= 0.6.13.
+
+* **Plugin type**: input
+* **Resume supported**: no
+* **Cleanup supported**: no
+* **Guess supported**: yes
+
+## Configuration
+
+Below parameters are shown in "Admin" > "Web Services" page in Marketo.
+
+- **endpoint** SOAP endpoint URL for your account (string, required)
+- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
+- **user_id** Your user id (string, reqiured)
+- **encryption_key** Your encryption key (string, reqiured)
+- **last_updated_at** Limit datetime that a lead has been updated (this plugin fetches leads updated after this datetime) (string, required)
+
+## Example
+
+```yaml
+in:
+  type: marketo
+  endpoint: https://soap-end-point.mktoapi.com/
+  wsdl: https://wsdl-url.mktoapi.com/?WSDL
+  user_id: user_ABC123
+  encryption_key: TOPSECRET
+  last_updated_at: "2015-06-30"
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+task default: :test
+
+desc "Run tests"
+task :test do
+  ruby("test/run-test.rb", "--use-color=yes")
+end

data/embulk-input-marketo.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-marketo"
+  spec.version       = "0.0.1"
+  spec.authors       = ["uu59", "yoshihara"]
+  spec.summary       = "Marketo input plugin for Embulk"
+  spec.description   = "Loads records from Marketo."
+  spec.email         = ["k@uu59.org", "h.yoshihara@everyleaf.com"]
+  spec.licenses      = ["Apache2"]
+  spec.homepage      = "https://github.com/treasure-data/embulk-input-marketo"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'savon', ['~> 2.11.1']
+  spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'test-unit'
+  spec.add_development_dependency 'test-unit-rr'
+  spec.add_development_dependency 'codeclimate-test-reporter'
+end

data/lib/embulk/input/marketo.rb

@@ -0,0 +1,138 @@
+require "embulk/input/marketo_api"
+
+module Embulk
+  module Input
+    class Marketo < InputPlugin
+      PREVIEW_COUNT = 15
+
+      Plugin.register_input("marketo", self)
+
+      def self.transaction(config, &control)
+        endpoint_url = config.param(:endpoint, :string)
+
+        task = {
+          endpoint_url: endpoint_url,
+          wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+          user_id: config.param(:user_id, :string),
+          encryption_key: config.param(:encryption_key, :string),
+          last_updated_at: config.param(:last_updated_at, :string),
+          columns: config.param(:columns, :array)
+        }
+
+        columns = []
+
+        task[:columns].each do |column|
+          name = column["name"]
+          type = column["type"].to_sym
+
+          columns << Column.new(nil, name, type, column["format"])
+        end
+
+        resume(task, columns, 1, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        commit_reports = yield(task, columns, count)
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def self.guess(config)
+        client = soap_client(config)
+        metadata = client.lead_metadata
+
+        return {"columns" => generate_columns(metadata)}
+      end
+
+      def self.soap_client(config)
+        @soap ||=
+          begin
+            endpoint_url = config.param(:endpoint, :string),
+            soap_config = {
+              endpoint_url: endpoint_url,
+              wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+              user_id: config.param(:user_id, :string),
+              encryption_key: config.param(:encryption_key, :string),
+            }
+
+            MarketoApi.soap_client(soap_config)
+          end
+      end
+
+      def self.generate_columns(metadata)
+        columns = [
+          {name: "id", type: "long"},
+          {name: "email", type: "string"},
+        ]
+
+        metadata.each do |field|
+          type =
+            case field[:data_type]
+            when "integer"
+              "long"
+            when "dateTime", "date"
+              "timestamp"
+            when "string", "text", "phone", "currency"
+              "string"
+            when "boolean"
+              "boolean"
+            when "float"
+              "double"
+            else
+              "string"
+            end
+
+          columns << {name: field[:name], type: type}
+        end
+
+        columns
+      end
+
+      def init
+        @last_updated_at = task[:last_updated_at]
+        @columns = task[:columns]
+        @soap = MarketoApi.soap_client(task)
+      end
+
+      def run
+        # TODO: preview
+        count = 0
+        @soap.each_lead(@last_updated_at) do |lead|
+          values = @columns.map do |column|
+            name = column["name"].to_s
+            (lead[name] || {})[:value]
+          end
+
+          page_builder.add(values)
+
+          count += 1
+          break if preview? && count >= PREVIEW_COUNT
+        end
+
+        page_builder.finish
+
+        commit_report = {}
+        return commit_report
+      end
+
+      def self.logger
+        Embulk.logger
+      end
+
+      def logger
+        self.class.logger
+      end
+
+      private
+
+      def preview?
+        begin
+          org.embulk.spi.Exec.isPreview()
+        rescue java.lang.NullPointerException => e
+          false
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo_api.rb

@@ -0,0 +1,11 @@
+require "embulk/input/marketo_api/soap"
+
+module Embulk
+  module Input
+    module MarketoApi
+      def self.soap_client(config)
+        MarketoApi::Soap.new(config[:endpoint_url], config[:wsdl_url], config[:user_id], config[:encryption_key])
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo_api/soap.rb

@@ -0,0 +1,112 @@
+require "savon"
+
+module Embulk
+  module Input
+    module MarketoApi
+      class Soap
+        attr_reader :endpoint, :wsdl, :user_id, :encryption_key
+
+        def initialize(endpoint, wsdl, user_id, encryption_key)
+          @endpoint = endpoint
+          @wsdl = wsdl
+          @user_id = user_id
+          @encryption_key = encryption_key
+        end
+
+        def lead_metadata
+          # http://developers.marketo.com/documentation/soap/describemobject/
+          response = savon.call(:describe_m_object, message: {object_name: "LeadRecord"})
+          response.body[:success_describe_m_object][:result][:metadata][:field_list][:field]
+        end
+
+        def each_lead(last_updated_at, &block)
+          # http://developers.marketo.com/documentation/soap/getmultipleleads/
+
+          last_updated_at = Time.parse(last_updated_at).iso8601
+          request = {
+            lead_selector: {
+              oldest_updated_at: last_updated_at,
+            },
+            attributes!: {
+              lead_selector: {"xsi:type" => "ns1:LastUpdateAtSelector"}
+            },
+            batch_size: 1000,
+          }
+
+          stream_position = fetch_leads(request, &block)
+
+          while stream_position
+            stream_position = fetch_leads(request.merge(stream_position: stream_position), &block)
+          end
+        end
+
+        private
+
+        def fetch_leads(request = {}, &block)
+          response = savon.call(:get_multiple_leads, message: request)
+
+          remaining = response.xpath('//remainingCount').text.to_i
+          Embulk.logger.info "Remaining records: #{remaining}"
+          response.xpath('//leadRecordList/leadRecord').each do |lead|
+            record = {
+              "id" => {type: :integer, value: lead.xpath('Id').text.to_i},
+              "email" => {type: :string, value: lead.xpath('Email').text}
+            }
+            lead.xpath('leadAttributeList/attribute').each do |attr|
+              name = attr.xpath('attrName').text
+              type = attr.xpath('attrType').text
+              value = attr.xpath('attrValue').text
+              record = record.merge(
+                name => {
+                  type: type,
+                  value: value
+                }
+              )
+            end
+
+            block.call(record)
+          end
+
+          if remaining > 0
+            response.xpath('//newStreamPosition').text
+          else
+            nil
+          end
+        end
+
+        def savon
+          headers = {
+            'ns1:AuthenticationHeader' => {
+              "mktowsUserId" => user_id,
+            }.merge(signature)
+          }
+          # NOTE: Do not memoize this to use always fresh signature (avoid 20016 error)
+          # ref. https://jira.talendforge.org/secure/attachmentzip/unzip/167201/49761%5B1%5D/Marketo%20Enterprise%20API%202%200.pdf (41 page)
+          Savon.client(
+            log: true,
+            logger: Embulk.logger,
+            wsdl: wsdl,
+            soap_header: headers,
+            endpoint: endpoint,
+            open_timeout: 90,
+            read_timeout: 300,
+            raise_errors: true,
+            namespace_identifier: :ns1,
+            env_namespace: 'SOAP-ENV'
+          )
+        end
+
+        def signature
+          timestamp = Time.now.to_s
+          encryption_string = timestamp + user_id
+          digest = OpenSSL::Digest.new('sha1')
+          hashed_signature = OpenSSL::HMAC.hexdigest(digest, encryption_key, encryption_string)
+          {
+            'requestTimestamp' => timestamp,
+            'requestSignature' => hashed_signature.to_s
+          }
+        end
+      end
+    end
+  end
+end

data/test/embulk/input/marketo_api/test_soap.rb

@@ -0,0 +1,123 @@
+require "embulk/input/marketo_api/soap"
+require "lead_fixtures"
+
+module Embulk
+  module Input
+    module MarketoApi
+      class SoapTest < Test::Unit::TestCase
+        include LeadFixtures
+
+        class TestSignature < self
+          def setup
+            @signature = soap.__send__(:signature)
+          end
+
+          def test_sigature_keys
+            assert_equal(%w(requestTimestamp requestSignature).sort, @signature.keys.sort)
+          end
+
+          def test_is_hash
+            assert_equal(Hash, @signature.class)
+          end
+        end
+
+        def test_each_lead
+          stub(Embulk).logger { ::Logger.new(IO::NULL) }
+          last_updated_at = "2015-07-06"
+
+          request = {
+            lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
+            attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
+            batch_size: 1000
+          }
+
+          any_instance_of(Savon::Client) do |klass|
+            mock(klass).call(:get_multiple_leads, message: request) do
+              next_stream_leads_response
+            end
+          end
+
+          proc = proc{ "" }
+          leads_count = next_stream_leads_response.xpath('//leadRecord').length
+          mock(proc).call(anything).times(leads_count)
+
+          soap.each_lead(last_updated_at, &proc)
+        end
+
+        class TestLeadMetadata < self
+          def setup
+            @savon = soap.__send__(:savon)
+            stub(soap).savon { @savon } # Pin savon instance for each call soap.savon for mocking/stubbing
+          end
+
+          def test_savon_call
+            mock(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
+              Struct.new(:body).new(body)
+            }
+            soap.lead_metadata
+          end
+
+          def test_return_fields
+            stub(@savon).call(:describe_m_object, message: {object_name: "LeadRecord"}) {
+              Struct.new(:body).new(body)
+            }
+            assert_equal(fields, soap.lead_metadata)
+          end
+
+          private
+
+          def body
+            {
+              success_describe_m_object: {
+                result: {
+                  metadata: {
+                    field_list: {
+                      field: fields
+                    }
+                  }
+                }
+              }
+            }
+          end
+
+          def fields
+            [
+              {
+                name: "FieldName",
+                description: nil,
+                display_name: "The Name of Field",
+                source_object: "Lead",
+                data_type: "datetime",
+                size: nil,
+                is_readonly: false,
+                is_update_blocked: false,
+                is_name: nil,
+                is_primary_key: false,
+                is_custom: true,
+                is_dynamic: true,
+                dynamic_field_ref: "leadAttributeList",
+                updated_at: DateTime.parse("2000-01-01 22:22:22")
+              }
+            ]
+          end
+        end
+
+        private
+
+        def soap
+          @soap ||= Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+        end
+
+        def settings
+          {
+            endpoint: "https://marketo.example.com",
+            wsdl: "https://marketo.example.com/?wsdl",
+            user_id: "user_id",
+            encryption_key: "TOPSECRET",
+          }
+        end
+      end
+    end
+  end
+end
+

data/test/embulk/input/test_marketo.rb

@@ -0,0 +1,176 @@
+require "prepare_embulk"
+require "lead_fixtures"
+require "embulk/input/marketo"
+
+module Embulk
+  module Input
+    class MarketoTest < Test::Unit::TestCase
+      include LeadFixtures
+
+      def setup_soap
+        @soap = MarketoApi::Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+        stub(MarketoApi).soap_client(task) { @soap }
+      end
+
+      def setup_plugin
+        @page_builder = Object.new
+        @plugin = Marketo.new(task, nil, nil, @page_builder)
+        stub(Embulk).logger { ::Logger.new(File::NULL) }
+      end
+
+      def test_transaction
+        control = proc {} # dummy
+        columns = task[:columns].map do |col|
+          Column.new(nil, col["name"], col["type"].to_sym)
+        end
+
+        mock(Marketo).resume(task, columns, 1, &control)
+        Marketo.transaction(config, &control)
+      end
+
+      class RunTest < self
+        def setup
+          setup_soap
+          setup_plugin
+        end
+
+        def test_run_through
+          stub(@plugin).preview? { false }
+
+          any_instance_of(Savon::Client) do |klass|
+            mock(klass).call(:get_multiple_leads, message: request) do
+              leads_response
+            end
+
+            mock(klass).call(:get_multiple_leads, message: request.merge(stream_position: stream_position)) do
+              next_stream_leads_response
+            end
+          end
+
+          mock(@page_builder).add(["manyo"])
+          mock(@page_builder).add(["everyleaf"])
+          mock(@page_builder).add(["ten-thousand-leaf"])
+          mock(@page_builder).finish
+
+          @plugin.run
+        end
+
+        def test_preview_through
+          stub(@plugin).preview? { true }
+
+          any_instance_of(Savon::Client) do |klass|
+            mock(klass).call(:get_multiple_leads, message: request) do
+              preview_leads_response
+            end
+          end
+
+          Marketo::PREVIEW_COUNT.times do |count|
+            mock(@page_builder).add(["manyo#{count}"])
+          end
+          mock(@page_builder).finish
+
+          @plugin.run
+        end
+
+        private
+
+        def request
+          {
+            lead_selector: {oldest_updated_at: Time.parse(last_updated_at).iso8601},
+            attributes!: {lead_selector: {"xsi:type"=>"ns1:LastUpdateAtSelector"}},
+            batch_size: 1000
+          }
+        end
+      end
+
+      class GuessTest < self
+        setup :setup_soap
+
+        def setup_soap
+          @soap = MarketoApi::Soap.new(settings[:endpoint], settings[:wsdl], settings[:user_id], settings[:encryption_key])
+
+          stub(Marketo).soap_client(config) { @soap }
+        end
+
+        def test_include_metadata
+          stub(@soap).lead_metadata { metadata }
+
+          assert_equal(
+            {"columns" => expected_guessed_columns},
+            Marketo.guess(config)
+          )
+        end
+      end
+
+      def test_generate_columns
+        assert_equal(expected_guessed_columns, Marketo.generate_columns(metadata))
+      end
+
+      private
+
+      def config
+        DataSource[settings.to_a]
+      end
+
+      def settings
+        {
+          endpoint: "https://marketo.example.com",
+          wsdl: "https://marketo.example.com/?wsdl",
+          user_id: "user_id",
+          encryption_key: "TOPSECRET",
+          last_updated_at: last_updated_at,
+          columns: [
+            {"name" => "Name", "type" => "string"},
+          ]
+        }
+      end
+
+      def last_updated_at
+        "2015-07-01 00:00:00+00:00"
+      end
+
+      def task
+        {
+          endpoint_url: "https://marketo.example.com",
+          wsdl_url: "https://marketo.example.com/?wsdl",
+          user_id: "user_id",
+          encryption_key: "TOPSECRET",
+          last_updated_at: last_updated_at,
+          columns: [
+            {"name" => "Name", "type" => "string"},
+          ]
+        }
+      end
+
+      def metadata
+        [
+          {
+            name: "FieldName",
+            description: nil,
+            display_name: "The Name of Field",
+            source_object: "Lead",
+            data_type: "datetime",
+            size: nil,
+            is_readonly: false,
+            is_update_blocked: false,
+            is_name: nil,
+            is_primary_key: false,
+            is_custom: true,
+            is_dynamic: true,
+            dynamic_field_ref: "leadAttributeList",
+            updated_at: DateTime.parse("2000-01-01 22:22:22")
+          }
+        ]
+      end
+
+      def expected_guessed_columns
+        [
+          {name: "id", type: "long"},
+          {name: "email", type: "string"},
+          {name: "FieldName", type: "string"},
+        ]
+      end
+    end
+  end
+end

data/test/lead_fixtures.rb

@@ -0,0 +1,119 @@
+module LeadFixtures
+  private
+
+  def leads_response
+    Nokogiri::XML(raw_response)
+  end
+
+  def next_stream_leads_response
+    Nokogiri::XML(raw_next_stream_response)
+  end
+
+  def preview_leads_response
+    Nokogiri::XML(raw_preview_response)
+  end
+
+  def leads(body)
+    <<XML
+<?xml version="1.0" encoding="UTF-8"?>
+<SOAP-ENV:Envelope xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:ns1="http://www.marketo.com/mktows/">
+  <SOAP-ENV:Body>
+    <ns1:successGetMultipleLeads>
+      <result>
+        #{body}
+      </result>
+    </ns1:successGetMultipleLeads>
+  </SOAP-ENV:Body>
+</SOAP-ENV:Envelope>
+XML
+  end
+
+  def raw_response
+    leads(<<XML)
+<remainingCount>1</remainingCount>
+<newStreamPosition>#{stream_position}</newStreamPosition>
+<leadRecordList>
+  <leadRecord>
+    <Id>65835</Id>
+    <Email>manyo@marbketo.com</Email>
+    <ForeignSysPersonId xsi:nil="true" />
+    <ForeignSysType xsi:nil="true" />
+    <leadAttributeList>
+      <attribute>
+        <attrName>Name</attrName>
+        <attrType>string</attrType>
+        <attrValue>manyo</attrValue>
+      </attribute>
+    </leadAttributeList>
+  </leadRecord>
+  <leadRecord>
+    <Id>67508</Id>
+    <Email>everyleaf@marketo.com</Email>
+    <ForeignSysPersonId xsi:nil="true" />
+    <ForeignSysType xsi:nil="true" />
+    <leadAttributeList>
+      <attribute>
+        <attrName>Name</attrName>
+        <attrType>string</attrType>
+        <attrValue>everyleaf</attrValue>
+      </attribute>
+    </leadAttributeList>
+  </leadRecord>
+</leadRecordList>
+XML
+  end
+
+  def stream_position
+    "next_steam_position"
+  end
+
+  def raw_next_stream_response
+    leads(<<XML)
+<returnCount>2</returnCount>
+<remainingCount>0</remainingCount>
+<newStreamPosition />
+<leadRecordList>
+  <leadRecord>
+    <Id>65835</Id>
+    <Email>ten-thousand-leaf@marketo.com</Email>
+    <ForeignSysPersonId xsi:nil="true" />
+    <ForeignSysType xsi:nil="true" />
+    <leadAttributeList>
+      <attribute>
+        <attrName>Name</attrName>
+        <attrType>string</attrType>
+        <attrValue>ten-thousand-leaf</attrValue>
+      </attribute>
+    </leadAttributeList>
+  </leadRecord>
+</leadRecordList>
+XML
+  end
+
+  def raw_preview_response
+    body = ""
+    15.times do |i|
+      body << <<XML
+<returnCount>2</returnCount>
+<remainingCount>0</remainingCount>
+<newStreamPosition />
+<leadRecordList>
+  <leadRecord>
+    <Id>#{65835 + i}</Id>
+    <Email>manyo#{i}@marketo.com</Email>
+    <ForeignSysPersonId xsi:nil="true" />
+    <ForeignSysType xsi:nil="true" />
+    <leadAttributeList>
+      <attribute>
+        <attrName>Name</attrName>
+        <attrType>string</attrType>
+        <attrValue>manyo#{i}</attrValue>
+      </attribute>
+    </leadAttributeList>
+  </leadRecord>
+</leadRecordList>
+XML
+    end
+    leads(body)
+  end
+end

data/test/prepare_embulk.rb

@@ -0,0 +1,10 @@
+require "embulk/command/embulk_run"
+
+classpath_dir = Embulk.home("classpath")
+jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
+jars.each do |jar|
+  require File.join(classpath_dir, jar)
+end
+require "embulk/java/bootstrap"
+
+require "embulk"

data/test/run-test.rb

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+require "test-unit"
+require "test/unit/rr"
+require "codeclimate-test-reporter"
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+ENV["TEST_UNIT_MAX_DIFF_TARGET_STRING_SIZE"] ||= "5000"
+
+CodeClimate::TestReporter.start
+
+exit Test::Unit::AutoRunner.run(true, test_dir)

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-marketo
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- uu59
+- yoshihara
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.1
+  name: savon
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.13
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.6.13
+    - - <
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: pry
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: codeclimate-test-reporter
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Loads records from Marketo.
+email:
+- k@uu59.org
+- h.yoshihara@everyleaf.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .ruby-version
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- embulk-input-marketo.gemspec
+- lib/embulk/input/marketo.rb
+- lib/embulk/input/marketo_api.rb
+- lib/embulk/input/marketo_api/soap.rb
+- test/embulk/input/marketo_api/test_soap.rb
+- test/embulk/input/test_marketo.rb
+- test/lead_fixtures.rb
+- test/prepare_embulk.rb
+- test/run-test.rb
+homepage: https://github.com/treasure-data/embulk-input-marketo
+licenses:
+- Apache2
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.6
+signing_key:
+specification_version: 4
+summary: Marketo input plugin for Embulk
+test_files:
+- test/embulk/input/marketo_api/test_soap.rb
+- test/embulk/input/test_marketo.rb
+- test/lead_fixtures.rb
+- test/prepare_embulk.rb
+- test/run-test.rb