embulk-input-marketo 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd2aca4cc3202db0cc6da4d33a1d564ad601ec15
-  data.tar.gz: c0b2a2bec618d140967d54f054c6efe0c40e25ee
+  metadata.gz: 803b16c2edca4432ffcb1f32e9914d944b3747dd
+  data.tar.gz: bb28951330b3b3ac1a8d3bf7a3e81541915268e2
 SHA512:
-  metadata.gz: 0b2aeaf51b0d648b8b0ea0c14f43caa6df2931bc732ea47d65d5829e99cb3378ccd12a634ffb579b643a30d629554559a449fbc30b994fb594e29d8c9cd735fe
-  data.tar.gz: efb4857536cd3050527a1eb9f13a040cdc91268fe7569bc3a3006fd93abf3855876078b06ba555902eb704b1abc1aa54e26069ea963310c26f00c31ce14a5a1b
+  metadata.gz: 53f1320084a90a824af96b5205ae05f3d8596e79169abc7b15282c67d359ca8c26b8c1e433dcab5ff7b678ba1b239347b7ebd9eda547eb79bbde6ca931227aed
+  data.tar.gz: e6a1e61e3a7d4d0b875b0e704298d7e96d0e1ab395125e461ef704ef069267ac407b515538e2576dbf7ada5420b2f8b473a5d04337f5600648dd5e3649d2c7f6

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.3.0 - 2015-09-30
+
+This version breaks backword compatibility of marketo/activity_log. Please check README.md to modify your config.
+
+* [enhancement] Also activity_log uses from_datetime/to_datetime same as lead [#39](https://github.com/treasure-data/embulk-input-marketo/pull/39)
+
 ## 0.2.5 - 2015-09-28
 
 * [fixed] lead: Fix the bug when `from_datetime` and `to_datetime` are same [#37](https://github.com/treasure-data/embulk-input-marketo/pull/37)

data/README.md

@@ -1,8 +1,7 @@
 [![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
-
 [![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
-
 [![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
+[![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
 
 # Marketo input plugin for Embulk
 
@@ -51,7 +50,8 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 - **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
 - **user_id** Your user id (string, reqiured)
 - **encryption_key** Your encryption key (string, reqiured)
-- **last_updated_at** Limit datetime that a lead has been updated (this plugin fetches leads updated after this datetime) (string, required)
+- **from_datetime** Fetch activity_logs since this time (string, required)
+- **to_datetime** Fetch activity_logs until this time (string, default: Time.now)
 
 ### Selecting plugin type
 

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.2.5"
+  spec.version       = "0.3.0"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."
@@ -13,6 +13,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'savon', ['~> 2.11.1']
+  spec.add_dependency 'httpclient'
   spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']

data/lib/embulk/input/marketo/activity_log.rb

@@ -10,11 +10,38 @@ module Embulk
           :activity_log
         end
 
+        def self.transaction(config, &control)
+          endpoint_url = config.param(:endpoint, :string)
+
+          range = format_range(config)
+
+          task = {
+            endpoint_url: endpoint_url,
+            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+            user_id: config.param(:user_id, :string),
+            encryption_key: config.param(:encryption_key, :string),
+            from_datetime: range[:from],
+            to_datetime: range[:to],
+            columns: config.param(:columns, :array)
+          }
+
+          columns = []
+
+          task[:columns].each do |column|
+            name = column["name"]
+            type = column["type"].to_sym
+
+            columns << Column.new(nil, name, type, column["format"])
+          end
+
+          resume(task, columns, 1, &control)
+        end
+
         def self.guess(config)
           client = soap_client(config)
-          last_updated_at = config.param(:last_updated_at, :string)
+          range = format_range(config)
 
-          schema = client.metadata(last_updated_at, batch_size: PREVIEW_COUNT)
+          schema = client.metadata(range[:from], batch_size: PREVIEW_COUNT)
           columns = schema.map do |c|
             column = {name: c.name, type: c.type}
             column[:format] = c.format if c.format
@@ -33,7 +60,7 @@ module Embulk
 
           count = 0
 
-          last_updated_at = @soap.each(@last_updated_at, batch_size: batch_size) do |activity_log|
+          last_updated_at = @soap.each(task[:from_datetime], batch_size: batch_size, to: task[:to_datetime]) do |activity_log|
             values = @columns.map do |column|
               name = column["name"].to_s
               value = activity_log[name]
@@ -60,7 +87,7 @@ module Embulk
 
           commit_report = {}
           if !preview? && last_updated_at
-            commit_report = {last_updated_at: last_updated_at}
+            commit_report = {from_datetime: last_updated_at}
           end
 
           return commit_report

data/lib/embulk/input/marketo/base.rb

@@ -13,30 +13,6 @@ module Embulk
           raise NotImplementedError
         end
 
-        def self.transaction(config, &control)
-          endpoint_url = config.param(:endpoint, :string)
-
-          task = {
-            endpoint_url: endpoint_url,
-            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-            user_id: config.param(:user_id, :string),
-            encryption_key: config.param(:encryption_key, :string),
-            last_updated_at: config.param(:last_updated_at, :string),
-            columns: config.param(:columns, :array)
-          }
-
-          columns = []
-
-          task[:columns].each do |column|
-            name = column["name"]
-            type = column["type"].to_sym
-
-            columns << Column.new(nil, name, type, column["format"])
-          end
-
-          resume(task, columns, 1, &control)
-        end
-
         def self.resume(task, columns, count, &control)
           commit_reports = yield(task, columns, count)
 
@@ -80,6 +56,33 @@ module Embulk
         def target
           self.class.target
         end
+
+        def self.format_range(config)
+          if config.param(:last_updated_at, :string, default: nil)
+            Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
+          end
+
+          from_datetime = config.param(:from_datetime, :string)
+          to_datetime = config.param(:to_datetime, :string, default: Time.now.to_s)
+
+          # check from/to format to parse
+          begin
+            Time.parse(from_datetime)
+            Time.parse(to_datetime)
+          rescue => e
+            # possibly Time.parse fail
+            raise ConfigError, e.message
+          end
+
+          if Time.parse(from_datetime) > Time.parse(to_datetime)
+            raise ConfigError, "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
+          end
+
+          {
+            from: from_datetime,
+            to: to_datetime,
+          }
+        end
       end
     end
   end

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -18,8 +18,7 @@ module Embulk
           def each(last_updated_at, options={}, &block)
             response = fetch_by_last_updated_at(last_updated_at, options, &block)
             while response[:remaining_count] > 0 do
-              offset = response[:offset]
-              response = fetch_by_offset(offset, options, &block)
+              response = fetch_by_last_updated_at(last_updated_at, options.merge(offset: response[:offset]), &block)
             end
 
             response[:last_updated_at]
@@ -31,28 +30,26 @@ module Embulk
             last_updated_at = last_updated_at.to_s
             last_updated_at = Time.parse(last_updated_at).iso8601
 
-            request = {
-              start_position: {
-                oldest_created_at: last_updated_at,
-              },
-            }
-
-            fetch(request, options, &block)
-          end
+            to =
+              if options[:to]
+                Time.parse(options[:to]).iso8601
+              else
+                Time.now.iso8601
+              end
 
-          def fetch_by_offset(offset, options={}, &block)
             request = {
               start_position: {
-                offset: offset,
+                oldest_created_at: last_updated_at,
+                latest_created_at: to,
               },
+              batch_size: options[:batch_size] || 100
             }
+            request[:start_position][:offset] = options[:offset] if options[:offset]
 
             fetch(request, options, &block)
           end
 
           def fetch(request, options={}, &block)
-            request[:batch_size] = options[:batch_size] || 100
-
             response = savon_call(:get_lead_changes, message: request)
             remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
             Embulk.logger.info "Remaining records: #{remaining}"

data/lib/embulk/input/marketo_api/soap/base.rb

@@ -1,4 +1,5 @@
 require "savon"
+require "httpclient" # net/http can't verify cert correctly
 
 module Embulk
   module Input

data/test/embulk/input/marketo/test_activity_log.rb

@@ -8,6 +8,64 @@ module Embulk
       class ActivityLogTest < Test::Unit::TestCase
         include ActivityLogFixtures
 
+        class TransactionTest < self
+          def test_generate_task
+            control = proc {} # dummy
+            columns = task[:columns].map do |col|
+              Column.new(nil, col["name"], col["type"].to_sym, col["format"])
+            end
+
+            mock(ActivityLog).resume(task, columns, 1, &control)
+            ActivityLog.transaction(config, &control)
+          end
+
+          private
+
+          def settings
+            {
+              endpoint: "https://marketo.example.com",
+              wsdl: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+              from_datetime: from_datetime,
+              to_datetime: to_datetime,
+              columns: [
+                {"name" => :id, "type" => :long},
+                {"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
+                {"name" => :activity_type, "type" => :string},
+                {"name" => :mktg_asset_name, "type" => :string},
+                {"name" => :mkt_person_id, "type" => :long},
+                {"name" => "Attribute Name", "type" => :string},
+                {"name" => "Old Value", "type" => :string},
+              ]
+            }
+          end
+
+          def config
+            DataSource[settings.to_a]
+          end
+
+          def task
+            {
+              endpoint_url: "https://marketo.example.com",
+              wsdl_url: "https://marketo.example.com/?wsdl",
+              user_id: "user_id",
+              encryption_key: "TOPSECRET",
+              from_datetime: from_datetime,
+              to_datetime: to_datetime,
+              columns: [
+                {"name" => :id, "type" => :long},
+                {"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
+                {"name" => :activity_type, "type" => :string},
+                {"name" => :mktg_asset_name, "type" => :string},
+                {"name" => :mkt_person_id, "type" => :long},
+                {"name" => "Attribute Name", "type" => :string},
+                {"name" => "Old Value", "type" => :string},
+              ]
+            }
+          end
+        end
+
         def test_target
           assert_equal(:activity_log, ActivityLog.target)
         end
@@ -22,7 +80,7 @@ module Embulk
           end
 
           def test_include_metadata
-            stub(@soap).metadata(last_updated_at, batch_size: ActivityLog::PREVIEW_COUNT) { Guess::SchemaGuess.from_hash_records(records) }
+            stub(@soap).metadata(from_datetime, batch_size: ActivityLog::PREVIEW_COUNT) { Guess::SchemaGuess.from_hash_records(records) }
 
             assert_equal(
               {"columns" => expected_guessed_columns},
@@ -166,7 +224,8 @@ module Embulk
           def request
             {
               start_position: {
-                oldest_created_at: Time.parse(last_updated_at).iso8601,
+                oldest_created_at: Time.parse(from_datetime).iso8601,
+                latest_created_at: Time.parse(to_datetime).iso8601,
               },
               batch_size: 100
             }
@@ -175,6 +234,8 @@ module Embulk
           def offset_request
             {
               start_position: {
+                oldest_created_at: Time.parse(from_datetime).iso8601,
+                latest_created_at: Time.parse(to_datetime).iso8601,
                 offset: "offset"
               },
               batch_size: 100
@@ -185,7 +246,7 @@ module Embulk
           def preview_request
             {
               start_position: {
-                oldest_created_at: Time.parse(last_updated_at).iso8601,
+                oldest_created_at: Time.parse(from_datetime).iso8601,
               },
               batch_size: ActivityLog::PREVIEW_COUNT
             }
@@ -199,7 +260,8 @@ module Embulk
             wsdl: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
+            to_datetime: to_datetime,
           }
         end
 
@@ -213,7 +275,8 @@ module Embulk
             wsdl_url: "https://marketo.example.com/?wsdl",
             user_id: "user_id",
             encryption_key: "TOPSECRET",
-            last_updated_at: last_updated_at,
+            from_datetime: from_datetime,
+            to_datetime: to_datetime,
             columns: [
               {"name" => :id, "type" => :long},
               {"name" => :activity_date_time, "type" => :timestamp, "format" => "%Y-%m-%dT%H:%M:%S%z"},
@@ -226,9 +289,13 @@ module Embulk
           }
         end
 
-        def last_updated_at
+        def from_datetime
           "2015-07-01 00:00:00+00:00"
         end
+
+        def to_datetime
+          "2015-11-01 00:00:00+00:00"
+        end
       end
     end
   end

data/test/embulk/input/marketo/test_base.rb

@@ -11,16 +11,6 @@ module Embulk
           end
         end
 
-        def test_transaction
-          control = proc {} # dummy
-          columns = task[:columns].map do |col|
-            Column.new(nil, col["name"], col["type"].to_sym)
-          end
-
-          mock(Base).resume(task, columns, 1, &control)
-          Base.transaction(config, &control)
-        end
-
         def test_resume
           next_config_diff = {last_updated_at: last_updated_at}
           control = proc { [next_config_diff] } # In actual, embulk prepares control block returning Array.

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-28 00:00:00.000000000 Z
+date: 2015-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -25,6 +25,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.11.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: httpclient
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: