embulk-input-marketo 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 297928aea65e91bd2295774e2c041b97a9c84906
-  data.tar.gz: 693b68594689b659b7e027d9cfacdf2c515b0f69
+  metadata.gz: 7de3f0f124846b84a0df2b1d4fd7824c1453b3ff
+  data.tar.gz: 7796e77550eb46c7f4208f373dc9c84b5b8b7e0d
 SHA512:
-  metadata.gz: 4d1d412ee4f8c711a16d26b952eadc2599fb2b04e17ad12d1c9646477965fb6e1436518d9f0e568af28300f4f816008994ea90ea168ef2860defd51e4416ea31
-  data.tar.gz: 41f2a41de9ed400df4bdd2a3fe042afc75c6047483f6c7bcddaff96b380f99a83213306699810cdf1ec97b84ecc57fe2cba1b405b997ac0e8f781ab56fb56a1c
+  metadata.gz: e765f8a460352604ee6b77f49e03f39d25db1a1b4a77d288c55f7e946bd9e88e8bcccd9c680fb2f281eb08e148024af47c86cc8cf3e14f6c1e26ce2e1e7c6c4b
+  data.tar.gz: 56c092c2965c8c36f2866953e2a74e123d22573cb48ab23ffd7dcbc793b6ab167f22f29c0cfc83d81a6b36a9d1256b2d182b950a0f8cee93d076a1591528bb44

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.1.0 - 2015-07-15
+
+We implemented activity_log plugin for marketo, so config generated from 0.0.1 should be modified. Please check README.md to do it.
+
+* [enhancement] Implement activity_log plugin [#13](https://github.com/treasure-data/embulk-input-marketo/pull/13) [#14](https://github.com/treasure-data/embulk-input-marketo/pull/14) [#15](https://github.com/treasure-data/embulk-input-marketo/pull/15)
+
 ## 0.0.1 - 2015-07-06
 
 The first release!!

data/README.md

@@ -6,7 +6,11 @@
 
 # Marketo input plugin for Embulk
 
-embulk-input-marketo is the Embulk input plugin for [Marketo](http://www.marketo.com/).
+embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
+
+- Lead
+- Activity log
+
 This plugin uses Marketo SOAP API.
 
 ## Overview
@@ -18,8 +22,16 @@ Required Embulk version >= 0.6.13.
 * **Cleanup supported**: no
 * **Guess supported**: yes
 
+## Install
+
+```
+$ embulk gem install embulk-input-marketo
+```
+
 ## Configuration
 
+### API
+
 Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 
 - **endpoint** SOAP endpoint URL for your account (string, required)
@@ -28,21 +40,28 @@ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
 - **encryption_key** Your encryption key (string, reqiured)
 - **last_updated_at** Limit datetime that a lead has been updated (this plugin fetches leads updated after this datetime) (string, required)
 
+### Selecting plugin type
+
+You should specify `type: marketo/lead` or `type: marketo/activity_log` on your demand.
+
+
 ## Example
 
+For lead, you have `partial-config.yml` like below:
+
 ```yaml
 in:
-  type: marketo
+  type: marketo/lead
   endpoint: https://soap-end-point.mktoapi.com/
   wsdl: https://wsdl-url.mktoapi.com/?WSDL
   user_id: user_ABC123
   encryption_key: TOPSECRET
   last_updated_at: "2015-06-30"
+out:
+  type: stdout
 ```
 
+You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
 
-## Build
+Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
 
-```
-$ rake
-```

data/Rakefile

@@ -1,8 +1,60 @@
 require "bundler/gem_tasks"
+require "json"
 
 task default: :test
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb", "--use-color=yes")
+  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
+end
+
+namespace :release do
+  desc "Add header of now version release to ChangeLog and bump up version"
+  task :prepare do
+    root_dir = Pathname.new(File.expand_path("../", __FILE__))
+    changelog_file = root_dir.join("CHANGELOG.md")
+    gemspec_file = root_dir.join("embulk-input-marketo.gemspec")
+
+    system("git fetch origin")
+
+    # detect merged PR
+    old_version = gemspec_file.read[/spec\.version += *"([0-9]+\.[0-9]+\.[0-9]+)"/, 1]
+    pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
+
+    if !$?.success? || pr_numbers.empty?
+      puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
+      exit(false)
+    end
+
+    # Generate new version
+    major, minor, patch = old_version.split(".").map(&:to_i)
+    new_version = "#{major}.#{minor}.#{patch + 1}"
+
+    # Update ChangeLog
+    pr_descriptions = pr_numbers.map do |number|
+      body = open("https://api.github.com/repos/treasure-data/embulk-input-marketo/issues/#{number.gsub("#", "")}").read
+      payload = JSON.parse(body)
+      "* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-marketo/pull/#{number.gsub('#', '')})"
+    end.join("\n")
+
+    new_changelog = <<-HEADER
+## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
+#{pr_descriptions}
+
+#{changelog_file.read.chomp}
+HEADER
+
+    File.open(changelog_file, "w") {|f| f.write(new_changelog) }
+
+    # Update version.rb
+    old_content = gemspec_file.read
+    File.open(gemspec_file, "w") do |f|
+      f.write old_content.gsub(/(spec\.version += *)".*?"/, %Q!\\1"#{new_version}"!)
+    end
+
+    # Update Gemfile.lock
+    system("bundle install")
+
+    puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
+  end
 end

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.0.1"
+  spec.version       = "0.1.0"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."

data/lib/embulk/input/marketo/activity_log.rb

@@ -0,0 +1,55 @@
+require "embulk/input/marketo/base"
+
+module Embulk
+  module Input
+    module Marketo
+      class ActivityLog < Base
+        Plugin.register_input("marketo/activity_log", self)
+
+        def self.target
+          :activity_log
+        end
+
+        def self.guess(config)
+          client = soap_client(config)
+          last_updated_at = config.param(:last_updated_at, :string)
+
+          schema = client.metadata(last_updated_at, batch_size: PREVIEW_COUNT)
+          columns = schema.map do |c|
+            column = {name: c.name, type: c.type}
+            column[:format] = c.format if c.format
+            column
+          end
+
+          return {"columns" => columns}
+        end
+
+        def run
+          if preview?
+            batch_size = PREVIEW_COUNT
+          else
+            batch_size = 100
+          end
+
+          count = 0
+
+          @soap.each(@last_updated_at, batch_size: batch_size) do |activity_log|
+            values = @columns.map do |column|
+              name = column["name"].to_s
+              activity_log[name]
+            end
+
+            page_builder.add(values)
+            count += 1
+            break if preview? && count >= PREVIEW_COUNT
+          end
+
+          page_builder.finish
+
+          commit_report = {}
+          return commit_report
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo/base.rb

@@ -0,0 +1,89 @@
+require "embulk/input/marketo_api"
+
+module Embulk
+  module Input
+    module Marketo
+      class Base < InputPlugin
+        PREVIEW_COUNT = 15
+
+        def self.target
+          raise NotImplementedError
+        end
+
+        def self.transaction(config, &control)
+          endpoint_url = config.param(:endpoint, :string)
+
+          task = {
+            endpoint_url: endpoint_url,
+            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+            user_id: config.param(:user_id, :string),
+            encryption_key: config.param(:encryption_key, :string),
+            last_updated_at: config.param(:last_updated_at, :string),
+            columns: config.param(:columns, :array)
+          }
+
+          columns = []
+
+          task[:columns].each do |column|
+            name = column["name"]
+            type = column["type"].to_sym
+
+            columns << Column.new(nil, name, type, column["format"])
+          end
+
+          resume(task, columns, 1, &control)
+        end
+
+        def self.resume(task, columns, count, &control)
+          commit_reports = yield(task, columns, count)
+
+          next_config_diff = {}
+          return next_config_diff
+        end
+
+        def self.soap_client(config)
+          @soap ||=
+            begin
+              endpoint_url = config.param(:endpoint, :string),
+              soap_config = {
+                endpoint_url: endpoint_url,
+                wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
+                user_id: config.param(:user_id, :string),
+                encryption_key: config.param(:encryption_key, :string),
+              }
+
+              MarketoApi.soap_client(soap_config, target)
+            end
+        end
+
+        def init
+          @last_updated_at = task[:last_updated_at]
+          @columns = task[:columns]
+          @soap = MarketoApi.soap_client(task, target)
+        end
+
+        def self.logger
+          Embulk.logger
+        end
+
+        def logger
+          self.class.logger
+        end
+
+        private
+
+        def preview?
+          begin
+            org.embulk.spi.Exec.isPreview()
+          rescue java.lang.NullPointerException => e
+            false
+          end
+        end
+
+        def target
+          self.class.target
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo/lead.rb

@@ -0,0 +1,73 @@
+require "embulk/input/marketo/base"
+
+module Embulk
+  module Input
+    module Marketo
+      class Lead < Base
+        PREVIEW_COUNT = 15
+
+        Plugin.register_input("marketo/lead", self)
+
+        def self.target
+          :lead
+        end
+
+        def self.guess(config)
+          client = soap_client(config)
+          metadata = client.metadata
+
+          return {"columns" => generate_columns(metadata)}
+        end
+
+        def self.generate_columns(metadata)
+          columns = [
+            {name: "id", type: "long"},
+            {name: "email", type: "string"},
+          ]
+
+          metadata.each do |field|
+            type =
+              case field[:data_type]
+              when "integer"
+                "long"
+              when "dateTime", "date"
+                "timestamp"
+              when "string", "text", "phone", "currency"
+                "string"
+              when "boolean"
+                "boolean"
+              when "float"
+                "double"
+              else
+                "string"
+              end
+
+            columns << {name: field[:name], type: type}
+          end
+
+          columns
+        end
+
+        def run
+          count = 0
+          @soap.each(@last_updated_at) do |lead|
+            values = @columns.map do |column|
+              name = column["name"].to_s
+              (lead[name] || {})[:value]
+            end
+
+            page_builder.add(values)
+
+            count += 1
+            break if preview? && count >= PREVIEW_COUNT
+          end
+
+          page_builder.finish
+
+          commit_report = {}
+          return commit_report
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input/marketo_api.rb

@@ -1,10 +1,21 @@
-require "embulk/input/marketo_api/soap"
+require "embulk/input/marketo_api/soap/base"
+require "embulk/input/marketo_api/soap/lead"
+require "embulk/input/marketo_api/soap/activity_log"
 
 module Embulk
   module Input
     module MarketoApi
-      def self.soap_client(config)
-        MarketoApi::Soap.new(config[:endpoint_url], config[:wsdl_url], config[:user_id], config[:encryption_key])
+      def self.soap_client(config, target)
+        arguments = [config[:endpoint_url], config[:wsdl_url], config[:user_id], config[:encryption_key]]
+
+        case target
+        when :activity_log
+          MarketoApi::Soap::ActivityLog.new(*arguments)
+        when :lead
+          MarketoApi::Soap::Lead.new(*arguments)
+        else
+          raise "unknown target: #{target}"
+        end
       end
     end
   end

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -0,0 +1,90 @@
+require "embulk/input/marketo_api/soap/base"
+
+module Embulk
+  module Input
+    module MarketoApi
+      module Soap
+        class ActivityLog < Base
+          def metadata(last_updated_at, options={})
+            activity_logs = []
+
+            fetch_by_last_updated_at(last_updated_at, options) do |record|
+              activity_logs << record
+            end
+
+            Guess::SchemaGuess.from_hash_records(activity_logs)
+          end
+
+          def each(last_updated_at, options={}, &block)
+            offset = fetch_by_last_updated_at(last_updated_at, options, &block)
+
+            while offset
+              offset = fetch_by_offset(offset, options, &block)
+            end
+          end
+
+          private
+
+          def fetch_by_last_updated_at(last_updated_at, options={}, &block)
+            last_updated_at = last_updated_at.to_s
+            last_updated_at = Time.parse(last_updated_at).iso8601
+
+            request = {
+              start_position: {
+                oldest_created_at: last_updated_at,
+              },
+            }
+
+            fetch(request, options, &block)
+          end
+
+          def fetch_by_offset(offset, options={}, &block)
+            request = {
+              start_position: {
+                offset: offset,
+              },
+            }
+
+            fetch(request, options, &block)
+          end
+
+          def fetch(request, options={}, &block)
+            request[:batch_size] = options[:batch_size] || 100
+
+            response = savon.call(:get_lead_changes, message: request)
+            remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
+            Embulk.logger.info "Remaining records: #{remaining}"
+
+            activities = response.body[:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
+            activities.each do |activity|
+              record = {
+                "id" => activity[:id],
+                # embulk can't treat DateTime
+                "activity_date_time" => activity[:activity_date_time].to_time,
+                "activity_type" => activity[:activity_type],
+                "mktg_asset_name" => activity[:mktg_asset_name],
+                "mkt_person_id" => activity[:mkt_person_id],
+              }
+
+              activity[:activity_attributes][:attribute].each do |attributes|
+                name = attributes[:attr_name]
+                value = attributes[:attr_value]
+
+                record[name] = value
+              end
+
+              block.call(record)
+            end
+
+            if remaining > 0
+              response.body[:success_get_lead_changes][:result][:new_start_position][:offset]
+            else
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end
+