embulk-input-marketo 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7de3f0f124846b84a0df2b1d4fd7824c1453b3ff
-  data.tar.gz: 7796e77550eb46c7f4208f373dc9c84b5b8b7e0d
+  metadata.gz: 9d395df72b2b9fafe6be7824001fb866c00f1f08
+  data.tar.gz: b344e828ecd3eb921ba38f9bc3c6bf26a3271ebe
 SHA512:
-  metadata.gz: e765f8a460352604ee6b77f49e03f39d25db1a1b4a77d288c55f7e946bd9e88e8bcccd9c680fb2f281eb08e148024af47c86cc8cf3e14f6c1e26ce2e1e7c6c4b
-  data.tar.gz: 56c092c2965c8c36f2866953e2a74e123d22573cb48ab23ffd7dcbc793b6ab167f22f29c0cfc83d81a6b36a9d1256b2d182b950a0f8cee93d076a1591528bb44
+  metadata.gz: 1893639b47f6268c154c4636fc432152b3a9061e2a8292edba2b03c0b32a51b85152f0d2c5f861ccb5404e4d5ca091670b92e411a9a25c3316470735483d2b07
+  data.tar.gz: c75fdc17d13700f6aef74022e9d5849777f5b1d861bd2a62ed2d24008c87245ee8cf7ed53c17acb432b8788222a9174a725ac11368571a88d482f355c9c55a5f

data/.travis.yml

@@ -10,3 +10,17 @@ notifications:
     secure: NP6hpwbwLnX9jFgENk3FN3B4jO4b2bIeK7gU9tRw87QmlUFDCcypSVYJrd7j1oQP4plZy+ohZMCs7RQJmtP1k2RQ99tu2ncWhslpz6f5zZo1Hjx4hleFpAv6N9tycfxEILcq/USxcTdM/8/YPtDr5WXM+w3hTuE21VlxqT6ED9LUYaiiUZeZApBXYAe36BIOGe+TtmwWrMfXvaqaoAt0A7pIzaXPD4i7WxR+qAMM4gjJGzH4JKs2zoQ2MgI81xD5Pe0yz/5ciiHATJj0WFkrOeZaoqFvW5MQ20eArvfW9dC7/fXnrPx8RPd6iaD29hQfnPI2mnWRvXxch0DsBcYOfFzYpdgx43XVTXVJ3zyudbgqnL8RrTbwIMsxIrEW3hAZ1MvxiKDBACWFdbzz8GAjywaq0zRZ/zKam1zcDRMQAWnZP54wUFHfSXvZWhRDzImSYTM75EqaspTabRQp89SpRuaGl+ab7wEDdJBXqLiUf7jHYc2K3F9o4B99luSvbxGv97M9D7P+Aee7tpFe4mjTuxjDnIVDbQzdza8qrZWXNfymSPb2pZOKP535alF92XalExBQnxKPzmUYDpTksrsgYPZeUHcnwO6J5lCPfPerWJ/U+s4PEm0HE5TpghhsZ901gYxCeDw/KnVQGsIy3f8YAI/+YVwjl4Ld4BOd1MV2PB8=
 jdk:
   - oraclejdk8
+gemfile:
+  - gemfiles/embulk-0.6.16
+  - gemfiles/embulk-0.6.17
+  - gemfiles/embulk-0.6.18
+  - gemfiles/embulk-0.6.19
+  - gemfiles/embulk-0.6.20
+  - gemfiles/embulk-0.6.21
+  - gemfiles/embulk-0.6.22
+  - gemfiles/embulk-0.6.23
+  - gemfiles/embulk-latest
+matrix:
+  allow_failures:
+    - gemfile: gemfiles/embulk-0.6.22
+    - gemfile: gemfiles/embulk-latest

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.1.1 - 2015-08-19
+
+* [enhanement] Support scheduled execution [#20](https://github.com/treasure-data/embulk-input-marketo/pull/20) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/18). Thanks!]
+* [maintenance] Use everyleaf-embulk_helper [#19](https://github.com/treasure-data/embulk-input-marketo/pull/19)
+
 ## 0.1.0 - 2015-07-15
 
 We implemented activity_log plugin for marketo, so config generated from 0.0.1 should be modified. Please check README.md to do it.

data/Rakefile

@@ -1,5 +1,5 @@
 require "bundler/gem_tasks"
-require "json"
+require "everyleaf/embulk_helper/tasks"
 
 task default: :test
 
@@ -8,53 +8,7 @@ task :test do
   ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
 end
 
-namespace :release do
-  desc "Add header of now version release to ChangeLog and bump up version"
-  task :prepare do
-    root_dir = Pathname.new(File.expand_path("../", __FILE__))
-    changelog_file = root_dir.join("CHANGELOG.md")
-    gemspec_file = root_dir.join("embulk-input-marketo.gemspec")
-
-    system("git fetch origin")
-
-    # detect merged PR
-    old_version = gemspec_file.read[/spec\.version += *"([0-9]+\.[0-9]+\.[0-9]+)"/, 1]
-    pr_numbers = `git log v#{old_version}..origin/master --oneline`.scan(/#[0-9]+/)
-
-    if !$?.success? || pr_numbers.empty?
-      puts "Detecting PR failed. Please confirm if any PR were merged after the latest release."
-      exit(false)
-    end
-
-    # Generate new version
-    major, minor, patch = old_version.split(".").map(&:to_i)
-    new_version = "#{major}.#{minor}.#{patch + 1}"
-
-    # Update ChangeLog
-    pr_descriptions = pr_numbers.map do |number|
-      body = open("https://api.github.com/repos/treasure-data/embulk-input-marketo/issues/#{number.gsub("#", "")}").read
-      payload = JSON.parse(body)
-      "* [] #{payload["title"]} [#{number}](https://github.com/treasure-data/embulk-input-marketo/pull/#{number.gsub('#', '')})"
-    end.join("\n")
-
-    new_changelog = <<-HEADER
-## #{new_version} - #{Time.now.strftime("%Y-%m-%d")}
-#{pr_descriptions}
-
-#{changelog_file.read.chomp}
-HEADER
-
-    File.open(changelog_file, "w") {|f| f.write(new_changelog) }
-
-    # Update version.rb
-    old_content = gemspec_file.read
-    File.open(gemspec_file, "w") do |f|
-      f.write old_content.gsub(/(spec\.version += *)".*?"/, %Q!\\1"#{new_version}"!)
-    end
-
-    # Update Gemfile.lock
-    system("bundle install")
-
-    puts "ChangeLog, version and Gemfile.lock were updated. New version is #{new_version}."
-  end
-end
+Everyleaf::EmbulkHelper::Tasks.install(
+  gemspec: "./embulk-input-marketo.gemspec",
+  github_name: "treasure-data/embulk-input-marketo",
+)

data/embulk-input-marketo.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-marketo"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["uu59", "yoshihara"]
   spec.summary       = "Marketo input plugin for Embulk"
   spec.description   = "Loads records from Marketo."
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'test-unit-rr'
   spec.add_development_dependency 'codeclimate-test-reporter'
+  spec.add_development_dependency 'everyleaf-embulk_helper'
 end

data/gemfiles/embulk-0.6.16

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.16"

data/gemfiles/embulk-0.6.17

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.17"

data/gemfiles/embulk-0.6.18

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.18"

data/gemfiles/embulk-0.6.19

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.19"

data/gemfiles/embulk-0.6.20

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.20"

data/gemfiles/embulk-0.6.21

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.21"

data/gemfiles/embulk-0.6.22

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.22"

data/gemfiles/embulk-0.6.23

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.23"

data/gemfiles/embulk-0.6.24

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.24"

data/gemfiles/embulk-0.6.25

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "0.6.25"

data/gemfiles/embulk-latest

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "> 0.6.16"

data/gemfiles/template.erb

@@ -0,0 +1,4 @@
+source 'https://rubygems.org/'
+gemspec :path => '../'
+
+gem "embulk", "<%= version %>"

data/lib/embulk/input/marketo/activity_log.rb

@@ -33,7 +33,7 @@ module Embulk
 
           count = 0
 
-          @soap.each(@last_updated_at, batch_size: batch_size) do |activity_log|
+          last_updated_at = @soap.each(@last_updated_at, batch_size: batch_size) do |activity_log|
             values = @columns.map do |column|
               name = column["name"].to_s
               activity_log[name]
@@ -47,6 +47,10 @@ module Embulk
           page_builder.finish
 
           commit_report = {}
+          if !preview? && last_updated_at
+            commit_report = {last_updated_at: last_updated_at}
+          end
+
           return commit_report
         end
       end

data/lib/embulk/input/marketo/base.rb

@@ -37,7 +37,9 @@ module Embulk
         def self.resume(task, columns, count, &control)
           commit_reports = yield(task, columns, count)
 
-          next_config_diff = {}
+          # NOTE: If this plugin supports to run by multi threads, this
+          # implementation is terrible.
+          next_config_diff = commit_reports.first
           return next_config_diff
         end
 

data/lib/embulk/input/marketo_api/soap/activity_log.rb

@@ -16,11 +16,13 @@ module Embulk
           end
 
           def each(last_updated_at, options={}, &block)
-            offset = fetch_by_last_updated_at(last_updated_at, options, &block)
-
-            while offset
-              offset = fetch_by_offset(offset, options, &block)
+            response = fetch_by_last_updated_at(last_updated_at, options, &block)
+            while response[:remaining_count] > 0 do
+              offset = response[:offset]
+              response = fetch_by_offset(offset, options, &block)
             end
+
+            response[:last_updated_at]
           end
 
           private
@@ -55,7 +57,15 @@ module Embulk
             remaining = response.body[:success_get_lead_changes][:result][:remaining_count].to_i
             Embulk.logger.info "Remaining records: #{remaining}"
 
-            activities = response.body[:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
+            activities_list = response.body[:success_get_lead_changes][:result][:lead_change_record_list]
+
+            if activities_list.nil?
+              Embulk.logger.info "No record is fetched."
+              return {remaining_count: 0, offset: nil, last_updated_at: nil}
+            end
+
+            activities = activities_list[:lead_change_record].sort { |activity| activity[:activity_date_time] }
+
             activities.each do |activity|
               record = {
                 "id" => activity[:id],
@@ -76,11 +86,11 @@ module Embulk
               block.call(record)
             end
 
-            if remaining > 0
-              response.body[:success_get_lead_changes][:result][:new_start_position][:offset]
-            else
-              nil
-            end
+            {
+              remaining_count: remaining,
+              offset: response.body[:success_get_lead_changes][:result][:new_start_position][:offset],
+              last_updated_at: activities.last[:activity_date_time]
+            }
           end
         end
       end

data/test/activity_log_fixtures.rb

@@ -13,6 +13,10 @@ module ActivityLogFixtures
     activity_logs(preview_response)
   end
 
+  def none_activity_log_response
+    activity_logs(none_response)
+  end
+
   def activity_logs(body)
     Struct.new(:body).new({
       success_get_lead_changes: {
@@ -152,4 +156,16 @@ module ActivityLogFixtures
       }
     }
   end
+
+  def none_response
+    {
+      result: {
+        return_count: 0,
+        remaining_count: 0,
+        new_start_position: {
+        },
+        lead_change_record_list: nil
+      }
+    }
+  end
 end

data/test/embulk/input/marketo/test_activity_log.rb

@@ -110,6 +110,20 @@ module Embulk
             @plugin.run
           end
 
+          def test_run_with_no_response
+            stub(@plugin).preview? { false }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: request) do
+                none_activity_log_response
+              end
+            end
+
+            mock(@page_builder).finish
+
+            @plugin.run
+          end
+
           def test_preview_through
             stub(@plugin).preview? { true }
 

data/test/embulk/input/marketo/test_base.rb

@@ -21,6 +21,17 @@ module Embulk
           Base.transaction(config, &control)
         end
 
+        def test_resume
+          next_config_diff = {last_updated_at: last_updated_at}
+          control = proc { [next_config_diff] } # In actual, embulk prepares control block returning Array.
+          columns = task[:columns].map do |col|
+            Column.new(nil, col["name"], col["type"].to_sym)
+          end
+
+          actual = Base.resume(task, columns, 1, &control)
+          assert_equal(next_config_diff, actual)
+        end
+
         private
 
         def config

data/test/embulk/input/marketo_api/soap/test_activity_log.rb

@@ -27,11 +27,30 @@ module Embulk
             end
 
             proc = proc{ "" }
-            activity_log_count = next_stream_activity_logs_response[:body][:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record].size
+            activity_logs = next_stream_activity_logs_response[:body][:success_get_lead_changes][:result][:lead_change_record_list][:lead_change_record]
+            last_activity_log = activity_logs.last
 
-            mock(proc).call(anything).times(activity_log_count)
+            mock(proc).call(anything).times(activity_logs.size)
+            assert_equal(last_activity_log[:activity_date_time], soap.each(last_updated_at, &proc))
+          end
+
+          def test_each_with_no_response
+            request = {
+              start_position: {
+                oldest_created_at: Time.parse(last_updated_at).iso8601,
+              },
+              batch_size: 100
+            }
+
+            any_instance_of(Savon::Client) do |klass|
+              mock(klass).call(:get_lead_changes, message: request) do
+                none_activity_log_response
+              end
+            end
+
+            proc = proc{ "" }
 
-            soap.each(last_updated_at, &proc)
+            assert_nil(soap.each(last_updated_at, &proc))
           end
 
           class TestMetadata < self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - uu59
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-15 00:00:00.000000000 Z
+date: 2015-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -129,6 +129,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: everyleaf-embulk_helper
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Loads records from Marketo.
 email:
 - k@uu59.org
@@ -146,6 +160,18 @@ files:
 - README.md
 - Rakefile
 - embulk-input-marketo.gemspec
+- gemfiles/embulk-0.6.16
+- gemfiles/embulk-0.6.17
+- gemfiles/embulk-0.6.18
+- gemfiles/embulk-0.6.19
+- gemfiles/embulk-0.6.20
+- gemfiles/embulk-0.6.21
+- gemfiles/embulk-0.6.22
+- gemfiles/embulk-0.6.23
+- gemfiles/embulk-0.6.24
+- gemfiles/embulk-0.6.25
+- gemfiles/embulk-latest
+- gemfiles/template.erb
 - lib/embulk/input/marketo/activity_log.rb
 - lib/embulk/input/marketo/base.rb
 - lib/embulk/input/marketo/lead.rb