embulk-input-marketo 0.5.6 → 0.5.7.alpha.1

data/.ruby-version

@@ -1 +0,0 @@
-jruby-9.0.5.0

data/.travis.yml.erb

@@ -1,42 +0,0 @@
-language: ruby
-
-addons:
-  code_climate:
-    repo_token:
-      secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
-
-before_install:
-  - |
-    # Currently, Travis can't treat jruby 9.0.1.0
-    ruby -v
-    rvm get head
-    rvm use jruby-9.0.5.0 --install
-    ruby -v
-    gem i bundler
-    bundle install
-
-jdk: oraclejdk8
-
-rvm:
-  - jruby-9.0.5.0
-
-script: bundle exec rake cov
-
-gemfile:
-<% versions.each do |file| -%>
-  - gemfiles/<%= file %>
-<% end -%>
-
-matrix:
-  exclude:
-    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
-  include:
-    <% matrix.each do |m| -%>
-<%= m %>
-    <% end %>
-
-  allow_failures:
-    # Ignore failure for *-latest
-    <% versions.find_all{|file| file.to_s.match(/-latest/)}.each do |file| -%>
-- gemfile: <%= file %>
-    <% end %>

data/Gemfile

@@ -1,3 +0,0 @@
-source 'https://rubygems.org/'
-
-gemspec

data/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2015 Everyleaf Corporation
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

data/Rakefile

@@ -1,20 +0,0 @@
-require "bundler/gem_tasks"
-require "everyleaf/embulk_helper/tasks"
-
-task default: :test
-
-desc "Run tests"
-task :test do
-  ruby("test/run-test.rb", "--use-color=yes", "--collector=dir")
-end
-
-desc "Run tests with coverage"
-task :cov do
-  ENV["COVERAGE"] = "1"
-  ruby("--debug", "test/run-test.rb", "--use-color=yes", "--collector=dir")
-end
-
-Everyleaf::EmbulkHelper::Tasks.install(
-  gemspec: "./embulk-input-marketo.gemspec",
-  github_name: "treasure-data/embulk-input-marketo",
-)

data/embulk-input-marketo.gemspec

@@ -1,28 +0,0 @@
-Gem::Specification.new do |spec|
-  spec.name          = "embulk-input-marketo"
-  spec.version       = "0.5.6"
-  spec.authors       = ["uu59", "yoshihara"]
-  spec.summary       = "Marketo input plugin for Embulk"
-  spec.description   = "Loads records from Marketo."
-  spec.email         = ["k@uu59.org", "h.yoshihara@everyleaf.com"]
-  spec.licenses      = ["Apache2"]
-  spec.homepage      = "https://github.com/treasure-data/embulk-input-marketo"
-
-  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
-  spec.require_paths = ["lib"]
-
-  spec.add_dependency 'savon', ['~> 2.11.1']
-  spec.add_dependency 'httpclient', '>= 2.8.3' # To use tcp_keepalive
-  spec.add_dependency 'httpi', '2.4.2' # To use tcp_keepalive with monky patch (See lib/embulk/input/marketo/base.rb)
-  spec.add_dependency 'perfect_retry', ["~> 0.5"]
-  spec.add_development_dependency 'embulk', [">= 0.6.13", "< 1.0"]
-  spec.add_development_dependency 'bundler', ['~> 1.0']
-  spec.add_development_dependency 'rake', ['>= 10.0']
-  spec.add_development_dependency 'pry'
-  spec.add_development_dependency 'test-unit'
-  spec.add_development_dependency 'rr', "1.1.2" # 1.2.0 causes strange error.. https://travis-ci.org/treasure-data/embulk-input-marketo/jobs/141537151
-  spec.add_development_dependency 'test-unit-rr'
-  spec.add_development_dependency 'codeclimate-test-reporter', "~> 0.5"
-  spec.add_development_dependency 'everyleaf-embulk_helper'
-end

data/gemfiles/embulk-latest

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "> 0.8.7"

data/gemfiles/template.erb

@@ -1,4 +0,0 @@
-source 'https://rubygems.org/'
-gemspec :path => '../'
-
-gem "embulk", "<%= version %>"

data/lib/embulk/input/marketo/activity_log.rb

@@ -1,103 +0,0 @@
-require "embulk/input/marketo/base"
-
-module Embulk
-  module Input
-    module Marketo
-      class ActivityLog < Base
-        BATCH_SIZE_DEFAULT = 100
-
-        Plugin.register_input("marketo/activity_log", self)
-
-        def self.target
-          :activity_log
-        end
-
-        def self.resume(task, columns, count, &control)
-          task_reports = yield(task, columns, count)
-
-          # NOTE: If this plugin supports to run by multi threads, this
-          # implementation is terrible.
-          next_config_diff = task_reports.first
-          return next_config_diff
-        end
-
-        def self.transaction(config, &control)
-          endpoint_url = config.param(:endpoint, :string)
-
-          range = format_range(config)
-
-          task = {
-            endpoint_url: endpoint_url,
-            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-            user_id: config.param(:user_id, :string),
-            encryption_key: config.param(:encryption_key, :string),
-            from_datetime: range[:from],
-            to_datetime: range[:to],
-            retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
-            retry_limit: config.param(:retry_limit, :integer, default: 5),
-            columns: config.param(:columns, :array)
-          }
-          validate_url(task[:endpoint_url], "endpoint")
-          validate_url(task[:wsdl_url], "wsdl")
-
-          resume(task, embulk_columns(config), 1, &control)
-        end
-
-        def self.guess(config)
-          client = soap_client(config)
-          range = format_range(config)
-
-          schema = client.metadata(range[:from], batch_size: PREVIEW_COUNT)
-          columns = schema.map do |c|
-            column = {name: c.name, type: c.type}
-            column[:format] = c.format if c.format
-            column
-          end
-
-          return {"columns" => columns}
-        end
-
-        def init
-          @columns = task[:columns]
-          @soap = MarketoApi.soap_client(task, target)
-          @options = {
-            retry_initial_wait_sec: task[:retry_initial_wait_sec],
-            retry_limit: task[:retry_limit],
-            to: task[:to_datetime],
-            batch_size: (preview? ? PREVIEW_COUNT : BATCH_SIZE_DEFAULT),
-          }
-        end
-
-        def run
-          counter = 0
-          latest_updated_at = @soap.each(task[:from_datetime], @options) do |activity_log|
-            page_builder.add(format_record(activity_log))
-            break if preview? && (counter += 1) >= PREVIEW_COUNT
-          end
-
-          page_builder.finish
-
-          task_report = {}
-          if !preview?
-            from_datetime = latest_updated_at || task[:from_datetime]
-            if from_datetime
-              task_report = {
-                from_datetime: from_datetime
-              }
-            end
-          end
-
-          return task_report
-        end
-
-        def format_record(activity_log)
-          @columns.map do |column|
-            name = column["name"].to_s
-            value = activity_log[name]
-            cast_value(column, value)
-          end
-        end
-      end
-    end
-  end
-end

data/lib/embulk/input/marketo/base.rb

@@ -1,139 +0,0 @@
-require "embulk/input/marketo_api"
-
-module Embulk
-  module Input
-    module Marketo
-      class Base < InputPlugin
-        PREVIEW_COUNT = 15
-
-        attr_reader :soap
-
-        def self.target
-          raise NotImplementedError
-        end
-
-        def self.soap_client(config)
-          endpoint_url = config.param(:endpoint, :string)
-          soap_config = {
-            endpoint_url: endpoint_url,
-            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-            user_id: config.param(:user_id, :string),
-            encryption_key: config.param(:encryption_key, :string),
-          }
-          validate_url(soap_config[:endpoint_url], "endpoint")
-          validate_url(soap_config[:wsdl_url], "wsdl")
-
-          MarketoApi.soap_client(soap_config, target)
-        end
-
-        def self.embulk_columns(config)
-          config.param(:columns, :array).map do |column|
-            name = column["name"]
-            type = column["type"].to_sym
-
-            Column.new(nil, name, type, column["format"])
-          end
-        end
-
-        def self.format_range(config)
-          if config.param(:last_updated_at, :string, default: nil)
-            Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
-          end
-
-          from_datetime = config.param(:from_datetime, :string)
-          to_datetime = config.param(:to_datetime, :string, default: Time.now.to_s)
-
-          # check from/to format to parse
-          begin
-            Time.parse(from_datetime)
-            Time.parse(to_datetime)
-          rescue => e
-            # possibly Time.parse fail
-            raise ConfigError.new e.message
-          end
-
-          if Time.parse(from_datetime) > Time.parse(to_datetime)
-            raise ConfigError.new "config: from_datetime '#{from_datetime}' is later than '#{to_datetime}'."
-          end
-
-          {
-            from: from_datetime,
-            to: to_datetime,
-          }
-        end
-
-        def self.timeslice(from, to, count)
-          generate_time_range(from, to).each_slice(count).to_a
-        end
-
-        def self.generate_time_range(from, to)
-          # e.g. from = 2010-01-01 15:00, to = 2010-01-03 09:30
-          # convert to such array:
-          # [
-          #   {from: 2010-01-01 15:00, to: 2010-01-01 16:00},
-          #   {from: 2010-01-01 16:00, to: 2010-01-01 17:00},
-          #   ...
-          #   {from: 2010-01-03 08:00, to: 2010-01-03 09:00},
-          #   {from: 2010-01-03 09:00, to: 2010-01-03 09:30},
-          # ]
-          # to fetch data from Marketo API with each day as
-          # desribed on official blog:
-          # http://developers.marketo.com/blog/performance-tuning-api-requests/
-          to ||= Time.now
-          from = Time.parse(from) unless from.is_a?(Time)
-          to = Time.parse(to) unless to.is_a?(Time)
-
-          result = []
-          since = from
-          while since < to
-            next_since = since + 3600
-            if to < next_since
-              next_since = to
-            end
-            result << {
-              "from" => since,
-              "to" => next_since
-            }
-            since = next_since
-          end
-          result
-        end
-
-        private
-
-        def self.validate_url(url, key)
-          URI.parse(url)
-        rescue URI::InvalidURIError
-          raise ConfigError.new("#{key}: '#{url}' is not a valid URL.")
-        end
-
-        def preview?
-          begin
-            org.embulk.spi.Exec.isPreview()
-          rescue java.lang.NullPointerException => e
-            false
-          end
-        end
-
-        def cast_value(column, value)
-          return if value.to_s.empty? # nil or empty string
-
-          case column["type"].to_s
-          when "timestamp"
-            begin
-              Time.parse(value)
-            rescue => e
-              raise ConfigError.new "Can't parse as Time '#{value}' (column is #{column["name"]})"
-            end
-          else
-            value
-          end
-        end
-
-        def target
-          self.class.target
-        end
-      end
-    end
-  end
-end

data/lib/embulk/input/marketo/lead.rb

@@ -1,143 +0,0 @@
-require "embulk/input/marketo/base"
-
-module Embulk
-  module Input
-    module Marketo
-      class Lead < Base
-        TIMESLICE_COUNT_PER_TASK = 24
-
-        Plugin.register_input("marketo/lead", self)
-
-        def self.target
-          :lead
-        end
-
-        def self.guess(config)
-          if config.param(:last_updated_at, :string, default: nil)
-            Embulk.logger.warn "config: last_updated_at is deprecated. Use from_datetime/to_datetime"
-          end
-
-          client = soap_client(config)
-          metadata = client.metadata
-
-          return {"columns" => generate_columns(metadata)}
-        end
-
-        def self.resume(task, columns, count, &control)
-          task_reports = yield(task, columns, count)
-
-          return {}
-        end
-
-        def self.transaction(config, &control)
-          endpoint_url = config.param(:endpoint, :string)
-
-          range = format_range(config)
-          ranges = timeslice(range[:from], range[:to], TIMESLICE_COUNT_PER_TASK)
-
-          append_processed_time_column = config.param(:append_processed_time_column, :bool, default: true)
-
-          task = {
-            endpoint_url: endpoint_url,
-            wsdl_url: config.param(:wsdl, :string, default: "#{endpoint_url}?WSDL"),
-            user_id: config.param(:user_id, :string),
-            encryption_key: config.param(:encryption_key, :string),
-            from_datetime: range[:from],
-            to_datetime: range[:to],
-            ranges: ranges,
-            retry_initial_wait_sec: config.param(:retry_initial_wait_sec, :integer, default: 1),
-            retry_limit: config.param(:retry_limit, :integer, default: 5),
-            append_processed_time_column: append_processed_time_column,
-            columns: config.param(:columns, :array),
-          }
-          validate_url(task[:endpoint_url], "endpoint")
-          validate_url(task[:wsdl_url], "wsdl")
-
-          columns = embulk_columns(config)
-          if append_processed_time_column
-            processed_time_column = Column.new(nil, :processed_time, :timestamp, "%Y-%m-%dT%H:%M:%S%z")
-            columns << processed_time_column
-          end
-
-          resume(task, columns, ranges.size, &control)
-        end
-
-        def self.generate_columns(metadata)
-          columns = [
-            {name: "id", type: "long"},
-            {name: "email", type: "string"},
-          ]
-
-          metadata.each do |field|
-            type =
-              case field[:data_type]
-              when "integer"
-                "long"
-              when "datetime", "date"
-                "timestamp"
-              when "string", "text", "phone", "currency"
-                "string"
-              when "boolean"
-                "boolean"
-              when "float"
-                "double"
-              else
-                "string"
-              end
-
-            columns << {name: field[:name], type: type}
-          end
-
-          columns
-        end
-
-        def init
-          @columns = task[:columns]
-          if preview?
-            # Try newer date at first to reduce cache miss hit
-            @ranges = task[:ranges].flatten.sort_by{|range| Time.parse(range["to"])}.reverse
-          else
-            @ranges = task[:ranges][index]
-          end
-          @soap = MarketoApi.soap_client(task, target)
-          @append_processed_time_column = task[:append_processed_time_column]
-          @options = {
-            retry_initial_wait_sec: task[:retry_initial_wait_sec],
-            retry_limit: task[:retry_limit],
-          }
-          @options[:batch_size] = PREVIEW_COUNT if preview?
-        end
-
-        def run
-          counter = 0
-          catch(:finish) do
-            @ranges.each do |range|
-              soap.each(range, @options) do |lead|
-                page_builder.add(format_record(lead, range))
-                throw(:finish) if preview? && (counter += 1) >= PREVIEW_COUNT
-              end
-            end
-          end
-
-          page_builder.finish
-
-          task_report = {}
-          return task_report
-        end
-
-        def format_record(lead, range)
-          values = @columns.map do |column|
-            name = column["name"].to_s
-            value = (lead[name] || {})[:value]
-            cast_value(column, value)
-          end
-
-          if @append_processed_time_column
-            values << Time.parse(range["from"])
-          end
-          values
-        end
-      end
-    end
-  end
-end