embulk-input-marketo 0.5.6 → 0.5.7.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bffcdd0cfda308bad6bf7b672bb37650be4286c6
-  data.tar.gz: e76f036709178a65f2f5ddc682e1012508ad0ea2
+  metadata.gz: 1b3e412c76a242171826a25d62d41126de0ec2bb
+  data.tar.gz: 97f8c071357997a07347e03c28ffa58b5d90cb4a
 SHA512:
-  metadata.gz: 61de5a01fc335d6d8c997efe1bf2a7228e33f6a6ff776ea205e6b0e89f997428872b37a80655763179f3a6efa4cf94d6d97cddbda724596b2a2b4755f07927fe
-  data.tar.gz: 403f20cf0ed5c6fdbfc021fc486b59846ad1a8b6312a2d87808817fe728789d3ba4398513d1545c7c73d01ffe2caf2e29e9162c0eede66a89b3bae2874b1a3a9
+  metadata.gz: a2969c47bc9ce9fe7968f2dcf256709c25d506ccf2e0d6e5c7510635f6220fa1df339f9b6af341f6a5204be7e66b332e6d8040b0b64213137ca0adf6f0a06a1b
+  data.tar.gz: 63d3dc7bed1e6930fa8c03584bcf4b83077c8e0b08ff7c8517fc92034debff28918bbfbcd86a1a0c8233d33f51b3d08c1f4b35206072b92279501f38d40465a8

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,37 @@
+### Are all connections created by the plugin secure?
+
+- [ ] Does it opt secure communication standard? Such as HTTPS, SSH, SFTP, SMTP STARTTLS. If not check with CISO to decide we can deploy the plugin.
+- [ ] Does support both authentication and encryption appropriately? Such as: "just encrypting without authentication" that is insecure.
+
+### Does the plugin connect only to its expected external site which the customer explicitly set in their config file?
+
+- [ ] Does NOT connect unexpected external site and our internal endpoints? Such as: “v3/job/:id/set_started” callback endpoint.
+
+### Does NOT the plugin persist any customers' private information? Identify the private information beforehand.
+
+- [ ] Does NOT include them in (temporary) files?
+- [ ] Does NOT include them in log messages and exception messages?
+
+### What kind of environments does the plugin interact with?
+
+- [ ] Does NOT execute any shell command?
+- [ ] Does NOT read any files on the running instance? Such as: "/etc/passwords". It’s ok to read temporary files that the plugin wrote.
+- [ ] Does use to create temporary files by spi.TempFileSpace utility to avoid the conflict of the file names.
+- [ ] Does NOT get environment variables or JVM system properties at runtime? Such as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment variables
+
+### Does NOT the plugin use insecure libraries?
+
+- [ ] Line up all depending library so that we can identify the impact of security incident of those library if any.
+- [ ] Check libraries usage of the plugin; all security check list must apply to the library usages. Such as "Are all connections created by the library secure?"
+
+### Does NOT the plugin source code repository contain kinds of credentials
+
+- [ ] API keys
+- [ ] Passwords
+
+### Make sure to free up all resources allocated during Embulk transaction “committing” or “rolling back”or before.
+
+- [ ] Network (connections, pooled connections)
+- [ ] Memory (cache in static variables)
+- [ ] File (temporary files)
+- [ ] CPU (threads, processes)

data/.gitignore

@@ -1,5 +1,14 @@
 *~
 /pkg/
 /tmp/
-/.bundle/
-/Gemfile.lock
+*.gemspec
+.gradle/
+classpath/
+classes/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+

data/.travis.yml

@@ -1,45 +1,5 @@
-language: ruby
-
-addons:
-  code_climate:
-    repo_token:
-      secure: "cYPXD2Dv1VOU0rdUSsevrGuHcjBajCTq8s961R8d2pPkX+V1AbmIMKK0if00qMgqR7D65p6jOnfiuZfXolF1z4awqLXNu7LhP6zOr1hMRCOHOfjj/SJLbm5MFreN81G+6k4XpfBlA9YShA9E/nNVxBmakxC8aCnlJHGfDUbkb9kmNX/LXSNf0efZMRVr3vN8tP6tHP0TUCx90A2FNols9qHgl+boxF090OpgFTspyeqC6K82Q4D97RwajnM+oKIBMfxuc4Pc1n+PZ+mQQfYNK7ze4WszXRtQydwDD7rtF5d6IwczElZ243GjXxdJWL6inUvcMJQogC5X8ayh7m0ZNT2awfbJXT1tsLte42j+/eUAPaCD07bE8XzgdmfTdmpxP6nHasAMkleDMlcBRrxht+cGD61qEXcsSKd0c7mnvg35L+hl39NGFde0yz1Xxx/D69p9KHE0pa7cZNC7D8n1w9AUxFdl8OX6rfDlgiapIonAF/QG/5Z3ltHDzZu6XK1MMFugrdV7SxuSepL3xgBv5jN0jMA/GVmvFbXf6FGDXxt/bhSUCZoTJ/c8BaRnIM+QTkBts6TqgQ6BeUQNAe5p0TtssNwMW8RC4DpkaLDeQRSScJJ6V6UDpjsPJ3hcozpzrCVwy/5F72MYzcHeKg85EF30aA94Q+EDbsKUR0BEeGo="
-
-before_install:
-  - |
-    # Currently, Travis can't treat jruby 9.0.1.0
-    ruby -v
-    rvm get head
-    rvm use jruby-9.0.5.0 --install
-    ruby -v
-    gem i bundler
-    bundle install
-
-jdk: oraclejdk8
-
-rvm:
-  - jruby-9.0.5.0
-
-script: bundle exec rake cov
-
-gemfile:
-  - gemfiles/embulk-0.8.0-latest
-  - gemfiles/embulk-0.8.7
-  - gemfiles/embulk-0.8.8
-  - gemfiles/embulk-latest
-
-matrix:
-  exclude:
-    - jdk: oraclejdk8 # Ignore all matrix at first, use `include` to allow build
-  include:
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.0-latest}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.7}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-0.8.8}
-    - {rvm: jruby-9.0.0.0, gemfile: gemfiles/embulk-latest}
-    
-
-  allow_failures:
-    # Ignore failure for *-latest
-    - gemfile: embulk-0.8.0-latest
-    - gemfile: embulk-latest
-    
+language: java
+jdk:
+- oraclejdk8
+script:
+- ./gradlew --info check jacocoTestReport

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -1,83 +1,32 @@
-[![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
-[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
-[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
-[![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
-
 # Marketo input plugin for Embulk
 
-embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
-
-- Lead
-- Activity log
-
-This plugin uses Marketo SOAP API.
+TODO: Write short description here and build.gradle file.
 
 ## Overview
 
-Required Embulk version >= 0.8.7 (since 0.5.0).
-
 * **Plugin type**: input
-* **Resume supported**: no for `marketo/lead`, yes for `marketo/activity_log`
-* **Cleanup supported**: no
-* **Guess supported**: yes
-
-## Install
-
-```
-$ embulk gem install embulk-input-marketo
-```
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+* **Guess supported**: no
 
 ## Configuration
 
-### API
-
-Below parameters are shown in "Admin" > "Web Services" page in Marketo.
-
-### marketo/lead
-
-- **endpoint** SOAP endpoint URL for your account (string, required)
-- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")
-- **user_id** Your user id (string, reqiured)
-- **encryption_key** Your encryption key (string, reqiured)
-- **from_datetime** Fetch leads since this time (string, required)
-- **to_datetime** Fetch leads until this time (string, default: Time.now)
-- **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
-- **retry_limit**: Try to retry this times (integer, default: 5)
-- **append_processed_time_column**: If you want the column for processed time (boolean, default: true)
-
-### marketo/activity_log
-
-- **endpoint** SOAP endpoint URL for your account (string, required)
-- **wsdl** SOAP endpoint URL for your account (string, default: endpoint + "?WSDL")(Note "?WSDL" needs to be in all caps)
-- **user_id** Your user id (string, reqiured)
-- **encryption_key** Your encryption key (string, reqiured)
-- **from_datetime** Fetch activity_logs since this time (string, required)
-- **to_datetime** Fetch activity_logs until this time (string, default: Time.now)
-- **retry_initial_wait_sec** Wait seconds for exponential backoff initial value (integer, default: 1)
-- **retry_limit**: Try to retry this times (integer, default: 5)
-
-### Selecting plugin type
-
-You should specify `type: marketo/lead` or `type: marketo/activity_log` on your demand.
-
+- **option1**: description (integer, required)
+- **option2**: description (string, default: `"myvalue"`)
+- **option3**: description (string, default: `null`)
 
 ## Example
 
-For lead, you have `partial-config.yml` like below:
-
 ```yaml
 in:
-  type: marketo/lead
-  endpoint: https://soap-end-point.mktoapi.com/
-  wsdl: https://wsdl-url.mktoapi.com/?WSDL
-  user_id: user_ABC123
-  encryption_key: TOPSECRET
-  from_datetime: "2015-06-30"
-out:
-  type: stdout
+  type: marketo
+  option1: example1
+  option2: example2
 ```
 
-You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
 
-Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
+## Build
 
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,102 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+    id "checkstyle"
+    id "jacoco"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+    maven {
+        url "https://dl.bintray.com/embulk-base-restclient/maven"
+    }
+}
+configurations {
+    provided
+}
+version = "0.5.7.alpha.1"
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+
+dependencies {
+    compile  "org.embulk:embulk-core:0.8.33"
+    provided "org.embulk:embulk-core:0.8.33"
+    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
+    compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
+    testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.16:tests"
+    testCompile "org.embulk:embulk-test:0.8.16"
+    testCompile "org.mockito:mockito-core:2.+"
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Tai Khuu"]
+  spec.summary       = %[Marketo input plugin for Embulk]
+  spec.description   = %[Loads records from Marketo.]
+  spec.email         = ["tai@treasuredata.com"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/khuutantai/embulk-input-marketo"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>