embulk-output-elasticsearch 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d90cf5dd0d129a6494e834700ade73ed349f32c8
-  data.tar.gz: 04daf21aaac3f43874b15f062a58628940ee1d93
+  metadata.gz: 1a3ec0cee5c598381d1fb7b9633e5b75ddd8a10b
+  data.tar.gz: c4168d8f8e4d74feea54657f95beb46c3faa424e
 SHA512:
-  metadata.gz: c6cd37883a5921dac22cd794952717c1a1eb0874e52a0bde5a8bd4cd953a40928c69ce676b3a64aa216eb6f67d3e185f258930671490982edb49cd4e49b9dbaa
-  data.tar.gz: 5c4da5add896bd7e3ae3ff8a2e3f2f107007dc53e7927fe8d9a60937563bd70b304f3f4a578d704280c538bf1ad84ab1b97cabb8c8dd579673c531c6f08c83df
+  metadata.gz: 3db949e32d6ec45729b717762b7a952013b40f32f2f7964415f215adeae29da4acf8fa229cffc69770dcb645bb7d80a236f135d38a38a5b76bfc3d4ff0fabc1e
+  data.tar.gz: b72a05b4607c9636febe5b67f825462d7df6a16cce257e70d952f79d873575cf055fc7c92f72d57d80dca9c66c3422e7b696ae7b89314462f988b8ef5be27586

data/.travis.yml

@@ -0,0 +1,26 @@
+language: java
+
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7
+
+env:
+  global:
+    - ES_HOST=localhost
+    - ES_PORT=9200
+    - ES_INDEX=embulk
+    - ES_INDEX_TYPE=embulk
+
+before_install:
+  - wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.3/elasticsearch-2.3.3.tar.gz
+  - tar zxvf elasticsearch-2.3.3.tar.gz
+  - ./elasticsearch-2.3.3/bin/elasticsearch &
+
+before_script:
+  - echo "Wait Elasticsearch wakeup"
+  - sleep 10
+
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.4.0 - 2017-03-28
+
+* [new feature] Support multiple Elasticsearch version [#32](https://github.com/muga/embulk-output-elasticsearch/pull/32)
+* [new feature] Support SSL and Basic authentication including 'Security'(formally 'Shield') [#33](https://github.com/muga/embulk-output-elasticsearch/pull/33)
+* [maintenance] Improve export logic [#34](https://github.com/muga/embulk-output-elasticsearch/pull/34)
+
 ## 0.3.1 - 2016-06-21
 
 * [maintenance] Update Elasticsearch client to 2.3.3 [#25](https://github.com/muga/embulk-output-elasticsearch/pull/25)

data/README.md

@@ -1,8 +1,7 @@
 # Elasticsearch output plugin for Embulk
 
 **Notice** This plugin doesn't support [Amazon(AWS) Elasticsearch Service](https://aws.amazon.com/elasticsearch-service/).
-Plugin uses [Transport Client](https://www.elastic.co/guide/en/elasticsearch/client/java-api/2.0/transport-client.html) but AWS Elasticsearch doesn't support this method.
-> The service supports HTTP on port 80, but does not support TCP transport.
+This plugin uses HTTP/REST Client and haven't be implemented AWS authentication.
 - *[Amazon Elasticsearch Service Limits](http://docs.aws.amazon.com/elasticsearch-service/latest/developerguide/aes-limits.html)*
 
 ## Overview
@@ -16,13 +15,18 @@ Plugin uses [Transport Client](https://www.elastic.co/guide/en/elasticsearch/cli
 
 - **mode**: "insert" or "replace". See below(string, optional, default is insert)
 - **nodes**: list of nodes. nodes are pairs of host and port (list, required)
-- **cluster_name**: name of the cluster (string, default is "elasticsearch")
+  - NOTE: This plugin uses HTTP/REST Clients and uses TCP:9200 as a default. TCP:9300 is usually used for Transport Client.
+- **use_ssl** Use SSL encryption (boolean, default is false)
+- **auth_method** (string, default is 'none') 'none'/'basic'. See also [Authentication](#authentication).
+- **user** Username for basic authentication (string, default is null)
+- **password** Password for above user (string, default is null)
+- ~~**cluster_name**: name of the cluster (string, default is "elasticsearch")~~ Not used now. May use in the future
 - **index**: index name (string, required)
 - **index_type**: index type (string, required)
 - **id**: document id column (string, default is null)
 - **bulk_actions**: Sets when to flush a new bulk request based on the number of actions currently added. (int, default is 1000)
 - **bulk_size**: Sets when to flush a new bulk request based on the size of actions currently added. (long, default is 5242880)
-- **concurrent_requests**: concurrent_requests (int, default is 5)
+- ~~**concurrent_requests**: concurrent_requests (int, default is 5)~~  Not used now. May use in the future
 
 ### Modes
 
@@ -45,11 +49,23 @@ out:
   type: elasticsearch
   mode: replace
   nodes:
-  - {host: localhost, port: 9300}
+  - {host: localhost, port: 9200}
   index: <alias name> # plugin generates index name like <index>_%Y%m%d-%H%M%S 
   index_type: <index type>
 ```
 
+### Authentication
+
+This plugin supports Basic authentication and works with [Elastic Cloud](https://cloud.elastic.co/) and 'Security'(formally Sield).
+'Security' also supports LDAP and Active Directory. This plugin doesn't supports these auth methods.
+
+```yaml
+use_ssl: true
+auth_method: basic
+user: <username>
+password: <password>
+```
+
 ## Example
 
 ```yaml
@@ -57,7 +73,7 @@ out:
   type: elasticsearch
   mode: insert
   nodes:
-  - {host: localhost, port: 9300}
+  - {host: localhost, port: 9200}
   index: <index name>
   index_type: <index type>
 ```
@@ -114,4 +130,4 @@ $ launchctl load ~/Library/LaunchAgents/environment.plist
 $ launchctl getenv ES_INDEX //try to get value.
 
 Then start your applications.
-```
+```

data/build.gradle

@@ -3,31 +3,36 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "jacoco"
+    id "checkstyle"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
-    mavenLocal()
+    // TODO: Remove this after embulk-base-restclient goes to jcenter.
+    maven {
+        url "https://dl.bintray.com/embulk-base-restclient/maven"
+    }
 }
 configurations {
     provided
 }
 
-version = "0.3.1"
+version = "0.4.0"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.5"
-    provided  "org.embulk:embulk-core:0.8.5"
-    compile 'org.elasticsearch:elasticsearch:2.3.3'
+    compile  "org.embulk:embulk-core:0.8.16"
+    provided  "org.embulk:embulk-core:0.8.16"
+    compile  "org.embulk.base.restclient:embulk-base-restclient:0.4.2"
+    compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.4.2"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.5:tests"
-    testCompile "org.embulk:embulk-standards:0.8.5"
+    testCompile "org.embulk:embulk-core:0.8.16:tests"
+    testCompile "org.embulk:embulk-standards:0.8.16"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -37,6 +42,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete 'classpath' }
 
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
 //task copyDependencies(type:Copy) {
 //    new File("$buildDir/libs/dependencies").mkdirs()
 //    into "$buildDir/libs/dependencies" from configurations.runtime

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,130 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName">
+            <property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
+        </module>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName">
+            <property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
+        </module>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 11 00:26:20 PDT 2015
+#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip

data/gradle/wrapper/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Wed Jan 13 12:41:02 JST 2016
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip