embulk-output-elasticsearch 0.4.4 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fee5ac83d6cdd1dd3d0f65f6b95e716f8b38389c
-  data.tar.gz: 67fc3c6af37486baa0f7dcbf05b189070d9d9e52
+  metadata.gz: 660287a649d29f3d92435f6b09f48a5db1a7239c
+  data.tar.gz: 8dcc2256309363657afc86d5b3b3bc2fd1dbaae8
 SHA512:
-  metadata.gz: 9e6bfc587cbaa9d535b78da022d7575e442a5619462ccca2833afa13b0a67a49a76acf94ae16693f38e5fe9f652d343c28eef489d06606b173fab2722f9926e1
-  data.tar.gz: 045a4f6338dbb233b6402021b1998fd4a4801d82f05bd4150ad1a5a121636b7c7a9b33ab23d1ab396247ea0e8331180036bfbb2e8ea95165493789ca511add3c
+  metadata.gz: 273cbdf5a463a58ac8a4e88bb71b098fe6c08a2365c1b4e98712be703a2875f92401231249817229a9aa0ac22172e3f4e68a1c596a8dd97c509af7890eaa272f
+  data.tar.gz: b7e7188312a602ca1cdca21b5dbb713166ee59bd328273b55f20f0ca707555ac2f0c6d54f9679aa3fe580f1c5b8c9ecbbefed7b455398689814c5cd583729bd0

data/.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: java
 
 jdk:
@@ -24,3 +25,7 @@ before_script:
 script:
   - ./gradlew gem
   - ./gradlew --info check jacocoTestReport
+addons:
+  hosts:
+    - example.com
+  hostname: example.com

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.5 - 2017-11-29
+* [new feature] Add "fill_null_for_empty_column" option and allow insert null value when column is empty [#47](https://github.com/embulk/embulk-output-elasticsearch/pull/47) Thanks! @kfitzgerald
+
 ## 0.4.4 - 2017-06-16
 
 * [maintenance] Improve retry logic - Create RetryHelper instance only at sendRequest() method [#41](https://github.com/muga/embulk-output-elasticsearch/pull/41)

data/README.md

@@ -26,13 +26,13 @@ This plugin uses HTTP/REST Client and haven't be implemented AWS authentication.
 - **id**: document id column (string, default is null)
 - **bulk_actions**: Sets when to flush a new bulk request based on the number of actions currently added. (int, default is 1000)
 - **bulk_size**: Sets when to flush a new bulk request based on the size of actions currently added. (long, default is 5242880)
+- **fill_null_for_empty_column**: Fill null value when column value is empty (boolean, optional, default is false)
 - ~~**concurrent_requests**: concurrent_requests (int, default is 5)~~  Not used now. May use in the future
 - **maximum_retries** Number of maximam retry times (int, optional, default is 7)
 - **initial_retry_interval_millis** Initial interval between retries in milliseconds (int, optional, default is 1000)
 - **maximum_retry_interval_millis** Maximum interval between retries in milliseconds (int, optional, default is 120000)
 - **timeout_millis** timeout in milliseconds for HTTP client(int, optional, default is 60000)
 - **max_snapshot_waiting_secs** maximam waiting time in second when snapshot is just creating before delete index. works when `mode: replace` (int, optional, default is 1800)
-
 ### Modes
 
 #### insert:
@@ -87,6 +87,7 @@ out:
 
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
+$ ./gradlew bintrayUpload # release embulk-output-elasticsearch to Bintray maven repo
 ```
 
 ## Test

data/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
+    id "com.jfrog.bintray" version "1.7"
+    id "maven-publish"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "jacoco"
@@ -18,21 +19,66 @@ configurations {
     provided
 }
 
-version = "0.4.4"
+group = "org.embulk.output.elasticsearch"
+version = "0.4.5"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.16"
-    provided  "org.embulk:embulk-core:0.8.16"
-    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
+    compile  "org.embulk:embulk-core:0.8.36"
+    provided  "org.embulk:embulk-core:0.8.36"
+    compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.5"
     compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.16:tests"
-    testCompile "org.embulk:embulk-standards:0.8.16"
+    testCompile "org.embulk:embulk-core:0.8.36:tests"
+    testCompile "org.embulk:embulk-standards:0.8.36"
+}
+
+javadoc {
+    options {
+        locale = 'en_US'
+        encoding = 'UTF-8'
+    }
+}
+
+// bintray
+bintray {
+    // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
+    user = project.hasProperty('bintray_user') ? bintray_user : ''
+    key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
+
+    publications = ['bintrayMavenRelease']
+    publish = true
+
+    pkg {
+        userOrg = 'embulk-output-elasticsearch'
+        repo = 'maven'
+        name = project.name
+        desc = 'Elasticsearch output plugin for Embulk'
+        websiteUrl = 'https://github.com/embulk/embulk-output-elasticsearch'
+        issueTrackerUrl = 'https://github.com/embulk/embulk-output-elasticsearch/issues'
+        vcsUrl = 'https://github.com/embulk/embulk-output-elasticsearch.git'
+        licenses = ['Apache 2.0']
+        labels = ['embulk', 'java']
+        publicDownloadNumbers = true
+
+        version {
+            name = project.version
+        }
+    }
+}
+publishing {
+    publications {
+        bintrayMavenRelease(MavenPublication) {
+            from components.java
+            artifact testsJar
+            artifact sourcesJar
+            artifact javadocJar
+        }
+    }
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -42,6 +88,20 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete 'classpath' }
 
+// add tests/javadoc/source jar tasks as artifacts to be released
+task testsJar(type: Jar, dependsOn: classes) {
+    classifier = 'tests'
+    from sourceSets.test.output
+}
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+}
+task javadocJar(type: Jar, dependsOn: javadoc) {
+    classifier = 'javadoc'
+    from javadoc.destinationDir
+}
+
 checkstyle {
     configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
     toolVersion = '6.14.1'
@@ -84,7 +144,7 @@ Gem::Specification.new do |spec|
   spec.description   = %[Elasticsearch output plugin is an Embulk plugin that loads records to Elasticsearch read by any input plugins. Search the input plugins by "embulk-input" keyword.]
   spec.email         = ["muga.nishizawa@gmail.com"]
   spec.licenses      = ["Apache 2.0"]
-  spec.homepage      = "https://github.com/muga/embulk-output-elasticsearch"
+  spec.homepage      = "https://github.com/embulk/embulk-output-elasticsearch"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Wed Jan 13 12:41:02 JST 2016
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,20 +6,38 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,26 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -85,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -150,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPluginDelegate.java

@@ -13,6 +13,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.spi.Exec;
@@ -128,6 +129,10 @@ public class ElasticsearchOutputPluginDelegate
         @Config("time_zone")
         @ConfigDefault("\"UTC\"")
         String getTimeZone();
+
+        @Config("fill_null_for_empty_column")
+        @ConfigDefault("false")
+        boolean getFillNullForEmptyColumn();
     }
 
     public enum Mode
@@ -217,13 +222,23 @@ public class ElasticsearchOutputPluginDelegate
         }
     }
 
+    private static interface FormatterIntlTask extends Task, TimestampFormatter.Task {}
+    private static interface FormatterIntlColumnOption extends Task, TimestampFormatter.TimestampColumnOption {}
+
     @Override  // Overridden from |ServiceRequestMapperBuildable|
     public JacksonServiceRequestMapper buildServiceRequestMapper(PluginTask task)
     {
-        TimestampFormatter formatter = new TimestampFormatter(task.getJRuby(), "%Y-%m-%dT%H:%M:%S.%3N%z", DateTimeZone.forID(task.getTimeZone()));
+        // TODO: Switch to a newer TimestampFormatter constructor after a reasonable interval.
+        // Traditional constructor is used here for compatibility.
+        final ConfigSource configSource = Exec.newConfigSource();
+        configSource.set("format", "%Y-%m-%dT%H:%M:%S.%3N%z");
+        configSource.set("timezone", DateTimeZone.forID(task.getTimeZone()));
+        TimestampFormatter formatter = new TimestampFormatter(
+            Exec.newConfigSource().loadConfig(FormatterIntlTask.class),
+            Optional.fromNullable(configSource.loadConfig(FormatterIntlColumnOption.class)));
 
         return JacksonServiceRequestMapper.builder()
-                .add(new JacksonAllInObjectScope(formatter), new JacksonTopLevelValueLocator("record"))
+                .add(new JacksonAllInObjectScope(formatter, task.getFillNullForEmptyColumn()), new JacksonTopLevelValueLocator("record"))
                 .build();
     }
 

data/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java

@@ -5,6 +5,8 @@ import com.google.common.collect.ImmutableMap;
 import org.embulk.config.ConfigSource;
 import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
 import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
 
 import java.lang.reflect.Method;
 import java.util.Arrays;
@@ -24,6 +26,7 @@ public class ElasticsearchTestUtils
     public static int ES_BULK_SIZE;
     public static int ES_CONCURRENT_REQUESTS;
     public static String PATH_PREFIX;
+    public static String JSON_PATH_PREFIX;
     public static String ES_INDEX2;
     public static String ES_ALIAS;
 
@@ -52,6 +55,7 @@ public class ElasticsearchTestUtils
         ES_NODES = Arrays.asList(ImmutableMap.of("host", ES_HOST, "port", ES_PORT));
 
         PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.csv").getPath();
+        JSON_PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.json").getPath();
     }
 
     public void prepareBeforeTest(PluginTask task) throws Exception
@@ -92,6 +96,24 @@ public class ElasticsearchTestUtils
                 .set("maximum_retries", 2);
     }
 
+    public ConfigSource configJSON()
+    {
+        return Exec.newConfigSource()
+                .set("in", inputConfigJSON())
+                .set("parser", parserConfigJSON())
+                .set("type", "elasticsearch")
+                .set("mode", "insert")
+                .set("nodes", ES_NODES)
+                .set("index", ES_INDEX)
+                .set("index_type", ES_INDEX_TYPE)
+                .set("id", ES_ID)
+                .set("bulk_actions", ES_BULK_ACTIONS)
+                .set("bulk_size", ES_BULK_SIZE)
+                .set("concurrent_requests", ES_CONCURRENT_REQUESTS)
+                .set("maximum_retries", 2)
+                .set("fill_null_for_empty_column", true);
+    }
+
     public ImmutableMap<String, Object> inputConfig()
     {
         ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
@@ -101,6 +123,15 @@ public class ElasticsearchTestUtils
         return builder.build();
     }
 
+    public ImmutableMap<String, Object> inputConfigJSON()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "file");
+        builder.put("path_prefix", JSON_PATH_PREFIX);
+        builder.put("last_path", "");
+        return builder.build();
+    }
+
     public ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
     {
         ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
@@ -117,6 +148,12 @@ public class ElasticsearchTestUtils
         return builder.build();
     }
 
+    public ImmutableMap<String, Object> parserConfigJSON()
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        return builder.build();
+    }
+
     public ImmutableList<Object> schemaConfig()
     {
         ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
@@ -129,4 +166,17 @@ public class ElasticsearchTestUtils
         builder.add(ImmutableMap.of("name", "comment", "type", "string"));
         return builder.build();
     }
+
+    public Schema JSONSchema()
+    {
+        return Schema.builder()
+                .add("id", Types.LONG)
+                .add("account", Types.LONG)
+                .add("time", Types.STRING)
+                .add("purchase", Types.STRING)
+                .add("flg", Types.BOOLEAN)
+                .add("score", Types.DOUBLE)
+                .add("comment", Types.STRING)
+                .build();
+    }
 }

data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java

@@ -0,0 +1,225 @@
+package org.embulk.output.elasticsearch;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.collect.Lists;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
+import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageTestUtils;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
+import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX_TYPE;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+public class TestElasticsearchOutputPluginJSON
+{
+    @BeforeClass
+    public static void initializeConstant()
+    {
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private ElasticsearchOutputPlugin plugin;
+    private ElasticsearchTestUtils utils;
+
+    @Before
+    public void createResources() throws Exception
+    {
+        utils = new ElasticsearchTestUtils();
+        utils.initializeConstant();
+        PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
+        utils.prepareBeforeTest(task);
+
+        plugin = new ElasticsearchOutputPlugin();
+    }
+
+    @Test
+    public void testDefaultValues()
+    {
+        PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
+        assertThat(task.getIndex(), is(ES_INDEX));
+    }
+
+    @Test
+    public void testTransaction()
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        plugin.transaction(config, schema, 0, new OutputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+        // no error happens
+    }
+
+    @Test
+    public void testResume()
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
+        // no error happens
+    }
+
+    @Test
+    public void testOutputByOpen() throws Exception
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+        TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
+
+        List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27",  true, 123.45, "embulk");
+        assertThat(pages.size(), is(1));
+        for (Page page : pages) {
+            output.add(page);
+        }
+
+        output.finish();
+        output.commit();
+        Thread.sleep(1500); // Need to wait until index done
+
+        ElasticsearchHttpClient client = new ElasticsearchHttpClient();
+        Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
+        sendRequest.setAccessible(true);
+        String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
+        String sort = "{\"sort\" : \"id\"}";
+        JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
+        assertThat(response.get("hits").get("total").asInt(), is(1));
+        if (response.size() > 0) {
+            JsonNode record = response.get("hits").get("hits").get(0).get("_source");
+            assertThat(record.get("id").asInt(), is(1));
+            assertThat(record.get("account").asInt(), is(32864));
+            assertThat(record.get("time").asText(), is("2015-01-27 19:23:49"));
+            assertThat(record.get("purchase").asText(), is("2015-01-27"));
+            assertThat(record.get("flg").asBoolean(), is(true));
+            assertThat(record.get("score").asDouble(), is(123.45));
+            assertThat(record.get("comment").asText(), is("embulk"));
+        }
+    }
+
+    @Test
+    public void testOutputByOpenWithNulls() throws Exception
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource)
+            {
+                return Lists.newArrayList(Exec.newTaskReport());
+            }
+        });
+        TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
+
+        List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27",  true, 123.45, "embulk");
+        assertThat(pages.size(), is(1));
+        for (Page page : pages) {
+            output.add(page);
+        }
+
+        output.finish();
+        output.commit();
+        Thread.sleep(1500); // Need to wait until index done
+
+        ElasticsearchHttpClient client = new ElasticsearchHttpClient();
+        Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
+        sendRequest.setAccessible(true);
+        String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
+        String sort = "{\"sort\" : \"id\"}";
+        JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
+        assertThat(response.get("hits").get("total").asInt(), is(1));
+        if (response.size() > 0) {
+            JsonNode record = response.get("hits").get("hits").get(0).get("_source");
+            assertThat(record.get("id").asInt(), is(2));
+            assertTrue(record.get("account").isNull());
+            assertTrue(record.get("time").isNull());
+            assertThat(record.get("purchase").asText(), is("2015-01-27"));
+            assertThat(record.get("flg").asBoolean(), is(true));
+            assertThat(record.get("score").asDouble(), is(123.45));
+            assertThat(record.get("comment").asText(), is("embulk"));
+        }
+    }
+
+    @Test
+    public void testOpenAbort()
+    {
+        ConfigSource config = utils.configJSON();
+        Schema schema = utils.JSONSchema();
+        PluginTask task = config.loadConfig(PluginTask.class);
+        TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
+        output.abort();
+        // no error happens.
+    }
+
+    @Test
+    public void testMode()
+    {
+        assertThat(Mode.values().length, is(2));
+        assertThat(Mode.valueOf("INSERT"), is(Mode.INSERT));
+    }
+
+    @Test
+    public void testAuthMethod()
+    {
+        assertThat(AuthMethod.values().length, is(2));
+        assertThat(AuthMethod.valueOf("BASIC"), is(AuthMethod.BASIC));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testModeThrowsConfigException()
+    {
+        Mode.fromString("non-exists-mode");
+    }
+}

data/src/test/resources/sample_01.json

@@ -0,0 +1,4 @@
+{"id":5, "account":32864, "time":"2015-01-27 19:23:49", "purchase":20150127, "flg": true, "score": 123.45, "comment": "embulk"}
+{"id":6, "account":14824, "time":"2015-01-27 19:01:23", "purchase":20150127, "flg": false, "score": 234.56, "comment": "embulk"}
+{"id":7, "account":55555, "time":"2015-01-28 02:20:02", "purchase":20150128, "flg": true, "score": 678.90, "comment": "embulk"}
+{"id":8, "account":11270, "time":"2015-01-29 11:54:36", "purchase":20150129, "flg": false, "score": 100.00, "comment": "embulk"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.4.5
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-16 00:00:00.000000000 Z
+date: 2017-11-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -81,15 +81,17 @@ files:
 - src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java
 - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
 - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
+- src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java
 - src/test/resources/sample_01.csv
-- classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-output-elasticsearch-0.4.4.jar
+- src/test/resources/sample_01.json
+- classpath/embulk-base-restclient-0.5.5.jar
+- classpath/embulk-output-elasticsearch-0.4.5.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-homepage: https://github.com/muga/embulk-output-elasticsearch
+homepage: https://github.com/embulk/embulk-output-elasticsearch
 licenses:
 - Apache 2.0
 metadata: {}