embulk-output-elasticsearch 0.4.4 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: fee5ac83d6cdd1dd3d0f65f6b95e716f8b38389c
4
- data.tar.gz: 67fc3c6af37486baa0f7dcbf05b189070d9d9e52
3
+ metadata.gz: 660287a649d29f3d92435f6b09f48a5db1a7239c
4
+ data.tar.gz: 8dcc2256309363657afc86d5b3b3bc2fd1dbaae8
5
5
  SHA512:
6
- metadata.gz: 9e6bfc587cbaa9d535b78da022d7575e442a5619462ccca2833afa13b0a67a49a76acf94ae16693f38e5fe9f652d343c28eef489d06606b173fab2722f9926e1
7
- data.tar.gz: 045a4f6338dbb233b6402021b1998fd4a4801d82f05bd4150ad1a5a121636b7c7a9b33ab23d1ab396247ea0e8331180036bfbb2e8ea95165493789ca511add3c
6
+ metadata.gz: 273cbdf5a463a58ac8a4e88bb71b098fe6c08a2365c1b4e98712be703a2875f92401231249817229a9aa0ac22172e3f4e68a1c596a8dd97c509af7890eaa272f
7
+ data.tar.gz: b7e7188312a602ca1cdca21b5dbb713166ee59bd328273b55f20f0ca707555ac2f0c6d54f9679aa3fe580f1c5b8c9ecbbefed7b455398689814c5cd583729bd0
data/.travis.yml CHANGED
@@ -1,3 +1,4 @@
1
+ dist: precise
1
2
  language: java
2
3
 
3
4
  jdk:
@@ -24,3 +25,7 @@ before_script:
24
25
  script:
25
26
  - ./gradlew gem
26
27
  - ./gradlew --info check jacocoTestReport
28
+ addons:
29
+ hosts:
30
+ - example.com
31
+ hostname: example.com
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## 0.4.5 - 2017-11-29
2
+ * [new feature] Add "fill_null_for_empty_column" option and allow insert null value when column is empty [#47](https://github.com/embulk/embulk-output-elasticsearch/pull/47) Thanks! @kfitzgerald
3
+
1
4
  ## 0.4.4 - 2017-06-16
2
5
 
3
6
  * [maintenance] Improve retry logic - Create RetryHelper instance only at sendRequest() method [#41](https://github.com/muga/embulk-output-elasticsearch/pull/41)
data/README.md CHANGED
@@ -26,13 +26,13 @@ This plugin uses HTTP/REST Client and haven't be implemented AWS authentication.
26
26
  - **id**: document id column (string, default is null)
27
27
  - **bulk_actions**: Sets when to flush a new bulk request based on the number of actions currently added. (int, default is 1000)
28
28
  - **bulk_size**: Sets when to flush a new bulk request based on the size of actions currently added. (long, default is 5242880)
29
+ - **fill_null_for_empty_column**: Fill null value when column value is empty (boolean, optional, default is false)
29
30
  - ~~**concurrent_requests**: concurrent_requests (int, default is 5)~~ Not used now. May use in the future
30
31
  - **maximum_retries** Number of maximam retry times (int, optional, default is 7)
31
32
  - **initial_retry_interval_millis** Initial interval between retries in milliseconds (int, optional, default is 1000)
32
33
  - **maximum_retry_interval_millis** Maximum interval between retries in milliseconds (int, optional, default is 120000)
33
34
  - **timeout_millis** timeout in milliseconds for HTTP client(int, optional, default is 60000)
34
35
  - **max_snapshot_waiting_secs** maximam waiting time in second when snapshot is just creating before delete index. works when `mode: replace` (int, optional, default is 1800)
35
-
36
36
  ### Modes
37
37
 
38
38
  #### insert:
@@ -87,6 +87,7 @@ out:
87
87
 
88
88
  ```
89
89
  $ ./gradlew gem # -t to watch change of files and rebuild continuously
90
+ $ ./gradlew bintrayUpload # release embulk-output-elasticsearch to Bintray maven repo
90
91
  ```
91
92
 
92
93
  ## Test
data/build.gradle CHANGED
@@ -1,5 +1,6 @@
1
1
  plugins {
2
- id "com.jfrog.bintray" version "1.1"
2
+ id "com.jfrog.bintray" version "1.7"
3
+ id "maven-publish"
3
4
  id "com.github.jruby-gradle.base" version "0.1.5"
4
5
  id "java"
5
6
  id "jacoco"
@@ -18,21 +19,66 @@ configurations {
18
19
  provided
19
20
  }
20
21
 
21
- version = "0.4.4"
22
+ group = "org.embulk.output.elasticsearch"
23
+ version = "0.4.5"
22
24
 
23
25
  compileJava.options.encoding = 'UTF-8' // source encoding
24
26
  sourceCompatibility = 1.7
25
27
  targetCompatibility = 1.7
26
28
 
27
29
  dependencies {
28
- compile "org.embulk:embulk-core:0.8.16"
29
- provided "org.embulk:embulk-core:0.8.16"
30
- compile "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
30
+ compile "org.embulk:embulk-core:0.8.36"
31
+ provided "org.embulk:embulk-core:0.8.36"
32
+ compile "org.embulk.base.restclient:embulk-base-restclient:0.5.5"
31
33
  compile "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
32
34
 
33
35
  testCompile "junit:junit:4.+"
34
- testCompile "org.embulk:embulk-core:0.8.16:tests"
35
- testCompile "org.embulk:embulk-standards:0.8.16"
36
+ testCompile "org.embulk:embulk-core:0.8.36:tests"
37
+ testCompile "org.embulk:embulk-standards:0.8.36"
38
+ }
39
+
40
+ javadoc {
41
+ options {
42
+ locale = 'en_US'
43
+ encoding = 'UTF-8'
44
+ }
45
+ }
46
+
47
+ // bintray
48
+ bintray {
49
+ // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
50
+ user = project.hasProperty('bintray_user') ? bintray_user : ''
51
+ key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
52
+
53
+ publications = ['bintrayMavenRelease']
54
+ publish = true
55
+
56
+ pkg {
57
+ userOrg = 'embulk-output-elasticsearch'
58
+ repo = 'maven'
59
+ name = project.name
60
+ desc = 'Elasticsearch output plugin for Embulk'
61
+ websiteUrl = 'https://github.com/embulk/embulk-output-elasticsearch'
62
+ issueTrackerUrl = 'https://github.com/embulk/embulk-output-elasticsearch/issues'
63
+ vcsUrl = 'https://github.com/embulk/embulk-output-elasticsearch.git'
64
+ licenses = ['Apache 2.0']
65
+ labels = ['embulk', 'java']
66
+ publicDownloadNumbers = true
67
+
68
+ version {
69
+ name = project.version
70
+ }
71
+ }
72
+ }
73
+ publishing {
74
+ publications {
75
+ bintrayMavenRelease(MavenPublication) {
76
+ from components.java
77
+ artifact testsJar
78
+ artifact sourcesJar
79
+ artifact javadocJar
80
+ }
81
+ }
36
82
  }
37
83
 
38
84
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -42,6 +88,20 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
42
88
  }
43
89
  clean { delete 'classpath' }
44
90
 
91
+ // add tests/javadoc/source jar tasks as artifacts to be released
92
+ task testsJar(type: Jar, dependsOn: classes) {
93
+ classifier = 'tests'
94
+ from sourceSets.test.output
95
+ }
96
+ task sourcesJar(type: Jar, dependsOn: classes) {
97
+ classifier = 'sources'
98
+ from sourceSets.main.allSource
99
+ }
100
+ task javadocJar(type: Jar, dependsOn: javadoc) {
101
+ classifier = 'javadoc'
102
+ from javadoc.destinationDir
103
+ }
104
+
45
105
  checkstyle {
46
106
  configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
47
107
  toolVersion = '6.14.1'
@@ -84,7 +144,7 @@ Gem::Specification.new do |spec|
84
144
  spec.description = %[Elasticsearch output plugin is an Embulk plugin that loads records to Elasticsearch read by any input plugins. Search the input plugins by "embulk-input" keyword.]
85
145
  spec.email = ["muga.nishizawa@gmail.com"]
86
146
  spec.licenses = ["Apache 2.0"]
87
- spec.homepage = "https://github.com/muga/embulk-output-elasticsearch"
147
+ spec.homepage = "https://github.com/embulk/embulk-output-elasticsearch"
88
148
 
89
149
  spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
90
150
  spec.test_files = spec.files.grep(%r"^(test|spec)/")
Binary file
@@ -1,6 +1,5 @@
1
- #Wed Jan 13 12:41:02 JST 2016
2
1
  distributionBase=GRADLE_USER_HOME
3
2
  distributionPath=wrapper/dists
4
3
  zipStoreBase=GRADLE_USER_HOME
5
4
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
5
+ distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
data/gradlew CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bash
1
+ #!/usr/bin/env sh
2
2
 
3
3
  ##############################################################################
4
4
  ##
@@ -6,20 +6,38 @@
6
6
  ##
7
7
  ##############################################################################
8
8
 
9
- # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
10
- DEFAULT_JVM_OPTS=""
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
11
26
 
12
27
  APP_NAME="Gradle"
13
28
  APP_BASE_NAME=`basename "$0"`
14
29
 
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
15
33
  # Use the maximum available, or set MAX_FD != -1 to use that value.
16
34
  MAX_FD="maximum"
17
35
 
18
- warn ( ) {
36
+ warn () {
19
37
  echo "$*"
20
38
  }
21
39
 
22
- die ( ) {
40
+ die () {
23
41
  echo
24
42
  echo "$*"
25
43
  echo
@@ -30,6 +48,7 @@ die ( ) {
30
48
  cygwin=false
31
49
  msys=false
32
50
  darwin=false
51
+ nonstop=false
33
52
  case "`uname`" in
34
53
  CYGWIN* )
35
54
  cygwin=true
@@ -40,26 +59,11 @@ case "`uname`" in
40
59
  MINGW* )
41
60
  msys=true
42
61
  ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
43
65
  esac
44
66
 
45
- # Attempt to set APP_HOME
46
- # Resolve links: $0 may be a link
47
- PRG="$0"
48
- # Need this for relative symlinks.
49
- while [ -h "$PRG" ] ; do
50
- ls=`ls -ld "$PRG"`
51
- link=`expr "$ls" : '.*-> \(.*\)$'`
52
- if expr "$link" : '/.*' > /dev/null; then
53
- PRG="$link"
54
- else
55
- PRG=`dirname "$PRG"`"/$link"
56
- fi
57
- done
58
- SAVED="`pwd`"
59
- cd "`dirname \"$PRG\"`/" >/dev/null
60
- APP_HOME="`pwd -P`"
61
- cd "$SAVED" >/dev/null
62
-
63
67
  CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
64
68
 
65
69
  # Determine the Java command to use to start the JVM.
@@ -85,7 +89,7 @@ location of your Java installation."
85
89
  fi
86
90
 
87
91
  # Increase the maximum file descriptors if we can.
88
- if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
89
93
  MAX_FD_LIMIT=`ulimit -H -n`
90
94
  if [ $? -eq 0 ] ; then
91
95
  if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -150,11 +154,19 @@ if $cygwin ; then
150
154
  esac
151
155
  fi
152
156
 
153
- # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
154
- function splitJvmOpts() {
155
- JVM_OPTS=("$@")
157
+ # Escape application args
158
+ save () {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
156
161
  }
157
- eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
158
- JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
159
171
 
160
- exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
172
+ exec "$JAVACMD" "$@"
@@ -13,6 +13,7 @@ import org.embulk.config.Config;
13
13
  import org.embulk.config.ConfigDefault;
14
14
  import org.embulk.config.ConfigDiff;
15
15
  import org.embulk.config.ConfigException;
16
+ import org.embulk.config.ConfigSource;
16
17
  import org.embulk.config.Task;
17
18
  import org.embulk.config.TaskReport;
18
19
  import org.embulk.spi.Exec;
@@ -128,6 +129,10 @@ public class ElasticsearchOutputPluginDelegate
128
129
  @Config("time_zone")
129
130
  @ConfigDefault("\"UTC\"")
130
131
  String getTimeZone();
132
+
133
+ @Config("fill_null_for_empty_column")
134
+ @ConfigDefault("false")
135
+ boolean getFillNullForEmptyColumn();
131
136
  }
132
137
 
133
138
  public enum Mode
@@ -217,13 +222,23 @@ public class ElasticsearchOutputPluginDelegate
217
222
  }
218
223
  }
219
224
 
225
+ private static interface FormatterIntlTask extends Task, TimestampFormatter.Task {}
226
+ private static interface FormatterIntlColumnOption extends Task, TimestampFormatter.TimestampColumnOption {}
227
+
220
228
  @Override // Overridden from |ServiceRequestMapperBuildable|
221
229
  public JacksonServiceRequestMapper buildServiceRequestMapper(PluginTask task)
222
230
  {
223
- TimestampFormatter formatter = new TimestampFormatter(task.getJRuby(), "%Y-%m-%dT%H:%M:%S.%3N%z", DateTimeZone.forID(task.getTimeZone()));
231
+ // TODO: Switch to a newer TimestampFormatter constructor after a reasonable interval.
232
+ // Traditional constructor is used here for compatibility.
233
+ final ConfigSource configSource = Exec.newConfigSource();
234
+ configSource.set("format", "%Y-%m-%dT%H:%M:%S.%3N%z");
235
+ configSource.set("timezone", DateTimeZone.forID(task.getTimeZone()));
236
+ TimestampFormatter formatter = new TimestampFormatter(
237
+ Exec.newConfigSource().loadConfig(FormatterIntlTask.class),
238
+ Optional.fromNullable(configSource.loadConfig(FormatterIntlColumnOption.class)));
224
239
 
225
240
  return JacksonServiceRequestMapper.builder()
226
- .add(new JacksonAllInObjectScope(formatter), new JacksonTopLevelValueLocator("record"))
241
+ .add(new JacksonAllInObjectScope(formatter, task.getFillNullForEmptyColumn()), new JacksonTopLevelValueLocator("record"))
227
242
  .build();
228
243
  }
229
244
 
@@ -5,6 +5,8 @@ import com.google.common.collect.ImmutableMap;
5
5
  import org.embulk.config.ConfigSource;
6
6
  import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
7
7
  import org.embulk.spi.Exec;
8
+ import org.embulk.spi.Schema;
9
+ import org.embulk.spi.type.Types;
8
10
 
9
11
  import java.lang.reflect.Method;
10
12
  import java.util.Arrays;
@@ -24,6 +26,7 @@ public class ElasticsearchTestUtils
24
26
  public static int ES_BULK_SIZE;
25
27
  public static int ES_CONCURRENT_REQUESTS;
26
28
  public static String PATH_PREFIX;
29
+ public static String JSON_PATH_PREFIX;
27
30
  public static String ES_INDEX2;
28
31
  public static String ES_ALIAS;
29
32
 
@@ -52,6 +55,7 @@ public class ElasticsearchTestUtils
52
55
  ES_NODES = Arrays.asList(ImmutableMap.of("host", ES_HOST, "port", ES_PORT));
53
56
 
54
57
  PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.csv").getPath();
58
+ JSON_PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.json").getPath();
55
59
  }
56
60
 
57
61
  public void prepareBeforeTest(PluginTask task) throws Exception
@@ -92,6 +96,24 @@ public class ElasticsearchTestUtils
92
96
  .set("maximum_retries", 2);
93
97
  }
94
98
 
99
+ public ConfigSource configJSON()
100
+ {
101
+ return Exec.newConfigSource()
102
+ .set("in", inputConfigJSON())
103
+ .set("parser", parserConfigJSON())
104
+ .set("type", "elasticsearch")
105
+ .set("mode", "insert")
106
+ .set("nodes", ES_NODES)
107
+ .set("index", ES_INDEX)
108
+ .set("index_type", ES_INDEX_TYPE)
109
+ .set("id", ES_ID)
110
+ .set("bulk_actions", ES_BULK_ACTIONS)
111
+ .set("bulk_size", ES_BULK_SIZE)
112
+ .set("concurrent_requests", ES_CONCURRENT_REQUESTS)
113
+ .set("maximum_retries", 2)
114
+ .set("fill_null_for_empty_column", true);
115
+ }
116
+
95
117
  public ImmutableMap<String, Object> inputConfig()
96
118
  {
97
119
  ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
@@ -101,6 +123,15 @@ public class ElasticsearchTestUtils
101
123
  return builder.build();
102
124
  }
103
125
 
126
+ public ImmutableMap<String, Object> inputConfigJSON()
127
+ {
128
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
129
+ builder.put("type", "file");
130
+ builder.put("path_prefix", JSON_PATH_PREFIX);
131
+ builder.put("last_path", "");
132
+ return builder.build();
133
+ }
134
+
104
135
  public ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
105
136
  {
106
137
  ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
@@ -117,6 +148,12 @@ public class ElasticsearchTestUtils
117
148
  return builder.build();
118
149
  }
119
150
 
151
+ public ImmutableMap<String, Object> parserConfigJSON()
152
+ {
153
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
154
+ return builder.build();
155
+ }
156
+
120
157
  public ImmutableList<Object> schemaConfig()
121
158
  {
122
159
  ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
@@ -129,4 +166,17 @@ public class ElasticsearchTestUtils
129
166
  builder.add(ImmutableMap.of("name", "comment", "type", "string"));
130
167
  return builder.build();
131
168
  }
169
+
170
+ public Schema JSONSchema()
171
+ {
172
+ return Schema.builder()
173
+ .add("id", Types.LONG)
174
+ .add("account", Types.LONG)
175
+ .add("time", Types.STRING)
176
+ .add("purchase", Types.STRING)
177
+ .add("flg", Types.BOOLEAN)
178
+ .add("score", Types.DOUBLE)
179
+ .add("comment", Types.STRING)
180
+ .build();
181
+ }
132
182
  }
@@ -0,0 +1,225 @@
1
+ package org.embulk.output.elasticsearch;
2
+
3
+ import com.fasterxml.jackson.databind.JsonNode;
4
+ import com.google.common.collect.Lists;
5
+ import org.eclipse.jetty.http.HttpMethod;
6
+ import org.embulk.EmbulkTestRuntime;
7
+ import org.embulk.config.ConfigException;
8
+ import org.embulk.config.ConfigSource;
9
+ import org.embulk.config.TaskReport;
10
+ import org.embulk.config.TaskSource;
11
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
12
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
13
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
14
+ import org.embulk.spi.Exec;
15
+ import org.embulk.spi.OutputPlugin;
16
+ import org.embulk.spi.Page;
17
+ import org.embulk.spi.PageTestUtils;
18
+ import org.embulk.spi.Schema;
19
+ import org.embulk.spi.TransactionalPageOutput;
20
+ import org.junit.Before;
21
+ import org.junit.BeforeClass;
22
+ import org.junit.Rule;
23
+ import org.junit.Test;
24
+
25
+ import java.lang.reflect.Method;
26
+ import java.util.Arrays;
27
+ import java.util.List;
28
+
29
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
30
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX_TYPE;
31
+ import static org.hamcrest.core.Is.is;
32
+ import static org.junit.Assert.assertThat;
33
+ import static org.junit.Assert.assertTrue;
34
+
35
+ public class TestElasticsearchOutputPluginJSON
36
+ {
37
+ @BeforeClass
38
+ public static void initializeConstant()
39
+ {
40
+ }
41
+
42
+ @Rule
43
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
44
+ private ElasticsearchOutputPlugin plugin;
45
+ private ElasticsearchTestUtils utils;
46
+
47
+ @Before
48
+ public void createResources() throws Exception
49
+ {
50
+ utils = new ElasticsearchTestUtils();
51
+ utils.initializeConstant();
52
+ PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
53
+ utils.prepareBeforeTest(task);
54
+
55
+ plugin = new ElasticsearchOutputPlugin();
56
+ }
57
+
58
+ @Test
59
+ public void testDefaultValues()
60
+ {
61
+ PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
62
+ assertThat(task.getIndex(), is(ES_INDEX));
63
+ }
64
+
65
+ @Test
66
+ public void testTransaction()
67
+ {
68
+ ConfigSource config = utils.configJSON();
69
+ Schema schema = utils.JSONSchema();
70
+ plugin.transaction(config, schema, 0, new OutputPlugin.Control()
71
+ {
72
+ @Override
73
+ public List<TaskReport> run(TaskSource taskSource)
74
+ {
75
+ return Lists.newArrayList(Exec.newTaskReport());
76
+ }
77
+ });
78
+ // no error happens
79
+ }
80
+
81
+ @Test
82
+ public void testResume()
83
+ {
84
+ ConfigSource config = utils.configJSON();
85
+ Schema schema = utils.JSONSchema();
86
+ PluginTask task = config.loadConfig(PluginTask.class);
87
+ plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
88
+ {
89
+ @Override
90
+ public List<TaskReport> run(TaskSource taskSource)
91
+ {
92
+ return Lists.newArrayList(Exec.newTaskReport());
93
+ }
94
+ });
95
+ }
96
+
97
+ @Test
98
+ public void testCleanup()
99
+ {
100
+ ConfigSource config = utils.configJSON();
101
+ Schema schema = utils.JSONSchema();
102
+ PluginTask task = config.loadConfig(PluginTask.class);
103
+ plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
104
+ // no error happens
105
+ }
106
+
107
+ @Test
108
+ public void testOutputByOpen() throws Exception
109
+ {
110
+ ConfigSource config = utils.configJSON();
111
+ Schema schema = utils.JSONSchema();
112
+ PluginTask task = config.loadConfig(PluginTask.class);
113
+ plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
114
+ @Override
115
+ public List<TaskReport> run(TaskSource taskSource)
116
+ {
117
+ return Lists.newArrayList(Exec.newTaskReport());
118
+ }
119
+ });
120
+ TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
121
+
122
+ List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
123
+ assertThat(pages.size(), is(1));
124
+ for (Page page : pages) {
125
+ output.add(page);
126
+ }
127
+
128
+ output.finish();
129
+ output.commit();
130
+ Thread.sleep(1500); // Need to wait until index done
131
+
132
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
133
+ Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
134
+ sendRequest.setAccessible(true);
135
+ String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
136
+ String sort = "{\"sort\" : \"id\"}";
137
+ JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
138
+ assertThat(response.get("hits").get("total").asInt(), is(1));
139
+ if (response.size() > 0) {
140
+ JsonNode record = response.get("hits").get("hits").get(0).get("_source");
141
+ assertThat(record.get("id").asInt(), is(1));
142
+ assertThat(record.get("account").asInt(), is(32864));
143
+ assertThat(record.get("time").asText(), is("2015-01-27 19:23:49"));
144
+ assertThat(record.get("purchase").asText(), is("2015-01-27"));
145
+ assertThat(record.get("flg").asBoolean(), is(true));
146
+ assertThat(record.get("score").asDouble(), is(123.45));
147
+ assertThat(record.get("comment").asText(), is("embulk"));
148
+ }
149
+ }
150
+
151
+ @Test
152
+ public void testOutputByOpenWithNulls() throws Exception
153
+ {
154
+ ConfigSource config = utils.configJSON();
155
+ Schema schema = utils.JSONSchema();
156
+ PluginTask task = config.loadConfig(PluginTask.class);
157
+ plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
158
+ @Override
159
+ public List<TaskReport> run(TaskSource taskSource)
160
+ {
161
+ return Lists.newArrayList(Exec.newTaskReport());
162
+ }
163
+ });
164
+ TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
165
+
166
+ List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
167
+ assertThat(pages.size(), is(1));
168
+ for (Page page : pages) {
169
+ output.add(page);
170
+ }
171
+
172
+ output.finish();
173
+ output.commit();
174
+ Thread.sleep(1500); // Need to wait until index done
175
+
176
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
177
+ Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
178
+ sendRequest.setAccessible(true);
179
+ String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
180
+ String sort = "{\"sort\" : \"id\"}";
181
+ JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
182
+ assertThat(response.get("hits").get("total").asInt(), is(1));
183
+ if (response.size() > 0) {
184
+ JsonNode record = response.get("hits").get("hits").get(0).get("_source");
185
+ assertThat(record.get("id").asInt(), is(2));
186
+ assertTrue(record.get("account").isNull());
187
+ assertTrue(record.get("time").isNull());
188
+ assertThat(record.get("purchase").asText(), is("2015-01-27"));
189
+ assertThat(record.get("flg").asBoolean(), is(true));
190
+ assertThat(record.get("score").asDouble(), is(123.45));
191
+ assertThat(record.get("comment").asText(), is("embulk"));
192
+ }
193
+ }
194
+
195
+ @Test
196
+ public void testOpenAbort()
197
+ {
198
+ ConfigSource config = utils.configJSON();
199
+ Schema schema = utils.JSONSchema();
200
+ PluginTask task = config.loadConfig(PluginTask.class);
201
+ TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
202
+ output.abort();
203
+ // no error happens.
204
+ }
205
+
206
+ @Test
207
+ public void testMode()
208
+ {
209
+ assertThat(Mode.values().length, is(2));
210
+ assertThat(Mode.valueOf("INSERT"), is(Mode.INSERT));
211
+ }
212
+
213
+ @Test
214
+ public void testAuthMethod()
215
+ {
216
+ assertThat(AuthMethod.values().length, is(2));
217
+ assertThat(AuthMethod.valueOf("BASIC"), is(AuthMethod.BASIC));
218
+ }
219
+
220
+ @Test(expected = ConfigException.class)
221
+ public void testModeThrowsConfigException()
222
+ {
223
+ Mode.fromString("non-exists-mode");
224
+ }
225
+ }
@@ -0,0 +1,4 @@
1
+ {"id":5, "account":32864, "time":"2015-01-27 19:23:49", "purchase":20150127, "flg": true, "score": 123.45, "comment": "embulk"}
2
+ {"id":6, "account":14824, "time":"2015-01-27 19:01:23", "purchase":20150127, "flg": false, "score": 234.56, "comment": "embulk"}
3
+ {"id":7, "account":55555, "time":"2015-01-28 02:20:02", "purchase":20150128, "flg": true, "score": 678.90, "comment": "embulk"}
4
+ {"id":8, "account":11270, "time":"2015-01-29 11:54:36", "purchase":20150129, "flg": false, "score": 100.00, "comment": "embulk"}
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.4
4
+ version: 0.4.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Muga Nishizawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-06-16 00:00:00.000000000 Z
11
+ date: 2017-11-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -81,15 +81,17 @@ files:
81
81
  - src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java
82
82
  - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
83
83
  - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
84
+ - src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java
84
85
  - src/test/resources/sample_01.csv
85
- - classpath/embulk-base-restclient-0.5.3.jar
86
- - classpath/embulk-output-elasticsearch-0.4.4.jar
86
+ - src/test/resources/sample_01.json
87
+ - classpath/embulk-base-restclient-0.5.5.jar
88
+ - classpath/embulk-output-elasticsearch-0.4.5.jar
87
89
  - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
88
90
  - classpath/jetty-client-9.2.14.v20151106.jar
89
91
  - classpath/jetty-http-9.2.14.v20151106.jar
90
92
  - classpath/jetty-io-9.2.14.v20151106.jar
91
93
  - classpath/jetty-util-9.2.14.v20151106.jar
92
- homepage: https://github.com/muga/embulk-output-elasticsearch
94
+ homepage: https://github.com/embulk/embulk-output-elasticsearch
93
95
  licenses:
94
96
  - Apache 2.0
95
97
  metadata: {}