embulk-output-elasticsearch 0.4.4 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +5 -0
- data/CHANGELOG.md +3 -0
- data/README.md +2 -1
- data/build.gradle +68 -8
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -2
- data/gradlew +42 -30
- data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPluginDelegate.java +17 -2
- data/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java +50 -0
- data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java +225 -0
- data/src/test/resources/sample_01.json +4 -0
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 660287a649d29f3d92435f6b09f48a5db1a7239c
|
4
|
+
data.tar.gz: 8dcc2256309363657afc86d5b3b3bc2fd1dbaae8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 273cbdf5a463a58ac8a4e88bb71b098fe6c08a2365c1b4e98712be703a2875f92401231249817229a9aa0ac22172e3f4e68a1c596a8dd97c509af7890eaa272f
|
7
|
+
data.tar.gz: b7e7188312a602ca1cdca21b5dbb713166ee59bd328273b55f20f0ca707555ac2f0c6d54f9679aa3fe580f1c5b8c9ecbbefed7b455398689814c5cd583729bd0
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
## 0.4.5 - 2017-11-29
|
2
|
+
* [new feature] Add "fill_null_for_empty_column" option and allow insert null value when column is empty [#47](https://github.com/embulk/embulk-output-elasticsearch/pull/47) Thanks! @kfitzgerald
|
3
|
+
|
1
4
|
## 0.4.4 - 2017-06-16
|
2
5
|
|
3
6
|
* [maintenance] Improve retry logic - Create RetryHelper instance only at sendRequest() method [#41](https://github.com/muga/embulk-output-elasticsearch/pull/41)
|
data/README.md
CHANGED
@@ -26,13 +26,13 @@ This plugin uses HTTP/REST Client and haven't be implemented AWS authentication.
|
|
26
26
|
- **id**: document id column (string, default is null)
|
27
27
|
- **bulk_actions**: Sets when to flush a new bulk request based on the number of actions currently added. (int, default is 1000)
|
28
28
|
- **bulk_size**: Sets when to flush a new bulk request based on the size of actions currently added. (long, default is 5242880)
|
29
|
+
- **fill_null_for_empty_column**: Fill null value when column value is empty (boolean, optional, default is false)
|
29
30
|
- ~~**concurrent_requests**: concurrent_requests (int, default is 5)~~ Not used now. May use in the future
|
30
31
|
- **maximum_retries** Number of maximam retry times (int, optional, default is 7)
|
31
32
|
- **initial_retry_interval_millis** Initial interval between retries in milliseconds (int, optional, default is 1000)
|
32
33
|
- **maximum_retry_interval_millis** Maximum interval between retries in milliseconds (int, optional, default is 120000)
|
33
34
|
- **timeout_millis** timeout in milliseconds for HTTP client(int, optional, default is 60000)
|
34
35
|
- **max_snapshot_waiting_secs** maximam waiting time in second when snapshot is just creating before delete index. works when `mode: replace` (int, optional, default is 1800)
|
35
|
-
|
36
36
|
### Modes
|
37
37
|
|
38
38
|
#### insert:
|
@@ -87,6 +87,7 @@ out:
|
|
87
87
|
|
88
88
|
```
|
89
89
|
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
90
|
+
$ ./gradlew bintrayUpload # release embulk-output-elasticsearch to Bintray maven repo
|
90
91
|
```
|
91
92
|
|
92
93
|
## Test
|
data/build.gradle
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
plugins {
|
2
|
-
id "com.jfrog.bintray" version "1.
|
2
|
+
id "com.jfrog.bintray" version "1.7"
|
3
|
+
id "maven-publish"
|
3
4
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
5
|
id "java"
|
5
6
|
id "jacoco"
|
@@ -18,21 +19,66 @@ configurations {
|
|
18
19
|
provided
|
19
20
|
}
|
20
21
|
|
21
|
-
|
22
|
+
group = "org.embulk.output.elasticsearch"
|
23
|
+
version = "0.4.5"
|
22
24
|
|
23
25
|
compileJava.options.encoding = 'UTF-8' // source encoding
|
24
26
|
sourceCompatibility = 1.7
|
25
27
|
targetCompatibility = 1.7
|
26
28
|
|
27
29
|
dependencies {
|
28
|
-
compile "org.embulk:embulk-core:0.8.
|
29
|
-
provided "org.embulk:embulk-core:0.8.
|
30
|
-
compile "org.embulk.base.restclient:embulk-base-restclient:0.5.
|
30
|
+
compile "org.embulk:embulk-core:0.8.36"
|
31
|
+
provided "org.embulk:embulk-core:0.8.36"
|
32
|
+
compile "org.embulk.base.restclient:embulk-base-restclient:0.5.5"
|
31
33
|
compile "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
|
32
34
|
|
33
35
|
testCompile "junit:junit:4.+"
|
34
|
-
testCompile "org.embulk:embulk-core:0.8.
|
35
|
-
testCompile "org.embulk:embulk-standards:0.8.
|
36
|
+
testCompile "org.embulk:embulk-core:0.8.36:tests"
|
37
|
+
testCompile "org.embulk:embulk-standards:0.8.36"
|
38
|
+
}
|
39
|
+
|
40
|
+
javadoc {
|
41
|
+
options {
|
42
|
+
locale = 'en_US'
|
43
|
+
encoding = 'UTF-8'
|
44
|
+
}
|
45
|
+
}
|
46
|
+
|
47
|
+
// bintray
|
48
|
+
bintray {
|
49
|
+
// write at your bintray user name and api key to ~/.gradle/gradle.properties file:
|
50
|
+
user = project.hasProperty('bintray_user') ? bintray_user : ''
|
51
|
+
key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
|
52
|
+
|
53
|
+
publications = ['bintrayMavenRelease']
|
54
|
+
publish = true
|
55
|
+
|
56
|
+
pkg {
|
57
|
+
userOrg = 'embulk-output-elasticsearch'
|
58
|
+
repo = 'maven'
|
59
|
+
name = project.name
|
60
|
+
desc = 'Elasticsearch output plugin for Embulk'
|
61
|
+
websiteUrl = 'https://github.com/embulk/embulk-output-elasticsearch'
|
62
|
+
issueTrackerUrl = 'https://github.com/embulk/embulk-output-elasticsearch/issues'
|
63
|
+
vcsUrl = 'https://github.com/embulk/embulk-output-elasticsearch.git'
|
64
|
+
licenses = ['Apache 2.0']
|
65
|
+
labels = ['embulk', 'java']
|
66
|
+
publicDownloadNumbers = true
|
67
|
+
|
68
|
+
version {
|
69
|
+
name = project.version
|
70
|
+
}
|
71
|
+
}
|
72
|
+
}
|
73
|
+
publishing {
|
74
|
+
publications {
|
75
|
+
bintrayMavenRelease(MavenPublication) {
|
76
|
+
from components.java
|
77
|
+
artifact testsJar
|
78
|
+
artifact sourcesJar
|
79
|
+
artifact javadocJar
|
80
|
+
}
|
81
|
+
}
|
36
82
|
}
|
37
83
|
|
38
84
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
@@ -42,6 +88,20 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
42
88
|
}
|
43
89
|
clean { delete 'classpath' }
|
44
90
|
|
91
|
+
// add tests/javadoc/source jar tasks as artifacts to be released
|
92
|
+
task testsJar(type: Jar, dependsOn: classes) {
|
93
|
+
classifier = 'tests'
|
94
|
+
from sourceSets.test.output
|
95
|
+
}
|
96
|
+
task sourcesJar(type: Jar, dependsOn: classes) {
|
97
|
+
classifier = 'sources'
|
98
|
+
from sourceSets.main.allSource
|
99
|
+
}
|
100
|
+
task javadocJar(type: Jar, dependsOn: javadoc) {
|
101
|
+
classifier = 'javadoc'
|
102
|
+
from javadoc.destinationDir
|
103
|
+
}
|
104
|
+
|
45
105
|
checkstyle {
|
46
106
|
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
47
107
|
toolVersion = '6.14.1'
|
@@ -84,7 +144,7 @@ Gem::Specification.new do |spec|
|
|
84
144
|
spec.description = %[Elasticsearch output plugin is an Embulk plugin that loads records to Elasticsearch read by any input plugins. Search the input plugins by "embulk-input" keyword.]
|
85
145
|
spec.email = ["muga.nishizawa@gmail.com"]
|
86
146
|
spec.licenses = ["Apache 2.0"]
|
87
|
-
spec.homepage = "https://github.com/
|
147
|
+
spec.homepage = "https://github.com/embulk/embulk-output-elasticsearch"
|
88
148
|
|
89
149
|
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
90
150
|
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
Binary file
|
@@ -1,6 +1,5 @@
|
|
1
|
-
#Wed Jan 13 12:41:02 JST 2016
|
2
1
|
distributionBase=GRADLE_USER_HOME
|
3
2
|
distributionPath=wrapper/dists
|
4
3
|
zipStoreBase=GRADLE_USER_HOME
|
5
4
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-
|
5
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
|
data/gradlew
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#!/usr/bin/env
|
1
|
+
#!/usr/bin/env sh
|
2
2
|
|
3
3
|
##############################################################################
|
4
4
|
##
|
@@ -6,20 +6,38 @@
|
|
6
6
|
##
|
7
7
|
##############################################################################
|
8
8
|
|
9
|
-
#
|
10
|
-
|
9
|
+
# Attempt to set APP_HOME
|
10
|
+
# Resolve links: $0 may be a link
|
11
|
+
PRG="$0"
|
12
|
+
# Need this for relative symlinks.
|
13
|
+
while [ -h "$PRG" ] ; do
|
14
|
+
ls=`ls -ld "$PRG"`
|
15
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
16
|
+
if expr "$link" : '/.*' > /dev/null; then
|
17
|
+
PRG="$link"
|
18
|
+
else
|
19
|
+
PRG=`dirname "$PRG"`"/$link"
|
20
|
+
fi
|
21
|
+
done
|
22
|
+
SAVED="`pwd`"
|
23
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
24
|
+
APP_HOME="`pwd -P`"
|
25
|
+
cd "$SAVED" >/dev/null
|
11
26
|
|
12
27
|
APP_NAME="Gradle"
|
13
28
|
APP_BASE_NAME=`basename "$0"`
|
14
29
|
|
30
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
31
|
+
DEFAULT_JVM_OPTS=""
|
32
|
+
|
15
33
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
34
|
MAX_FD="maximum"
|
17
35
|
|
18
|
-
warn (
|
36
|
+
warn () {
|
19
37
|
echo "$*"
|
20
38
|
}
|
21
39
|
|
22
|
-
die (
|
40
|
+
die () {
|
23
41
|
echo
|
24
42
|
echo "$*"
|
25
43
|
echo
|
@@ -30,6 +48,7 @@ die ( ) {
|
|
30
48
|
cygwin=false
|
31
49
|
msys=false
|
32
50
|
darwin=false
|
51
|
+
nonstop=false
|
33
52
|
case "`uname`" in
|
34
53
|
CYGWIN* )
|
35
54
|
cygwin=true
|
@@ -40,26 +59,11 @@ case "`uname`" in
|
|
40
59
|
MINGW* )
|
41
60
|
msys=true
|
42
61
|
;;
|
62
|
+
NONSTOP* )
|
63
|
+
nonstop=true
|
64
|
+
;;
|
43
65
|
esac
|
44
66
|
|
45
|
-
# Attempt to set APP_HOME
|
46
|
-
# Resolve links: $0 may be a link
|
47
|
-
PRG="$0"
|
48
|
-
# Need this for relative symlinks.
|
49
|
-
while [ -h "$PRG" ] ; do
|
50
|
-
ls=`ls -ld "$PRG"`
|
51
|
-
link=`expr "$ls" : '.*-> \(.*\)$'`
|
52
|
-
if expr "$link" : '/.*' > /dev/null; then
|
53
|
-
PRG="$link"
|
54
|
-
else
|
55
|
-
PRG=`dirname "$PRG"`"/$link"
|
56
|
-
fi
|
57
|
-
done
|
58
|
-
SAVED="`pwd`"
|
59
|
-
cd "`dirname \"$PRG\"`/" >/dev/null
|
60
|
-
APP_HOME="`pwd -P`"
|
61
|
-
cd "$SAVED" >/dev/null
|
62
|
-
|
63
67
|
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
64
68
|
|
65
69
|
# Determine the Java command to use to start the JVM.
|
@@ -85,7 +89,7 @@ location of your Java installation."
|
|
85
89
|
fi
|
86
90
|
|
87
91
|
# Increase the maximum file descriptors if we can.
|
88
|
-
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
92
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
89
93
|
MAX_FD_LIMIT=`ulimit -H -n`
|
90
94
|
if [ $? -eq 0 ] ; then
|
91
95
|
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
@@ -150,11 +154,19 @@ if $cygwin ; then
|
|
150
154
|
esac
|
151
155
|
fi
|
152
156
|
|
153
|
-
#
|
154
|
-
|
155
|
-
|
157
|
+
# Escape application args
|
158
|
+
save () {
|
159
|
+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
160
|
+
echo " "
|
156
161
|
}
|
157
|
-
|
158
|
-
|
162
|
+
APP_ARGS=$(save "$@")
|
163
|
+
|
164
|
+
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
165
|
+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
166
|
+
|
167
|
+
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
168
|
+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
169
|
+
cd "$(dirname "$0")"
|
170
|
+
fi
|
159
171
|
|
160
|
-
exec "$JAVACMD" "
|
172
|
+
exec "$JAVACMD" "$@"
|
@@ -13,6 +13,7 @@ import org.embulk.config.Config;
|
|
13
13
|
import org.embulk.config.ConfigDefault;
|
14
14
|
import org.embulk.config.ConfigDiff;
|
15
15
|
import org.embulk.config.ConfigException;
|
16
|
+
import org.embulk.config.ConfigSource;
|
16
17
|
import org.embulk.config.Task;
|
17
18
|
import org.embulk.config.TaskReport;
|
18
19
|
import org.embulk.spi.Exec;
|
@@ -128,6 +129,10 @@ public class ElasticsearchOutputPluginDelegate
|
|
128
129
|
@Config("time_zone")
|
129
130
|
@ConfigDefault("\"UTC\"")
|
130
131
|
String getTimeZone();
|
132
|
+
|
133
|
+
@Config("fill_null_for_empty_column")
|
134
|
+
@ConfigDefault("false")
|
135
|
+
boolean getFillNullForEmptyColumn();
|
131
136
|
}
|
132
137
|
|
133
138
|
public enum Mode
|
@@ -217,13 +222,23 @@ public class ElasticsearchOutputPluginDelegate
|
|
217
222
|
}
|
218
223
|
}
|
219
224
|
|
225
|
+
private static interface FormatterIntlTask extends Task, TimestampFormatter.Task {}
|
226
|
+
private static interface FormatterIntlColumnOption extends Task, TimestampFormatter.TimestampColumnOption {}
|
227
|
+
|
220
228
|
@Override // Overridden from |ServiceRequestMapperBuildable|
|
221
229
|
public JacksonServiceRequestMapper buildServiceRequestMapper(PluginTask task)
|
222
230
|
{
|
223
|
-
|
231
|
+
// TODO: Switch to a newer TimestampFormatter constructor after a reasonable interval.
|
232
|
+
// Traditional constructor is used here for compatibility.
|
233
|
+
final ConfigSource configSource = Exec.newConfigSource();
|
234
|
+
configSource.set("format", "%Y-%m-%dT%H:%M:%S.%3N%z");
|
235
|
+
configSource.set("timezone", DateTimeZone.forID(task.getTimeZone()));
|
236
|
+
TimestampFormatter formatter = new TimestampFormatter(
|
237
|
+
Exec.newConfigSource().loadConfig(FormatterIntlTask.class),
|
238
|
+
Optional.fromNullable(configSource.loadConfig(FormatterIntlColumnOption.class)));
|
224
239
|
|
225
240
|
return JacksonServiceRequestMapper.builder()
|
226
|
-
.add(new JacksonAllInObjectScope(formatter), new JacksonTopLevelValueLocator("record"))
|
241
|
+
.add(new JacksonAllInObjectScope(formatter, task.getFillNullForEmptyColumn()), new JacksonTopLevelValueLocator("record"))
|
227
242
|
.build();
|
228
243
|
}
|
229
244
|
|
@@ -5,6 +5,8 @@ import com.google.common.collect.ImmutableMap;
|
|
5
5
|
import org.embulk.config.ConfigSource;
|
6
6
|
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
|
7
7
|
import org.embulk.spi.Exec;
|
8
|
+
import org.embulk.spi.Schema;
|
9
|
+
import org.embulk.spi.type.Types;
|
8
10
|
|
9
11
|
import java.lang.reflect.Method;
|
10
12
|
import java.util.Arrays;
|
@@ -24,6 +26,7 @@ public class ElasticsearchTestUtils
|
|
24
26
|
public static int ES_BULK_SIZE;
|
25
27
|
public static int ES_CONCURRENT_REQUESTS;
|
26
28
|
public static String PATH_PREFIX;
|
29
|
+
public static String JSON_PATH_PREFIX;
|
27
30
|
public static String ES_INDEX2;
|
28
31
|
public static String ES_ALIAS;
|
29
32
|
|
@@ -52,6 +55,7 @@ public class ElasticsearchTestUtils
|
|
52
55
|
ES_NODES = Arrays.asList(ImmutableMap.of("host", ES_HOST, "port", ES_PORT));
|
53
56
|
|
54
57
|
PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.csv").getPath();
|
58
|
+
JSON_PATH_PREFIX = ElasticsearchTestUtils.class.getClassLoader().getResource("sample_01.json").getPath();
|
55
59
|
}
|
56
60
|
|
57
61
|
public void prepareBeforeTest(PluginTask task) throws Exception
|
@@ -92,6 +96,24 @@ public class ElasticsearchTestUtils
|
|
92
96
|
.set("maximum_retries", 2);
|
93
97
|
}
|
94
98
|
|
99
|
+
public ConfigSource configJSON()
|
100
|
+
{
|
101
|
+
return Exec.newConfigSource()
|
102
|
+
.set("in", inputConfigJSON())
|
103
|
+
.set("parser", parserConfigJSON())
|
104
|
+
.set("type", "elasticsearch")
|
105
|
+
.set("mode", "insert")
|
106
|
+
.set("nodes", ES_NODES)
|
107
|
+
.set("index", ES_INDEX)
|
108
|
+
.set("index_type", ES_INDEX_TYPE)
|
109
|
+
.set("id", ES_ID)
|
110
|
+
.set("bulk_actions", ES_BULK_ACTIONS)
|
111
|
+
.set("bulk_size", ES_BULK_SIZE)
|
112
|
+
.set("concurrent_requests", ES_CONCURRENT_REQUESTS)
|
113
|
+
.set("maximum_retries", 2)
|
114
|
+
.set("fill_null_for_empty_column", true);
|
115
|
+
}
|
116
|
+
|
95
117
|
public ImmutableMap<String, Object> inputConfig()
|
96
118
|
{
|
97
119
|
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
@@ -101,6 +123,15 @@ public class ElasticsearchTestUtils
|
|
101
123
|
return builder.build();
|
102
124
|
}
|
103
125
|
|
126
|
+
public ImmutableMap<String, Object> inputConfigJSON()
|
127
|
+
{
|
128
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
129
|
+
builder.put("type", "file");
|
130
|
+
builder.put("path_prefix", JSON_PATH_PREFIX);
|
131
|
+
builder.put("last_path", "");
|
132
|
+
return builder.build();
|
133
|
+
}
|
134
|
+
|
104
135
|
public ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
|
105
136
|
{
|
106
137
|
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
@@ -117,6 +148,12 @@ public class ElasticsearchTestUtils
|
|
117
148
|
return builder.build();
|
118
149
|
}
|
119
150
|
|
151
|
+
public ImmutableMap<String, Object> parserConfigJSON()
|
152
|
+
{
|
153
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
154
|
+
return builder.build();
|
155
|
+
}
|
156
|
+
|
120
157
|
public ImmutableList<Object> schemaConfig()
|
121
158
|
{
|
122
159
|
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
@@ -129,4 +166,17 @@ public class ElasticsearchTestUtils
|
|
129
166
|
builder.add(ImmutableMap.of("name", "comment", "type", "string"));
|
130
167
|
return builder.build();
|
131
168
|
}
|
169
|
+
|
170
|
+
public Schema JSONSchema()
|
171
|
+
{
|
172
|
+
return Schema.builder()
|
173
|
+
.add("id", Types.LONG)
|
174
|
+
.add("account", Types.LONG)
|
175
|
+
.add("time", Types.STRING)
|
176
|
+
.add("purchase", Types.STRING)
|
177
|
+
.add("flg", Types.BOOLEAN)
|
178
|
+
.add("score", Types.DOUBLE)
|
179
|
+
.add("comment", Types.STRING)
|
180
|
+
.build();
|
181
|
+
}
|
132
182
|
}
|
@@ -0,0 +1,225 @@
|
|
1
|
+
package org.embulk.output.elasticsearch;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
|
+
import com.google.common.collect.Lists;
|
5
|
+
import org.eclipse.jetty.http.HttpMethod;
|
6
|
+
import org.embulk.EmbulkTestRuntime;
|
7
|
+
import org.embulk.config.ConfigException;
|
8
|
+
import org.embulk.config.ConfigSource;
|
9
|
+
import org.embulk.config.TaskReport;
|
10
|
+
import org.embulk.config.TaskSource;
|
11
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
|
12
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
|
13
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
|
14
|
+
import org.embulk.spi.Exec;
|
15
|
+
import org.embulk.spi.OutputPlugin;
|
16
|
+
import org.embulk.spi.Page;
|
17
|
+
import org.embulk.spi.PageTestUtils;
|
18
|
+
import org.embulk.spi.Schema;
|
19
|
+
import org.embulk.spi.TransactionalPageOutput;
|
20
|
+
import org.junit.Before;
|
21
|
+
import org.junit.BeforeClass;
|
22
|
+
import org.junit.Rule;
|
23
|
+
import org.junit.Test;
|
24
|
+
|
25
|
+
import java.lang.reflect.Method;
|
26
|
+
import java.util.Arrays;
|
27
|
+
import java.util.List;
|
28
|
+
|
29
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
|
30
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX_TYPE;
|
31
|
+
import static org.hamcrest.core.Is.is;
|
32
|
+
import static org.junit.Assert.assertThat;
|
33
|
+
import static org.junit.Assert.assertTrue;
|
34
|
+
|
35
|
+
public class TestElasticsearchOutputPluginJSON
|
36
|
+
{
|
37
|
+
@BeforeClass
|
38
|
+
public static void initializeConstant()
|
39
|
+
{
|
40
|
+
}
|
41
|
+
|
42
|
+
@Rule
|
43
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
44
|
+
private ElasticsearchOutputPlugin plugin;
|
45
|
+
private ElasticsearchTestUtils utils;
|
46
|
+
|
47
|
+
@Before
|
48
|
+
public void createResources() throws Exception
|
49
|
+
{
|
50
|
+
utils = new ElasticsearchTestUtils();
|
51
|
+
utils.initializeConstant();
|
52
|
+
PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
|
53
|
+
utils.prepareBeforeTest(task);
|
54
|
+
|
55
|
+
plugin = new ElasticsearchOutputPlugin();
|
56
|
+
}
|
57
|
+
|
58
|
+
@Test
|
59
|
+
public void testDefaultValues()
|
60
|
+
{
|
61
|
+
PluginTask task = utils.configJSON().loadConfig(PluginTask.class);
|
62
|
+
assertThat(task.getIndex(), is(ES_INDEX));
|
63
|
+
}
|
64
|
+
|
65
|
+
@Test
|
66
|
+
public void testTransaction()
|
67
|
+
{
|
68
|
+
ConfigSource config = utils.configJSON();
|
69
|
+
Schema schema = utils.JSONSchema();
|
70
|
+
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
71
|
+
{
|
72
|
+
@Override
|
73
|
+
public List<TaskReport> run(TaskSource taskSource)
|
74
|
+
{
|
75
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
76
|
+
}
|
77
|
+
});
|
78
|
+
// no error happens
|
79
|
+
}
|
80
|
+
|
81
|
+
@Test
|
82
|
+
public void testResume()
|
83
|
+
{
|
84
|
+
ConfigSource config = utils.configJSON();
|
85
|
+
Schema schema = utils.JSONSchema();
|
86
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
87
|
+
plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
|
88
|
+
{
|
89
|
+
@Override
|
90
|
+
public List<TaskReport> run(TaskSource taskSource)
|
91
|
+
{
|
92
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
93
|
+
}
|
94
|
+
});
|
95
|
+
}
|
96
|
+
|
97
|
+
@Test
|
98
|
+
public void testCleanup()
|
99
|
+
{
|
100
|
+
ConfigSource config = utils.configJSON();
|
101
|
+
Schema schema = utils.JSONSchema();
|
102
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
103
|
+
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
|
104
|
+
// no error happens
|
105
|
+
}
|
106
|
+
|
107
|
+
@Test
|
108
|
+
public void testOutputByOpen() throws Exception
|
109
|
+
{
|
110
|
+
ConfigSource config = utils.configJSON();
|
111
|
+
Schema schema = utils.JSONSchema();
|
112
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
113
|
+
plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
|
114
|
+
@Override
|
115
|
+
public List<TaskReport> run(TaskSource taskSource)
|
116
|
+
{
|
117
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
118
|
+
}
|
119
|
+
});
|
120
|
+
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
121
|
+
|
122
|
+
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27 19:23:49", "2015-01-27", true, 123.45, "embulk");
|
123
|
+
assertThat(pages.size(), is(1));
|
124
|
+
for (Page page : pages) {
|
125
|
+
output.add(page);
|
126
|
+
}
|
127
|
+
|
128
|
+
output.finish();
|
129
|
+
output.commit();
|
130
|
+
Thread.sleep(1500); // Need to wait until index done
|
131
|
+
|
132
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
133
|
+
Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
|
134
|
+
sendRequest.setAccessible(true);
|
135
|
+
String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
|
136
|
+
String sort = "{\"sort\" : \"id\"}";
|
137
|
+
JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
|
138
|
+
assertThat(response.get("hits").get("total").asInt(), is(1));
|
139
|
+
if (response.size() > 0) {
|
140
|
+
JsonNode record = response.get("hits").get("hits").get(0).get("_source");
|
141
|
+
assertThat(record.get("id").asInt(), is(1));
|
142
|
+
assertThat(record.get("account").asInt(), is(32864));
|
143
|
+
assertThat(record.get("time").asText(), is("2015-01-27 19:23:49"));
|
144
|
+
assertThat(record.get("purchase").asText(), is("2015-01-27"));
|
145
|
+
assertThat(record.get("flg").asBoolean(), is(true));
|
146
|
+
assertThat(record.get("score").asDouble(), is(123.45));
|
147
|
+
assertThat(record.get("comment").asText(), is("embulk"));
|
148
|
+
}
|
149
|
+
}
|
150
|
+
|
151
|
+
@Test
|
152
|
+
public void testOutputByOpenWithNulls() throws Exception
|
153
|
+
{
|
154
|
+
ConfigSource config = utils.configJSON();
|
155
|
+
Schema schema = utils.JSONSchema();
|
156
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
157
|
+
plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
|
158
|
+
@Override
|
159
|
+
public List<TaskReport> run(TaskSource taskSource)
|
160
|
+
{
|
161
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
162
|
+
}
|
163
|
+
});
|
164
|
+
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
165
|
+
|
166
|
+
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 2L, null, null, "2015-01-27", true, 123.45, "embulk");
|
167
|
+
assertThat(pages.size(), is(1));
|
168
|
+
for (Page page : pages) {
|
169
|
+
output.add(page);
|
170
|
+
}
|
171
|
+
|
172
|
+
output.finish();
|
173
|
+
output.commit();
|
174
|
+
Thread.sleep(1500); // Need to wait until index done
|
175
|
+
|
176
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
177
|
+
Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, String.class);
|
178
|
+
sendRequest.setAccessible(true);
|
179
|
+
String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
|
180
|
+
String sort = "{\"sort\" : \"id\"}";
|
181
|
+
JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, sort);
|
182
|
+
assertThat(response.get("hits").get("total").asInt(), is(1));
|
183
|
+
if (response.size() > 0) {
|
184
|
+
JsonNode record = response.get("hits").get("hits").get(0).get("_source");
|
185
|
+
assertThat(record.get("id").asInt(), is(2));
|
186
|
+
assertTrue(record.get("account").isNull());
|
187
|
+
assertTrue(record.get("time").isNull());
|
188
|
+
assertThat(record.get("purchase").asText(), is("2015-01-27"));
|
189
|
+
assertThat(record.get("flg").asBoolean(), is(true));
|
190
|
+
assertThat(record.get("score").asDouble(), is(123.45));
|
191
|
+
assertThat(record.get("comment").asText(), is("embulk"));
|
192
|
+
}
|
193
|
+
}
|
194
|
+
|
195
|
+
@Test
|
196
|
+
public void testOpenAbort()
|
197
|
+
{
|
198
|
+
ConfigSource config = utils.configJSON();
|
199
|
+
Schema schema = utils.JSONSchema();
|
200
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
201
|
+
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
202
|
+
output.abort();
|
203
|
+
// no error happens.
|
204
|
+
}
|
205
|
+
|
206
|
+
@Test
|
207
|
+
public void testMode()
|
208
|
+
{
|
209
|
+
assertThat(Mode.values().length, is(2));
|
210
|
+
assertThat(Mode.valueOf("INSERT"), is(Mode.INSERT));
|
211
|
+
}
|
212
|
+
|
213
|
+
@Test
|
214
|
+
public void testAuthMethod()
|
215
|
+
{
|
216
|
+
assertThat(AuthMethod.values().length, is(2));
|
217
|
+
assertThat(AuthMethod.valueOf("BASIC"), is(AuthMethod.BASIC));
|
218
|
+
}
|
219
|
+
|
220
|
+
@Test(expected = ConfigException.class)
|
221
|
+
public void testModeThrowsConfigException()
|
222
|
+
{
|
223
|
+
Mode.fromString("non-exists-mode");
|
224
|
+
}
|
225
|
+
}
|
@@ -0,0 +1,4 @@
|
|
1
|
+
{"id":5, "account":32864, "time":"2015-01-27 19:23:49", "purchase":20150127, "flg": true, "score": 123.45, "comment": "embulk"}
|
2
|
+
{"id":6, "account":14824, "time":"2015-01-27 19:01:23", "purchase":20150127, "flg": false, "score": 234.56, "comment": "embulk"}
|
3
|
+
{"id":7, "account":55555, "time":"2015-01-28 02:20:02", "purchase":20150128, "flg": true, "score": 678.90, "comment": "embulk"}
|
4
|
+
{"id":8, "account":11270, "time":"2015-01-29 11:54:36", "purchase":20150129, "flg": false, "score": 100.00, "comment": "embulk"}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-11-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -81,15 +81,17 @@ files:
|
|
81
81
|
- src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java
|
82
82
|
- src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java
|
83
83
|
- src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java
|
84
|
+
- src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPluginJSON.java
|
84
85
|
- src/test/resources/sample_01.csv
|
85
|
-
-
|
86
|
-
- classpath/embulk-
|
86
|
+
- src/test/resources/sample_01.json
|
87
|
+
- classpath/embulk-base-restclient-0.5.5.jar
|
88
|
+
- classpath/embulk-output-elasticsearch-0.4.5.jar
|
87
89
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
88
90
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
89
91
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
90
92
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
91
93
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
92
|
-
homepage: https://github.com/
|
94
|
+
homepage: https://github.com/embulk/embulk-output-elasticsearch
|
93
95
|
licenses:
|
94
96
|
- Apache 2.0
|
95
97
|
metadata: {}
|