embulk-output-elasticsearch 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +26 -0
- data/CHANGELOG.md +6 -0
- data/README.md +23 -7
- data/build.gradle +29 -7
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradle/wrapper/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +3 -7
- data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchHttpClient.java +376 -0
- data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPlugin.java +3 -577
- data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchOutputPluginDelegate.java +286 -0
- data/src/main/java/org/embulk/output/elasticsearch/ElasticsearchRecordBuffer.java +99 -0
- data/src/test/java/org/embulk/output/elasticsearch/ElasticsearchTestUtils.java +159 -0
- data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchHttpClient.java +170 -0
- data/src/test/java/org/embulk/output/elasticsearch/TestElasticsearchOutputPlugin.java +70 -326
- metadata +19 -34
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
package org.embulk.output.elasticsearch;
|
|
2
|
+
|
|
3
|
+
import org.eclipse.jetty.http.HttpMethod;
|
|
4
|
+
import org.embulk.EmbulkTestRuntime;
|
|
5
|
+
import org.embulk.config.ConfigException;
|
|
6
|
+
import org.embulk.config.ConfigSource;
|
|
7
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
|
|
8
|
+
import org.embulk.spi.Exec;
|
|
9
|
+
import org.embulk.spi.time.Timestamp;
|
|
10
|
+
import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
|
|
11
|
+
import org.junit.Before;
|
|
12
|
+
import org.junit.BeforeClass;
|
|
13
|
+
import org.junit.Rule;
|
|
14
|
+
import org.junit.Test;
|
|
15
|
+
|
|
16
|
+
import java.lang.reflect.Method;
|
|
17
|
+
import java.text.SimpleDateFormat;
|
|
18
|
+
|
|
19
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_ALIAS;
|
|
20
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
|
|
21
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX2;
|
|
22
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_NODES;
|
|
23
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
|
24
|
+
import static org.hamcrest.core.Is.is;
|
|
25
|
+
|
|
26
|
+
public class TestElasticsearchHttpClient
|
|
27
|
+
{
|
|
28
|
+
@BeforeClass
|
|
29
|
+
public static void initializeConstant()
|
|
30
|
+
{
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
@Before
|
|
34
|
+
public void createResources() throws Exception
|
|
35
|
+
{
|
|
36
|
+
utils = new ElasticsearchTestUtils();
|
|
37
|
+
utils.initializeConstant();
|
|
38
|
+
|
|
39
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
40
|
+
utils.prepareBeforeTest(task);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
@Rule
|
|
44
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
45
|
+
private ElasticsearchTestUtils utils;
|
|
46
|
+
|
|
47
|
+
@Test
|
|
48
|
+
public void testValidateIndexOrAliasName()
|
|
49
|
+
{
|
|
50
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
51
|
+
client.validateIndexOrAliasName("embulk", "index");
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
@Test(expected = ConfigException.class)
|
|
55
|
+
public void testIndexNameContainsUpperCase()
|
|
56
|
+
{
|
|
57
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
58
|
+
client.validateIndexOrAliasName("Embulk", "index");
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
@Test(expected = ConfigException.class)
|
|
62
|
+
public void testIndexNameStartsInvalidChars()
|
|
63
|
+
{
|
|
64
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
65
|
+
client.validateIndexOrAliasName("_embulk", "index");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
@Test(expected = ConfigException.class)
|
|
69
|
+
public void testIndexNameContainsInvalidChars()
|
|
70
|
+
{
|
|
71
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
72
|
+
client.validateIndexOrAliasName("em#bulk", "index");
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
@Test(expected = ConfigException.class)
|
|
76
|
+
public void testIndexNameTooLong()
|
|
77
|
+
{
|
|
78
|
+
String index = "embulk";
|
|
79
|
+
for (int i = 0; i < 255; i++) {
|
|
80
|
+
index += "s";
|
|
81
|
+
}
|
|
82
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
83
|
+
client.validateIndexOrAliasName(index, "index");
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
@Test(expected = ConfigException.class)
|
|
87
|
+
public void testIndexNameEqDot()
|
|
88
|
+
{
|
|
89
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
90
|
+
client.validateIndexOrAliasName(".", "index");
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
@Test
|
|
94
|
+
public void testGenerateNewIndex()
|
|
95
|
+
{
|
|
96
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
97
|
+
String newIndexName = client.generateNewIndexName(ES_INDEX);
|
|
98
|
+
Timestamp time = Exec.getTransactionTime();
|
|
99
|
+
assertThat(newIndexName, is(ES_INDEX + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli())));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
@Test
|
|
103
|
+
public void testCreateAlias() throws Exception
|
|
104
|
+
{
|
|
105
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
106
|
+
try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
|
|
107
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
108
|
+
// delete index
|
|
109
|
+
Method method = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class, Jetty92RetryHelper.class);
|
|
110
|
+
method.setAccessible(true);
|
|
111
|
+
method.invoke(client, "newindex", task, retryHelper);
|
|
112
|
+
|
|
113
|
+
// create index
|
|
114
|
+
Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class);
|
|
115
|
+
sendRequest.setAccessible(true);
|
|
116
|
+
String path = String.format("/%s/", ES_INDEX);
|
|
117
|
+
sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
|
|
118
|
+
|
|
119
|
+
path = String.format("/%s/", ES_INDEX2);
|
|
120
|
+
sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
|
|
121
|
+
|
|
122
|
+
// create alias
|
|
123
|
+
client.reassignAlias(ES_ALIAS, ES_INDEX, task, retryHelper);
|
|
124
|
+
|
|
125
|
+
// check alias
|
|
126
|
+
assertThat(client.isAliasExisting(ES_ALIAS, task, retryHelper), is(true));
|
|
127
|
+
assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX + "]"));
|
|
128
|
+
|
|
129
|
+
// reassign index
|
|
130
|
+
client.reassignAlias(ES_ALIAS, ES_INDEX2, task, retryHelper);
|
|
131
|
+
assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX2 + "]"));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
@Test
|
|
136
|
+
public void testIsIndexExistingWithNonExistsIndex()
|
|
137
|
+
{
|
|
138
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
139
|
+
try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
|
|
140
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
141
|
+
assertThat(client.isIndexExisting("non-existing-index", task, retryHelper), is(false));
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
@Test
|
|
146
|
+
public void testIsAliasExistingWithNonExistsAlias()
|
|
147
|
+
{
|
|
148
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
149
|
+
try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
|
|
150
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
151
|
+
assertThat(client.isAliasExisting("non-existing-alias", task, retryHelper), is(false));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
@Test
|
|
156
|
+
public void testGetAuthorizationHeader() throws Exception
|
|
157
|
+
{
|
|
158
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
159
|
+
|
|
160
|
+
ConfigSource config = Exec.newConfigSource()
|
|
161
|
+
.set("auth_method", "basic")
|
|
162
|
+
.set("user", "username")
|
|
163
|
+
.set("password", "password")
|
|
164
|
+
.set("index", "idx")
|
|
165
|
+
.set("index_type", "idx_type")
|
|
166
|
+
.set("nodes", ES_NODES);
|
|
167
|
+
|
|
168
|
+
assertThat(client.getAuthorizationHeader(config.loadConfig(PluginTask.class)), is("Basic dXNlcm5hbWU6cGFzc3dvcmQ="));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
@@ -1,148 +1,84 @@
|
|
|
1
1
|
package org.embulk.output.elasticsearch;
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import java.io.InputStream;
|
|
5
|
-
import java.io.IOException;
|
|
6
|
-
import java.lang.reflect.InvocationTargetException;
|
|
7
|
-
import java.lang.reflect.Method;
|
|
8
|
-
import java.net.UnknownHostException;
|
|
9
|
-
import java.text.ParseException;
|
|
10
|
-
import java.text.SimpleDateFormat;
|
|
11
|
-
import java.util.Arrays;
|
|
12
|
-
import java.util.List;
|
|
13
|
-
import java.util.Map;
|
|
14
|
-
import java.security.GeneralSecurityException;
|
|
15
|
-
|
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
|
16
4
|
import com.google.common.collect.Lists;
|
|
17
|
-
import
|
|
18
|
-
import com.google.common.collect.ImmutableMap;
|
|
19
|
-
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
|
|
20
|
-
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
|
|
21
|
-
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
|
|
22
|
-
import org.elasticsearch.action.get.GetResponse;
|
|
23
|
-
import org.elasticsearch.client.Client;
|
|
5
|
+
import org.eclipse.jetty.http.HttpMethod;
|
|
24
6
|
import org.embulk.EmbulkTestRuntime;
|
|
25
7
|
import org.embulk.config.ConfigException;
|
|
8
|
+
import org.embulk.config.ConfigSource;
|
|
26
9
|
import org.embulk.config.TaskReport;
|
|
27
10
|
import org.embulk.config.TaskSource;
|
|
28
|
-
import org.embulk.
|
|
11
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
|
|
12
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
|
|
13
|
+
import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
|
|
29
14
|
import org.embulk.spi.Exec;
|
|
30
15
|
import org.embulk.spi.OutputPlugin;
|
|
31
16
|
import org.embulk.spi.Page;
|
|
32
17
|
import org.embulk.spi.PageTestUtils;
|
|
33
18
|
import org.embulk.spi.Schema;
|
|
34
|
-
import org.embulk.spi.time.Timestamp;
|
|
35
19
|
import org.embulk.spi.TransactionalPageOutput;
|
|
36
|
-
import org.embulk.spi.
|
|
20
|
+
import org.embulk.spi.time.Timestamp;
|
|
37
21
|
import org.embulk.standards.CsvParserPlugin;
|
|
38
|
-
import org.embulk.
|
|
22
|
+
import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
|
|
39
23
|
import org.junit.Before;
|
|
40
24
|
import org.junit.BeforeClass;
|
|
41
25
|
import org.junit.Rule;
|
|
42
26
|
import org.junit.Test;
|
|
43
|
-
|
|
27
|
+
|
|
28
|
+
import java.lang.reflect.Method;
|
|
29
|
+
import java.util.Arrays;
|
|
30
|
+
import java.util.List;
|
|
31
|
+
|
|
32
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_BULK_ACTIONS;
|
|
33
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_BULK_SIZE;
|
|
34
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_CONCURRENT_REQUESTS;
|
|
35
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_ID;
|
|
36
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
|
|
37
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX_TYPE;
|
|
38
|
+
import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_NODES;
|
|
39
|
+
import static org.hamcrest.core.Is.is;
|
|
40
|
+
import static org.junit.Assert.assertThat;
|
|
44
41
|
import static org.junit.Assert.assertTrue;
|
|
45
|
-
import static org.junit.Assume.assumeNotNull;
|
|
46
42
|
|
|
47
43
|
public class TestElasticsearchOutputPlugin
|
|
48
44
|
{
|
|
49
|
-
private static String ES_HOST;
|
|
50
|
-
private static int ES_PORT;
|
|
51
|
-
private static List ES_NODES;
|
|
52
|
-
private static String ES_CLUSTER_NAME;
|
|
53
|
-
private static String ES_INDEX;
|
|
54
|
-
private static String ES_INDEX_TYPE;
|
|
55
|
-
private static String ES_ID;
|
|
56
|
-
private static int ES_BULK_ACTIONS;
|
|
57
|
-
private static int ES_BULK_SIZE;
|
|
58
|
-
private static int ES_CONCURRENT_REQUESTS;
|
|
59
|
-
private static String PATH_PREFIX;
|
|
60
|
-
|
|
61
|
-
private MockPageOutput pageOutput;
|
|
62
|
-
|
|
63
|
-
final String ES_TEST_INDEX = "index_for_unittest";
|
|
64
|
-
final String ES_TEST_INDEX2 = "index_for_unittest2";
|
|
65
|
-
final String ES_TEST_ALIAS = "alias_for_unittest";
|
|
66
|
-
|
|
67
|
-
/*
|
|
68
|
-
* This test case requires environment variables
|
|
69
|
-
* ES_HOST
|
|
70
|
-
* ES_INDEX
|
|
71
|
-
* ES_INDEX_TYPE
|
|
72
|
-
*/
|
|
73
45
|
@BeforeClass
|
|
74
46
|
public static void initializeConstant()
|
|
75
47
|
{
|
|
76
|
-
ES_HOST = System.getenv("ES_HOST") != null ? System.getenv("ES_HOST") : "";
|
|
77
|
-
ES_PORT = System.getenv("ES_PORT") != null ? Integer.valueOf(System.getenv("ES_PORT")) : 9300;
|
|
78
|
-
|
|
79
|
-
ES_CLUSTER_NAME = System.getenv("ES_CLUSTER_NAME") != null ? System.getenv("ES_CLUSTER_NAME") : "";
|
|
80
|
-
ES_INDEX = System.getenv("ES_INDEX");
|
|
81
|
-
ES_INDEX_TYPE = System.getenv("ES_INDEX_TYPE");
|
|
82
|
-
ES_ID = "id";
|
|
83
|
-
ES_BULK_ACTIONS = System.getenv("ES_BULK_ACTIONS") != null ? Integer.valueOf(System.getenv("ES_BULK_ACTIONS")) : 1000;
|
|
84
|
-
ES_BULK_SIZE = System.getenv("ES_BULK_SIZE") != null ? Integer.valueOf(System.getenv("ES_BULK_SIZE")) : 5242880;
|
|
85
|
-
ES_CONCURRENT_REQUESTS = System.getenv("ES_CONCURRENT_REQUESTS") != null ? Integer.valueOf(System.getenv("ES_CONCURRENT_REQUESTS")) : 5;
|
|
86
|
-
|
|
87
|
-
assumeNotNull(ES_HOST, ES_INDEX, ES_INDEX_TYPE);
|
|
88
|
-
|
|
89
|
-
ES_NODES = Arrays.asList(ImmutableMap.of("host", ES_HOST, "port", ES_PORT));
|
|
90
|
-
|
|
91
|
-
PATH_PREFIX = ElasticsearchOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
|
|
92
48
|
}
|
|
93
49
|
|
|
94
|
-
|
|
95
50
|
@Rule
|
|
96
51
|
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
97
52
|
private ElasticsearchOutputPlugin plugin;
|
|
53
|
+
private ElasticsearchTestUtils utils;
|
|
98
54
|
|
|
99
55
|
@Before
|
|
100
|
-
public void createResources()
|
|
101
|
-
throws GeneralSecurityException, NoSuchMethodException,
|
|
102
|
-
IllegalAccessException, InvocationTargetException
|
|
56
|
+
public void createResources() throws Exception
|
|
103
57
|
{
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
|
110
|
-
createClient.setAccessible(true);
|
|
111
|
-
try (Client client = (Client) createClient.invoke(plugin, task)) {
|
|
112
|
-
// Delete alias
|
|
113
|
-
if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasAlias(ES_TEST_ALIAS)) {
|
|
114
|
-
client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_ALIAS)).actionGet();
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
// Delete index
|
|
118
|
-
if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX)) {
|
|
119
|
-
client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX)).actionGet();
|
|
120
|
-
}
|
|
58
|
+
utils = new ElasticsearchTestUtils();
|
|
59
|
+
utils.initializeConstant();
|
|
60
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
61
|
+
utils.prepareBeforeTest(task);
|
|
121
62
|
|
|
122
|
-
|
|
123
|
-
client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX2)).actionGet();
|
|
124
|
-
}
|
|
125
|
-
}
|
|
63
|
+
plugin = new ElasticsearchOutputPlugin();
|
|
126
64
|
}
|
|
127
65
|
|
|
128
66
|
@Test
|
|
129
67
|
public void testDefaultValues()
|
|
130
68
|
{
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
assertEquals(ES_INDEX, task.getIndex());
|
|
69
|
+
PluginTask task = utils.config().loadConfig(PluginTask.class);
|
|
70
|
+
assertThat(task.getIndex(), is(ES_INDEX));
|
|
134
71
|
}
|
|
135
72
|
|
|
136
73
|
@Test
|
|
137
74
|
public void testDefaultValuesNull()
|
|
138
75
|
{
|
|
139
76
|
ConfigSource config = Exec.newConfigSource()
|
|
140
|
-
.set("in", inputConfig())
|
|
141
|
-
.set("parser", parserConfig(schemaConfig()))
|
|
77
|
+
.set("in", utils.inputConfig())
|
|
78
|
+
.set("parser", utils.parserConfig(utils.schemaConfig()))
|
|
142
79
|
.set("type", "elasticsearch")
|
|
143
80
|
.set("mode", "") // NULL
|
|
144
81
|
.set("nodes", ES_NODES)
|
|
145
|
-
.set("cluster_name", ES_CLUSTER_NAME)
|
|
146
82
|
.set("index", ES_INDEX)
|
|
147
83
|
.set("index_type", ES_INDEX_TYPE)
|
|
148
84
|
.set("id", ES_ID)
|
|
@@ -160,7 +96,8 @@ public class TestElasticsearchOutputPlugin
|
|
|
160
96
|
return Lists.newArrayList(Exec.newTaskReport());
|
|
161
97
|
}
|
|
162
98
|
});
|
|
163
|
-
}
|
|
99
|
+
}
|
|
100
|
+
catch (Throwable t) {
|
|
164
101
|
if (t instanceof RuntimeException) {
|
|
165
102
|
assertTrue(t.getCause().getCause() instanceof ConfigException);
|
|
166
103
|
}
|
|
@@ -170,7 +107,7 @@ public class TestElasticsearchOutputPlugin
|
|
|
170
107
|
@Test
|
|
171
108
|
public void testTransaction()
|
|
172
109
|
{
|
|
173
|
-
ConfigSource config = config();
|
|
110
|
+
ConfigSource config = utils.config();
|
|
174
111
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
175
112
|
plugin.transaction(config, schema, 0, new OutputPlugin.Control()
|
|
176
113
|
{
|
|
@@ -186,7 +123,7 @@ public class TestElasticsearchOutputPlugin
|
|
|
186
123
|
@Test
|
|
187
124
|
public void testResume()
|
|
188
125
|
{
|
|
189
|
-
ConfigSource config = config();
|
|
126
|
+
ConfigSource config = utils.config();
|
|
190
127
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
191
128
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
192
129
|
plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
|
|
@@ -202,7 +139,7 @@ public class TestElasticsearchOutputPlugin
|
|
|
202
139
|
@Test
|
|
203
140
|
public void testCleanup()
|
|
204
141
|
{
|
|
205
|
-
ConfigSource config = config();
|
|
142
|
+
ConfigSource config = utils.config();
|
|
206
143
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
207
144
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
208
145
|
plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
|
|
@@ -210,44 +147,47 @@ public class TestElasticsearchOutputPlugin
|
|
|
210
147
|
}
|
|
211
148
|
|
|
212
149
|
@Test
|
|
213
|
-
public void testOutputByOpen()
|
|
214
|
-
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
215
|
-
IllegalAccessException, InvocationTargetException, ParseException
|
|
150
|
+
public void testOutputByOpen() throws Exception
|
|
216
151
|
{
|
|
217
|
-
ConfigSource config = config();
|
|
152
|
+
ConfigSource config = utils.config();
|
|
218
153
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
219
154
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
220
155
|
plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
|
|
221
156
|
@Override
|
|
222
|
-
public List<TaskReport> run(TaskSource taskSource)
|
|
157
|
+
public List<TaskReport> run(TaskSource taskSource)
|
|
158
|
+
{
|
|
223
159
|
return Lists.newArrayList(Exec.newTaskReport());
|
|
224
160
|
}
|
|
225
161
|
});
|
|
226
162
|
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
|
227
163
|
|
|
228
164
|
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, Timestamp.ofEpochSecond(1422386629), Timestamp.ofEpochSecond(1422316800), true, 123.45, "embulk");
|
|
229
|
-
|
|
165
|
+
assertThat(pages.size(), is(1));
|
|
230
166
|
for (Page page : pages) {
|
|
231
167
|
output.add(page);
|
|
232
168
|
}
|
|
233
169
|
|
|
234
170
|
output.finish();
|
|
235
171
|
output.commit();
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
172
|
+
Thread.sleep(1500); // Need to wait until index done
|
|
173
|
+
|
|
174
|
+
try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
|
|
175
|
+
ElasticsearchHttpClient client = new ElasticsearchHttpClient();
|
|
176
|
+
Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class, String.class);
|
|
177
|
+
sendRequest.setAccessible(true);
|
|
178
|
+
String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
|
|
179
|
+
String sort = "{\"sort\" : \"id\"}";
|
|
180
|
+
JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, retryHelper, sort);
|
|
181
|
+
assertThat(response.get("hits").get("total").asInt(), is(1));
|
|
182
|
+
if (response.size() > 0) {
|
|
183
|
+
JsonNode record = response.get("hits").get("hits").get(0).get("_source");
|
|
184
|
+
assertThat(record.get("id").asInt(), is(1));
|
|
185
|
+
assertThat(record.get("account").asInt(), is(32864));
|
|
186
|
+
assertThat(record.get("time").asText(), is("2015-01-27T19:23:49.000+0000"));
|
|
187
|
+
assertThat(record.get("purchase").asText(), is("2015-01-27T00:00:00.000+0000"));
|
|
188
|
+
assertThat(record.get("flg").asBoolean(), is(true));
|
|
189
|
+
assertThat(record.get("score").asDouble(), is(123.45));
|
|
190
|
+
assertThat(record.get("comment").asText(), is("embulk"));
|
|
251
191
|
}
|
|
252
192
|
}
|
|
253
193
|
}
|
|
@@ -255,7 +195,7 @@ public class TestElasticsearchOutputPlugin
|
|
|
255
195
|
@Test
|
|
256
196
|
public void testOpenAbort()
|
|
257
197
|
{
|
|
258
|
-
ConfigSource config = config();
|
|
198
|
+
ConfigSource config = utils.config();
|
|
259
199
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
260
200
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
261
201
|
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
|
@@ -264,218 +204,22 @@ public class TestElasticsearchOutputPlugin
|
|
|
264
204
|
}
|
|
265
205
|
|
|
266
206
|
@Test
|
|
267
|
-
public void
|
|
268
|
-
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
269
|
-
IllegalAccessException, InvocationTargetException
|
|
207
|
+
public void testMode()
|
|
270
208
|
{
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
.set("parser", parserConfig(schemaConfig()))
|
|
274
|
-
.set("type", "elasticsearch")
|
|
275
|
-
.set("mode", "replace")
|
|
276
|
-
.set("nodes", Arrays.asList(ImmutableMap.of("host", "unknown-host", "port", 9300)))
|
|
277
|
-
.set("cluster_name", ES_CLUSTER_NAME)
|
|
278
|
-
.set("index", ES_INDEX)
|
|
279
|
-
.set("index_type", ES_INDEX_TYPE)
|
|
280
|
-
.set("id", ES_ID)
|
|
281
|
-
.set("bulk_actions", ES_BULK_ACTIONS)
|
|
282
|
-
.set("bulk_size", ES_BULK_SIZE)
|
|
283
|
-
.set("concurrent_requests", ES_CONCURRENT_REQUESTS
|
|
284
|
-
);
|
|
285
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
286
|
-
|
|
287
|
-
Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
|
288
|
-
createClient.setAccessible(true);
|
|
289
|
-
try (Client client = (Client) createClient.invoke(plugin, task)) {
|
|
290
|
-
} catch (Throwable t) {
|
|
291
|
-
if (t instanceof InvocationTargetException) {
|
|
292
|
-
assertTrue(t.getCause().getCause() instanceof UnknownHostException);
|
|
293
|
-
}
|
|
294
|
-
}
|
|
209
|
+
assertThat(Mode.values().length, is(2));
|
|
210
|
+
assertThat(Mode.valueOf("INSERT"), is(Mode.INSERT));
|
|
295
211
|
}
|
|
296
212
|
|
|
297
213
|
@Test
|
|
298
|
-
public void
|
|
214
|
+
public void testAuthMethod()
|
|
299
215
|
{
|
|
300
|
-
|
|
301
|
-
|
|
216
|
+
assertThat(AuthMethod.values().length, is(2));
|
|
217
|
+
assertThat(AuthMethod.valueOf("BASIC"), is(AuthMethod.BASIC));
|
|
302
218
|
}
|
|
303
219
|
|
|
304
220
|
@Test(expected = ConfigException.class)
|
|
305
221
|
public void testModeThrowsConfigException()
|
|
306
222
|
{
|
|
307
|
-
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
@Test
|
|
311
|
-
public void testDeleteIndex()
|
|
312
|
-
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
313
|
-
IllegalAccessException, InvocationTargetException
|
|
314
|
-
{
|
|
315
|
-
ConfigSource config = config();
|
|
316
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
317
|
-
|
|
318
|
-
Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
|
319
|
-
createClient.setAccessible(true);
|
|
320
|
-
try (Client client = (Client) createClient.invoke(plugin, task)) {
|
|
321
|
-
// Create Index
|
|
322
|
-
client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
|
|
323
|
-
|
|
324
|
-
Method deleteIndex = ElasticsearchOutputPlugin.class.getDeclaredMethod("deleteIndex", String.class, Client.class);
|
|
325
|
-
deleteIndex.setAccessible(true);
|
|
326
|
-
deleteIndex.invoke(plugin, ES_TEST_INDEX, client);
|
|
327
|
-
|
|
328
|
-
assertEquals(false, client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX));
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
@Test
|
|
333
|
-
public void testAlias()
|
|
334
|
-
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
335
|
-
IllegalAccessException, InvocationTargetException
|
|
336
|
-
{
|
|
337
|
-
ConfigSource config = config();
|
|
338
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
339
|
-
|
|
340
|
-
Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
|
341
|
-
createClient.setAccessible(true);
|
|
342
|
-
try (Client client = (Client) createClient.invoke(plugin, task)) {
|
|
343
|
-
|
|
344
|
-
Method isAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("isAlias", String.class, Client.class);
|
|
345
|
-
isAlias.setAccessible(true);
|
|
346
|
-
|
|
347
|
-
Method isExistsAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("isExistsAlias", String.class, Client.class);
|
|
348
|
-
isExistsAlias.setAccessible(true);
|
|
349
|
-
|
|
350
|
-
Method getIndexByAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("getIndexByAlias", String.class, Client.class);
|
|
351
|
-
getIndexByAlias.setAccessible(true);
|
|
352
|
-
|
|
353
|
-
Method reAssignAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("reAssignAlias", String.class, String.class, Client.class);
|
|
354
|
-
reAssignAlias.setAccessible(true);
|
|
355
|
-
|
|
356
|
-
assertEquals(false, isAlias.invoke(plugin, ES_TEST_ALIAS, client));
|
|
357
|
-
assertEquals(false, isExistsAlias.invoke(plugin, ES_TEST_ALIAS, client));
|
|
358
|
-
List<String> indicesBefore = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
|
|
359
|
-
assertEquals(0, indicesBefore.size());
|
|
360
|
-
|
|
361
|
-
// Create Index
|
|
362
|
-
client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
|
|
363
|
-
client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX2)).actionGet();
|
|
364
|
-
// Assign Alias
|
|
365
|
-
reAssignAlias.invoke(plugin, ES_TEST_ALIAS, ES_TEST_INDEX, client);
|
|
366
|
-
|
|
367
|
-
assertEquals(true, isAlias.invoke(plugin, ES_TEST_ALIAS, client));
|
|
368
|
-
assertEquals(true, isExistsAlias.invoke(plugin, ES_TEST_ALIAS, client));
|
|
369
|
-
List<String> indicesAfter = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
|
|
370
|
-
assertEquals(1, indicesAfter.size());
|
|
371
|
-
|
|
372
|
-
// ReAssginAlias
|
|
373
|
-
reAssignAlias.invoke(plugin, ES_TEST_ALIAS, ES_TEST_INDEX2, client);
|
|
374
|
-
List<String> indicesReassign = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
|
|
375
|
-
assertEquals(1, indicesReassign.size());
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
@Test
|
|
380
|
-
public void testIsExistsIndex()
|
|
381
|
-
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
382
|
-
IllegalAccessException, InvocationTargetException
|
|
383
|
-
{
|
|
384
|
-
ConfigSource config = config();
|
|
385
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
386
|
-
|
|
387
|
-
Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
|
388
|
-
createClient.setAccessible(true);
|
|
389
|
-
try (Client client = (Client) createClient.invoke(plugin, task)) {
|
|
390
|
-
Method isExistsIndex = ElasticsearchOutputPlugin.class.getDeclaredMethod("isExistsIndex", String.class, Client.class);
|
|
391
|
-
isExistsIndex.setAccessible(true);
|
|
392
|
-
|
|
393
|
-
// Delete index
|
|
394
|
-
if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX)) {
|
|
395
|
-
client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX)).actionGet();
|
|
396
|
-
}
|
|
397
|
-
assertEquals(false, isExistsIndex.invoke(plugin, ES_TEST_INDEX, client));
|
|
398
|
-
|
|
399
|
-
// Create Index
|
|
400
|
-
client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
|
|
401
|
-
assertEquals(true, isExistsIndex.invoke(plugin, ES_TEST_INDEX, client));
|
|
402
|
-
}
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
@Test
|
|
406
|
-
public void testGenerateNewIndex()
|
|
407
|
-
{
|
|
408
|
-
String newIndexName = plugin.generateNewIndexName(ES_INDEX);
|
|
409
|
-
Timestamp time = Exec.getTransactionTime();
|
|
410
|
-
assertEquals(ES_INDEX + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli()), newIndexName);
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
private byte[] convertInputStreamToByte(InputStream is) throws IOException
|
|
414
|
-
{
|
|
415
|
-
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
|
416
|
-
byte [] buffer = new byte[1024];
|
|
417
|
-
while(true) {
|
|
418
|
-
int len = is.read(buffer);
|
|
419
|
-
if(len < 0) {
|
|
420
|
-
break;
|
|
421
|
-
}
|
|
422
|
-
bo.write(buffer, 0, len);
|
|
423
|
-
}
|
|
424
|
-
return bo.toByteArray();
|
|
425
|
-
}
|
|
426
|
-
|
|
427
|
-
private ConfigSource config()
|
|
428
|
-
{
|
|
429
|
-
return Exec.newConfigSource()
|
|
430
|
-
.set("in", inputConfig())
|
|
431
|
-
.set("parser", parserConfig(schemaConfig()))
|
|
432
|
-
.set("type", "elasticsearch")
|
|
433
|
-
.set("mode", "insert")
|
|
434
|
-
.set("nodes", ES_NODES)
|
|
435
|
-
.set("cluster_name", ES_CLUSTER_NAME)
|
|
436
|
-
.set("index", ES_INDEX)
|
|
437
|
-
.set("index_type", ES_INDEX_TYPE)
|
|
438
|
-
.set("id", ES_ID)
|
|
439
|
-
.set("bulk_actions", ES_BULK_ACTIONS)
|
|
440
|
-
.set("bulk_size", ES_BULK_SIZE)
|
|
441
|
-
.set("concurrent_requests", ES_CONCURRENT_REQUESTS);
|
|
442
|
-
}
|
|
443
|
-
|
|
444
|
-
private ImmutableMap<String, Object> inputConfig()
|
|
445
|
-
{
|
|
446
|
-
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
|
447
|
-
builder.put("type", "file");
|
|
448
|
-
builder.put("path_prefix", PATH_PREFIX);
|
|
449
|
-
builder.put("last_path", "");
|
|
450
|
-
return builder.build();
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
|
|
454
|
-
{
|
|
455
|
-
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
|
456
|
-
builder.put("type", "csv");
|
|
457
|
-
builder.put("newline", "CRLF");
|
|
458
|
-
builder.put("delimiter", ",");
|
|
459
|
-
builder.put("quote", "\"");
|
|
460
|
-
builder.put("escape", "\"");
|
|
461
|
-
builder.put("trim_if_not_quoted", false);
|
|
462
|
-
builder.put("skip_header_lines", 1);
|
|
463
|
-
builder.put("allow_extra_columns", false);
|
|
464
|
-
builder.put("allow_optional_columns", false);
|
|
465
|
-
builder.put("columns", schemaConfig);
|
|
466
|
-
return builder.build();
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
private ImmutableList<Object> schemaConfig()
|
|
470
|
-
{
|
|
471
|
-
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
|
472
|
-
builder.add(ImmutableMap.of("name", "id", "type", "long"));
|
|
473
|
-
builder.add(ImmutableMap.of("name", "account", "type", "long"));
|
|
474
|
-
builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
|
|
475
|
-
builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
|
|
476
|
-
builder.add(ImmutableMap.of("name", "flg", "type", "boolean"));
|
|
477
|
-
builder.add(ImmutableMap.of("name", "score", "type", "double"));
|
|
478
|
-
builder.add(ImmutableMap.of("name", "comment", "type", "string"));
|
|
479
|
-
return builder.build();
|
|
223
|
+
Mode.fromString("non-exists-mode");
|
|
480
224
|
}
|
|
481
225
|
}
|