embulk-output-elasticsearch 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,170 @@
1
+ package org.embulk.output.elasticsearch;
2
+
3
+ import org.eclipse.jetty.http.HttpMethod;
4
+ import org.embulk.EmbulkTestRuntime;
5
+ import org.embulk.config.ConfigException;
6
+ import org.embulk.config.ConfigSource;
7
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
8
+ import org.embulk.spi.Exec;
9
+ import org.embulk.spi.time.Timestamp;
10
+ import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
11
+ import org.junit.Before;
12
+ import org.junit.BeforeClass;
13
+ import org.junit.Rule;
14
+ import org.junit.Test;
15
+
16
+ import java.lang.reflect.Method;
17
+ import java.text.SimpleDateFormat;
18
+
19
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_ALIAS;
20
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
21
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX2;
22
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_NODES;
23
+ import static org.hamcrest.MatcherAssert.assertThat;
24
+ import static org.hamcrest.core.Is.is;
25
+
26
+ public class TestElasticsearchHttpClient
27
+ {
28
+ @BeforeClass
29
+ public static void initializeConstant()
30
+ {
31
+ }
32
+
33
+ @Before
34
+ public void createResources() throws Exception
35
+ {
36
+ utils = new ElasticsearchTestUtils();
37
+ utils.initializeConstant();
38
+
39
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
40
+ utils.prepareBeforeTest(task);
41
+ }
42
+
43
+ @Rule
44
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
45
+ private ElasticsearchTestUtils utils;
46
+
47
+ @Test
48
+ public void testValidateIndexOrAliasName()
49
+ {
50
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
51
+ client.validateIndexOrAliasName("embulk", "index");
52
+ }
53
+
54
+ @Test(expected = ConfigException.class)
55
+ public void testIndexNameContainsUpperCase()
56
+ {
57
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
58
+ client.validateIndexOrAliasName("Embulk", "index");
59
+ }
60
+
61
+ @Test(expected = ConfigException.class)
62
+ public void testIndexNameStartsInvalidChars()
63
+ {
64
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
65
+ client.validateIndexOrAliasName("_embulk", "index");
66
+ }
67
+
68
+ @Test(expected = ConfigException.class)
69
+ public void testIndexNameContainsInvalidChars()
70
+ {
71
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
72
+ client.validateIndexOrAliasName("em#bulk", "index");
73
+ }
74
+
75
+ @Test(expected = ConfigException.class)
76
+ public void testIndexNameTooLong()
77
+ {
78
+ String index = "embulk";
79
+ for (int i = 0; i < 255; i++) {
80
+ index += "s";
81
+ }
82
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
83
+ client.validateIndexOrAliasName(index, "index");
84
+ }
85
+
86
+ @Test(expected = ConfigException.class)
87
+ public void testIndexNameEqDot()
88
+ {
89
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
90
+ client.validateIndexOrAliasName(".", "index");
91
+ }
92
+
93
+ @Test
94
+ public void testGenerateNewIndex()
95
+ {
96
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
97
+ String newIndexName = client.generateNewIndexName(ES_INDEX);
98
+ Timestamp time = Exec.getTransactionTime();
99
+ assertThat(newIndexName, is(ES_INDEX + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli())));
100
+ }
101
+
102
+ @Test
103
+ public void testCreateAlias() throws Exception
104
+ {
105
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
106
+ try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
107
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
108
+ // delete index
109
+ Method method = ElasticsearchHttpClient.class.getDeclaredMethod("deleteIndex", String.class, PluginTask.class, Jetty92RetryHelper.class);
110
+ method.setAccessible(true);
111
+ method.invoke(client, "newindex", task, retryHelper);
112
+
113
+ // create index
114
+ Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class);
115
+ sendRequest.setAccessible(true);
116
+ String path = String.format("/%s/", ES_INDEX);
117
+ sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
118
+
119
+ path = String.format("/%s/", ES_INDEX2);
120
+ sendRequest.invoke(client, path, HttpMethod.PUT, task, retryHelper);
121
+
122
+ // create alias
123
+ client.reassignAlias(ES_ALIAS, ES_INDEX, task, retryHelper);
124
+
125
+ // check alias
126
+ assertThat(client.isAliasExisting(ES_ALIAS, task, retryHelper), is(true));
127
+ assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX + "]"));
128
+
129
+ // reassign index
130
+ client.reassignAlias(ES_ALIAS, ES_INDEX2, task, retryHelper);
131
+ assertThat(client.getIndexByAlias(ES_ALIAS, task, retryHelper).toString(), is("[" + ES_INDEX2 + "]"));
132
+ }
133
+ }
134
+
135
+ @Test
136
+ public void testIsIndexExistingWithNonExistsIndex()
137
+ {
138
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
139
+ try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
140
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
141
+ assertThat(client.isIndexExisting("non-existing-index", task, retryHelper), is(false));
142
+ }
143
+ }
144
+
145
+ @Test
146
+ public void testIsAliasExistingWithNonExistsAlias()
147
+ {
148
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
149
+ try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
150
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
151
+ assertThat(client.isAliasExisting("non-existing-alias", task, retryHelper), is(false));
152
+ }
153
+ }
154
+
155
+ @Test
156
+ public void testGetAuthorizationHeader() throws Exception
157
+ {
158
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
159
+
160
+ ConfigSource config = Exec.newConfigSource()
161
+ .set("auth_method", "basic")
162
+ .set("user", "username")
163
+ .set("password", "password")
164
+ .set("index", "idx")
165
+ .set("index_type", "idx_type")
166
+ .set("nodes", ES_NODES);
167
+
168
+ assertThat(client.getAuthorizationHeader(config.loadConfig(PluginTask.class)), is("Basic dXNlcm5hbWU6cGFzc3dvcmQ="));
169
+ }
170
+ }
@@ -1,148 +1,84 @@
1
1
  package org.embulk.output.elasticsearch;
2
2
 
3
- import java.io.ByteArrayOutputStream;
4
- import java.io.InputStream;
5
- import java.io.IOException;
6
- import java.lang.reflect.InvocationTargetException;
7
- import java.lang.reflect.Method;
8
- import java.net.UnknownHostException;
9
- import java.text.ParseException;
10
- import java.text.SimpleDateFormat;
11
- import java.util.Arrays;
12
- import java.util.List;
13
- import java.util.Map;
14
- import java.security.GeneralSecurityException;
15
-
3
+ import com.fasterxml.jackson.databind.JsonNode;
16
4
  import com.google.common.collect.Lists;
17
- import com.google.common.collect.ImmutableList;
18
- import com.google.common.collect.ImmutableMap;
19
- import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
20
- import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
21
- import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
22
- import org.elasticsearch.action.get.GetResponse;
23
- import org.elasticsearch.client.Client;
5
+ import org.eclipse.jetty.http.HttpMethod;
24
6
  import org.embulk.EmbulkTestRuntime;
25
7
  import org.embulk.config.ConfigException;
8
+ import org.embulk.config.ConfigSource;
26
9
  import org.embulk.config.TaskReport;
27
10
  import org.embulk.config.TaskSource;
28
- import org.embulk.config.ConfigSource;
11
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.AuthMethod;
12
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.Mode;
13
+ import org.embulk.output.elasticsearch.ElasticsearchOutputPluginDelegate.PluginTask;
29
14
  import org.embulk.spi.Exec;
30
15
  import org.embulk.spi.OutputPlugin;
31
16
  import org.embulk.spi.Page;
32
17
  import org.embulk.spi.PageTestUtils;
33
18
  import org.embulk.spi.Schema;
34
- import org.embulk.spi.time.Timestamp;
35
19
  import org.embulk.spi.TransactionalPageOutput;
36
- import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
20
+ import org.embulk.spi.time.Timestamp;
37
21
  import org.embulk.standards.CsvParserPlugin;
38
- import org.embulk.output.elasticsearch.ElasticsearchOutputPlugin.PluginTask;
22
+ import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
39
23
  import org.junit.Before;
40
24
  import org.junit.BeforeClass;
41
25
  import org.junit.Rule;
42
26
  import org.junit.Test;
43
- import static org.junit.Assert.assertEquals;
27
+
28
+ import java.lang.reflect.Method;
29
+ import java.util.Arrays;
30
+ import java.util.List;
31
+
32
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_BULK_ACTIONS;
33
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_BULK_SIZE;
34
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_CONCURRENT_REQUESTS;
35
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_ID;
36
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX;
37
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_INDEX_TYPE;
38
+ import static org.embulk.output.elasticsearch.ElasticsearchTestUtils.ES_NODES;
39
+ import static org.hamcrest.core.Is.is;
40
+ import static org.junit.Assert.assertThat;
44
41
  import static org.junit.Assert.assertTrue;
45
- import static org.junit.Assume.assumeNotNull;
46
42
 
47
43
  public class TestElasticsearchOutputPlugin
48
44
  {
49
- private static String ES_HOST;
50
- private static int ES_PORT;
51
- private static List ES_NODES;
52
- private static String ES_CLUSTER_NAME;
53
- private static String ES_INDEX;
54
- private static String ES_INDEX_TYPE;
55
- private static String ES_ID;
56
- private static int ES_BULK_ACTIONS;
57
- private static int ES_BULK_SIZE;
58
- private static int ES_CONCURRENT_REQUESTS;
59
- private static String PATH_PREFIX;
60
-
61
- private MockPageOutput pageOutput;
62
-
63
- final String ES_TEST_INDEX = "index_for_unittest";
64
- final String ES_TEST_INDEX2 = "index_for_unittest2";
65
- final String ES_TEST_ALIAS = "alias_for_unittest";
66
-
67
- /*
68
- * This test case requires environment variables
69
- * ES_HOST
70
- * ES_INDEX
71
- * ES_INDEX_TYPE
72
- */
73
45
  @BeforeClass
74
46
  public static void initializeConstant()
75
47
  {
76
- ES_HOST = System.getenv("ES_HOST") != null ? System.getenv("ES_HOST") : "";
77
- ES_PORT = System.getenv("ES_PORT") != null ? Integer.valueOf(System.getenv("ES_PORT")) : 9300;
78
-
79
- ES_CLUSTER_NAME = System.getenv("ES_CLUSTER_NAME") != null ? System.getenv("ES_CLUSTER_NAME") : "";
80
- ES_INDEX = System.getenv("ES_INDEX");
81
- ES_INDEX_TYPE = System.getenv("ES_INDEX_TYPE");
82
- ES_ID = "id";
83
- ES_BULK_ACTIONS = System.getenv("ES_BULK_ACTIONS") != null ? Integer.valueOf(System.getenv("ES_BULK_ACTIONS")) : 1000;
84
- ES_BULK_SIZE = System.getenv("ES_BULK_SIZE") != null ? Integer.valueOf(System.getenv("ES_BULK_SIZE")) : 5242880;
85
- ES_CONCURRENT_REQUESTS = System.getenv("ES_CONCURRENT_REQUESTS") != null ? Integer.valueOf(System.getenv("ES_CONCURRENT_REQUESTS")) : 5;
86
-
87
- assumeNotNull(ES_HOST, ES_INDEX, ES_INDEX_TYPE);
88
-
89
- ES_NODES = Arrays.asList(ImmutableMap.of("host", ES_HOST, "port", ES_PORT));
90
-
91
- PATH_PREFIX = ElasticsearchOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
92
48
  }
93
49
 
94
-
95
50
  @Rule
96
51
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
97
52
  private ElasticsearchOutputPlugin plugin;
53
+ private ElasticsearchTestUtils utils;
98
54
 
99
55
  @Before
100
- public void createResources()
101
- throws GeneralSecurityException, NoSuchMethodException,
102
- IllegalAccessException, InvocationTargetException
56
+ public void createResources() throws Exception
103
57
  {
104
- ConfigSource config = config();
105
- plugin = new ElasticsearchOutputPlugin();
106
- PluginTask task = config.loadConfig(PluginTask.class);
107
- pageOutput = new MockPageOutput();
108
-
109
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
110
- createClient.setAccessible(true);
111
- try (Client client = (Client) createClient.invoke(plugin, task)) {
112
- // Delete alias
113
- if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasAlias(ES_TEST_ALIAS)) {
114
- client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_ALIAS)).actionGet();
115
- }
116
-
117
- // Delete index
118
- if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX)) {
119
- client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX)).actionGet();
120
- }
58
+ utils = new ElasticsearchTestUtils();
59
+ utils.initializeConstant();
60
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
61
+ utils.prepareBeforeTest(task);
121
62
 
122
- if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX2)) {
123
- client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX2)).actionGet();
124
- }
125
- }
63
+ plugin = new ElasticsearchOutputPlugin();
126
64
  }
127
65
 
128
66
  @Test
129
67
  public void testDefaultValues()
130
68
  {
131
- ConfigSource config = config();
132
- ElasticsearchOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
133
- assertEquals(ES_INDEX, task.getIndex());
69
+ PluginTask task = utils.config().loadConfig(PluginTask.class);
70
+ assertThat(task.getIndex(), is(ES_INDEX));
134
71
  }
135
72
 
136
73
  @Test
137
74
  public void testDefaultValuesNull()
138
75
  {
139
76
  ConfigSource config = Exec.newConfigSource()
140
- .set("in", inputConfig())
141
- .set("parser", parserConfig(schemaConfig()))
77
+ .set("in", utils.inputConfig())
78
+ .set("parser", utils.parserConfig(utils.schemaConfig()))
142
79
  .set("type", "elasticsearch")
143
80
  .set("mode", "") // NULL
144
81
  .set("nodes", ES_NODES)
145
- .set("cluster_name", ES_CLUSTER_NAME)
146
82
  .set("index", ES_INDEX)
147
83
  .set("index_type", ES_INDEX_TYPE)
148
84
  .set("id", ES_ID)
@@ -160,7 +96,8 @@ public class TestElasticsearchOutputPlugin
160
96
  return Lists.newArrayList(Exec.newTaskReport());
161
97
  }
162
98
  });
163
- } catch (Throwable t) {
99
+ }
100
+ catch (Throwable t) {
164
101
  if (t instanceof RuntimeException) {
165
102
  assertTrue(t.getCause().getCause() instanceof ConfigException);
166
103
  }
@@ -170,7 +107,7 @@ public class TestElasticsearchOutputPlugin
170
107
  @Test
171
108
  public void testTransaction()
172
109
  {
173
- ConfigSource config = config();
110
+ ConfigSource config = utils.config();
174
111
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
175
112
  plugin.transaction(config, schema, 0, new OutputPlugin.Control()
176
113
  {
@@ -186,7 +123,7 @@ public class TestElasticsearchOutputPlugin
186
123
  @Test
187
124
  public void testResume()
188
125
  {
189
- ConfigSource config = config();
126
+ ConfigSource config = utils.config();
190
127
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
191
128
  PluginTask task = config.loadConfig(PluginTask.class);
192
129
  plugin.resume(task.dump(), schema, 0, new OutputPlugin.Control()
@@ -202,7 +139,7 @@ public class TestElasticsearchOutputPlugin
202
139
  @Test
203
140
  public void testCleanup()
204
141
  {
205
- ConfigSource config = config();
142
+ ConfigSource config = utils.config();
206
143
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
207
144
  PluginTask task = config.loadConfig(PluginTask.class);
208
145
  plugin.cleanup(task.dump(), schema, 0, Arrays.asList(Exec.newTaskReport()));
@@ -210,44 +147,47 @@ public class TestElasticsearchOutputPlugin
210
147
  }
211
148
 
212
149
  @Test
213
- public void testOutputByOpen()
214
- throws GeneralSecurityException, IOException, NoSuchMethodException,
215
- IllegalAccessException, InvocationTargetException, ParseException
150
+ public void testOutputByOpen() throws Exception
216
151
  {
217
- ConfigSource config = config();
152
+ ConfigSource config = utils.config();
218
153
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
219
154
  PluginTask task = config.loadConfig(PluginTask.class);
220
155
  plugin.transaction(config, schema, 0, new OutputPlugin.Control() {
221
156
  @Override
222
- public List<TaskReport> run(TaskSource taskSource) {
157
+ public List<TaskReport> run(TaskSource taskSource)
158
+ {
223
159
  return Lists.newArrayList(Exec.newTaskReport());
224
160
  }
225
161
  });
226
162
  TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
227
163
 
228
164
  List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, Timestamp.ofEpochSecond(1422386629), Timestamp.ofEpochSecond(1422316800), true, 123.45, "embulk");
229
- assertEquals(1, pages.size());
165
+ assertThat(pages.size(), is(1));
230
166
  for (Page page : pages) {
231
167
  output.add(page);
232
168
  }
233
169
 
234
170
  output.finish();
235
171
  output.commit();
236
-
237
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
238
- createClient.setAccessible(true);
239
- try (Client client = (Client) createClient.invoke(plugin, task)) {
240
- GetResponse response = client.prepareGet(ES_INDEX, ES_INDEX_TYPE, "1").execute().actionGet();
241
- assertTrue(response.isExists());
242
- if (response.isExists()) {
243
- Map<String, Object> map = response.getSourceAsMap();
244
- assertEquals(1, map.get("id"));
245
- assertEquals(32864, map.get("account"));
246
- assertEquals("2015-01-27T19:23:49.000Z", map.get("time"));
247
- assertEquals("2015-01-27T00:00:00.000Z", map.get("purchase"));
248
- assertEquals(true, map.get("flg"));
249
- assertEquals(123.45, map.get("score"));
250
- assertEquals("embulk", map.get("comment"));
172
+ Thread.sleep(1500); // Need to wait until index done
173
+
174
+ try (Jetty92RetryHelper retryHelper = utils.createRetryHelper()) {
175
+ ElasticsearchHttpClient client = new ElasticsearchHttpClient();
176
+ Method sendRequest = ElasticsearchHttpClient.class.getDeclaredMethod("sendRequest", String.class, HttpMethod.class, PluginTask.class, Jetty92RetryHelper.class, String.class);
177
+ sendRequest.setAccessible(true);
178
+ String path = String.format("/%s/%s/_search", ES_INDEX, ES_INDEX_TYPE);
179
+ String sort = "{\"sort\" : \"id\"}";
180
+ JsonNode response = (JsonNode) sendRequest.invoke(client, path, HttpMethod.POST, task, retryHelper, sort);
181
+ assertThat(response.get("hits").get("total").asInt(), is(1));
182
+ if (response.size() > 0) {
183
+ JsonNode record = response.get("hits").get("hits").get(0).get("_source");
184
+ assertThat(record.get("id").asInt(), is(1));
185
+ assertThat(record.get("account").asInt(), is(32864));
186
+ assertThat(record.get("time").asText(), is("2015-01-27T19:23:49.000+0000"));
187
+ assertThat(record.get("purchase").asText(), is("2015-01-27T00:00:00.000+0000"));
188
+ assertThat(record.get("flg").asBoolean(), is(true));
189
+ assertThat(record.get("score").asDouble(), is(123.45));
190
+ assertThat(record.get("comment").asText(), is("embulk"));
251
191
  }
252
192
  }
253
193
  }
@@ -255,7 +195,7 @@ public class TestElasticsearchOutputPlugin
255
195
  @Test
256
196
  public void testOpenAbort()
257
197
  {
258
- ConfigSource config = config();
198
+ ConfigSource config = utils.config();
259
199
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
260
200
  PluginTask task = config.loadConfig(PluginTask.class);
261
201
  TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
@@ -264,218 +204,22 @@ public class TestElasticsearchOutputPlugin
264
204
  }
265
205
 
266
206
  @Test
267
- public void testCreateClientThrowsException()
268
- throws GeneralSecurityException, IOException, NoSuchMethodException,
269
- IllegalAccessException, InvocationTargetException
207
+ public void testMode()
270
208
  {
271
- ConfigSource config = Exec.newConfigSource()
272
- .set("in", inputConfig())
273
- .set("parser", parserConfig(schemaConfig()))
274
- .set("type", "elasticsearch")
275
- .set("mode", "replace")
276
- .set("nodes", Arrays.asList(ImmutableMap.of("host", "unknown-host", "port", 9300)))
277
- .set("cluster_name", ES_CLUSTER_NAME)
278
- .set("index", ES_INDEX)
279
- .set("index_type", ES_INDEX_TYPE)
280
- .set("id", ES_ID)
281
- .set("bulk_actions", ES_BULK_ACTIONS)
282
- .set("bulk_size", ES_BULK_SIZE)
283
- .set("concurrent_requests", ES_CONCURRENT_REQUESTS
284
- );
285
- PluginTask task = config.loadConfig(PluginTask.class);
286
-
287
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
288
- createClient.setAccessible(true);
289
- try (Client client = (Client) createClient.invoke(plugin, task)) {
290
- } catch (Throwable t) {
291
- if (t instanceof InvocationTargetException) {
292
- assertTrue(t.getCause().getCause() instanceof UnknownHostException);
293
- }
294
- }
209
+ assertThat(Mode.values().length, is(2));
210
+ assertThat(Mode.valueOf("INSERT"), is(Mode.INSERT));
295
211
  }
296
212
 
297
213
  @Test
298
- public void testMode()
214
+ public void testAuthMethod()
299
215
  {
300
- assertEquals(2, ElasticsearchOutputPlugin.Mode.values().length);
301
- assertEquals(ElasticsearchOutputPlugin.Mode.INSERT, ElasticsearchOutputPlugin.Mode.valueOf("INSERT"));
216
+ assertThat(AuthMethod.values().length, is(2));
217
+ assertThat(AuthMethod.valueOf("BASIC"), is(AuthMethod.BASIC));
302
218
  }
303
219
 
304
220
  @Test(expected = ConfigException.class)
305
221
  public void testModeThrowsConfigException()
306
222
  {
307
- ElasticsearchOutputPlugin.Mode.fromString("non-exists-mode");
308
- }
309
-
310
- @Test
311
- public void testDeleteIndex()
312
- throws GeneralSecurityException, IOException, NoSuchMethodException,
313
- IllegalAccessException, InvocationTargetException
314
- {
315
- ConfigSource config = config();
316
- PluginTask task = config.loadConfig(PluginTask.class);
317
-
318
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
319
- createClient.setAccessible(true);
320
- try (Client client = (Client) createClient.invoke(plugin, task)) {
321
- // Create Index
322
- client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
323
-
324
- Method deleteIndex = ElasticsearchOutputPlugin.class.getDeclaredMethod("deleteIndex", String.class, Client.class);
325
- deleteIndex.setAccessible(true);
326
- deleteIndex.invoke(plugin, ES_TEST_INDEX, client);
327
-
328
- assertEquals(false, client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX));
329
- }
330
- }
331
-
332
- @Test
333
- public void testAlias()
334
- throws GeneralSecurityException, IOException, NoSuchMethodException,
335
- IllegalAccessException, InvocationTargetException
336
- {
337
- ConfigSource config = config();
338
- PluginTask task = config.loadConfig(PluginTask.class);
339
-
340
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
341
- createClient.setAccessible(true);
342
- try (Client client = (Client) createClient.invoke(plugin, task)) {
343
-
344
- Method isAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("isAlias", String.class, Client.class);
345
- isAlias.setAccessible(true);
346
-
347
- Method isExistsAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("isExistsAlias", String.class, Client.class);
348
- isExistsAlias.setAccessible(true);
349
-
350
- Method getIndexByAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("getIndexByAlias", String.class, Client.class);
351
- getIndexByAlias.setAccessible(true);
352
-
353
- Method reAssignAlias = ElasticsearchOutputPlugin.class.getDeclaredMethod("reAssignAlias", String.class, String.class, Client.class);
354
- reAssignAlias.setAccessible(true);
355
-
356
- assertEquals(false, isAlias.invoke(plugin, ES_TEST_ALIAS, client));
357
- assertEquals(false, isExistsAlias.invoke(plugin, ES_TEST_ALIAS, client));
358
- List<String> indicesBefore = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
359
- assertEquals(0, indicesBefore.size());
360
-
361
- // Create Index
362
- client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
363
- client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX2)).actionGet();
364
- // Assign Alias
365
- reAssignAlias.invoke(plugin, ES_TEST_ALIAS, ES_TEST_INDEX, client);
366
-
367
- assertEquals(true, isAlias.invoke(plugin, ES_TEST_ALIAS, client));
368
- assertEquals(true, isExistsAlias.invoke(plugin, ES_TEST_ALIAS, client));
369
- List<String> indicesAfter = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
370
- assertEquals(1, indicesAfter.size());
371
-
372
- // ReAssginAlias
373
- reAssignAlias.invoke(plugin, ES_TEST_ALIAS, ES_TEST_INDEX2, client);
374
- List<String> indicesReassign = (List<String>) getIndexByAlias.invoke(plugin, ES_TEST_ALIAS, client);
375
- assertEquals(1, indicesReassign.size());
376
- }
377
- }
378
-
379
- @Test
380
- public void testIsExistsIndex()
381
- throws GeneralSecurityException, IOException, NoSuchMethodException,
382
- IllegalAccessException, InvocationTargetException
383
- {
384
- ConfigSource config = config();
385
- PluginTask task = config.loadConfig(PluginTask.class);
386
-
387
- Method createClient = ElasticsearchOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
388
- createClient.setAccessible(true);
389
- try (Client client = (Client) createClient.invoke(plugin, task)) {
390
- Method isExistsIndex = ElasticsearchOutputPlugin.class.getDeclaredMethod("isExistsIndex", String.class, Client.class);
391
- isExistsIndex.setAccessible(true);
392
-
393
- // Delete index
394
- if (client.admin().cluster().state(new ClusterStateRequest()).actionGet().getState().getMetaData().hasIndex(ES_TEST_INDEX)) {
395
- client.admin().indices().delete(new DeleteIndexRequest(ES_TEST_INDEX)).actionGet();
396
- }
397
- assertEquals(false, isExistsIndex.invoke(plugin, ES_TEST_INDEX, client));
398
-
399
- // Create Index
400
- client.admin().indices().create(new CreateIndexRequest(ES_TEST_INDEX)).actionGet();
401
- assertEquals(true, isExistsIndex.invoke(plugin, ES_TEST_INDEX, client));
402
- }
403
- }
404
-
405
- @Test
406
- public void testGenerateNewIndex()
407
- {
408
- String newIndexName = plugin.generateNewIndexName(ES_INDEX);
409
- Timestamp time = Exec.getTransactionTime();
410
- assertEquals(ES_INDEX + new SimpleDateFormat("_yyyyMMdd-HHmmss").format(time.toEpochMilli()), newIndexName);
411
- }
412
-
413
- private byte[] convertInputStreamToByte(InputStream is) throws IOException
414
- {
415
- ByteArrayOutputStream bo = new ByteArrayOutputStream();
416
- byte [] buffer = new byte[1024];
417
- while(true) {
418
- int len = is.read(buffer);
419
- if(len < 0) {
420
- break;
421
- }
422
- bo.write(buffer, 0, len);
423
- }
424
- return bo.toByteArray();
425
- }
426
-
427
- private ConfigSource config()
428
- {
429
- return Exec.newConfigSource()
430
- .set("in", inputConfig())
431
- .set("parser", parserConfig(schemaConfig()))
432
- .set("type", "elasticsearch")
433
- .set("mode", "insert")
434
- .set("nodes", ES_NODES)
435
- .set("cluster_name", ES_CLUSTER_NAME)
436
- .set("index", ES_INDEX)
437
- .set("index_type", ES_INDEX_TYPE)
438
- .set("id", ES_ID)
439
- .set("bulk_actions", ES_BULK_ACTIONS)
440
- .set("bulk_size", ES_BULK_SIZE)
441
- .set("concurrent_requests", ES_CONCURRENT_REQUESTS);
442
- }
443
-
444
- private ImmutableMap<String, Object> inputConfig()
445
- {
446
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
447
- builder.put("type", "file");
448
- builder.put("path_prefix", PATH_PREFIX);
449
- builder.put("last_path", "");
450
- return builder.build();
451
- }
452
-
453
- private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
454
- {
455
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
456
- builder.put("type", "csv");
457
- builder.put("newline", "CRLF");
458
- builder.put("delimiter", ",");
459
- builder.put("quote", "\"");
460
- builder.put("escape", "\"");
461
- builder.put("trim_if_not_quoted", false);
462
- builder.put("skip_header_lines", 1);
463
- builder.put("allow_extra_columns", false);
464
- builder.put("allow_optional_columns", false);
465
- builder.put("columns", schemaConfig);
466
- return builder.build();
467
- }
468
-
469
- private ImmutableList<Object> schemaConfig()
470
- {
471
- ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
472
- builder.add(ImmutableMap.of("name", "id", "type", "long"));
473
- builder.add(ImmutableMap.of("name", "account", "type", "long"));
474
- builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
475
- builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
476
- builder.add(ImmutableMap.of("name", "flg", "type", "boolean"));
477
- builder.add(ImmutableMap.of("name", "score", "type", "double"));
478
- builder.add(ImmutableMap.of("name", "comment", "type", "string"));
479
- return builder.build();
223
+ Mode.fromString("non-exists-mode");
480
224
  }
481
225
  }