embulk-output-elasticsearch 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +8 -4
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/output/ElasticsearchOutputPlugin.java +32 -35
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b72c4cd366c2fa4f119b7c7c51d181363680bbca
|
4
|
+
data.tar.gz: 1261ff1b930b18629e625434c2952a1d63c1a1db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d2acb45304e3a79c05c6a68e83816cb157154ad587bc3032775fb67c876fba120b5a22d39b5314ff80bcd4e4ba33627a2e3586c6e56e9c1a74ff7072dd6efb4a
|
7
|
+
data.tar.gz: 592397a3ca552668f0fb7b31cbed40dcf6600817cd526f17731b6c3e744202e271064086900ac6808dbc750b34a79437e47d9f56cabb2b8dc0fe31c726756860
|
data/README.md
CHANGED
@@ -9,18 +9,22 @@
|
|
9
9
|
|
10
10
|
## Configuration
|
11
11
|
|
12
|
-
- **
|
12
|
+
- **nodes**: list of nodes. nodes are pairs of host and port (list, required)
|
13
13
|
- **index_name**: index name (string, required)
|
14
14
|
- **index_type**: index type (string, required)
|
15
|
+
- **doc_id_column**: document id column (string, default is null)
|
16
|
+
- **bulk_actions**: bulk_actions (int, default is 1000)
|
17
|
+
- **concurrent_requests**: concurrent_requests (int, default is 5)
|
15
18
|
|
16
19
|
## Example
|
17
20
|
|
18
21
|
```yaml
|
19
22
|
out:
|
20
23
|
type: elasticsearch
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
+
node:
|
25
|
+
- {host: localhost, port: 9300}
|
26
|
+
index_name: <index name>
|
27
|
+
index_type: <index type>
|
24
28
|
```
|
25
29
|
|
26
30
|
## Build
|
data/build.gradle
CHANGED
@@ -2,6 +2,7 @@ package org.embulk.output;
|
|
2
2
|
|
3
3
|
import com.google.common.base.Optional;
|
4
4
|
import com.google.common.base.Throwables;
|
5
|
+
import com.google.common.collect.ImmutableList;
|
5
6
|
import com.google.inject.Inject;
|
6
7
|
import org.elasticsearch.action.bulk.BulkItemResponse;
|
7
8
|
import org.elasticsearch.action.bulk.BulkProcessor;
|
@@ -10,8 +11,10 @@ import org.elasticsearch.action.bulk.BulkResponse;
|
|
10
11
|
import org.elasticsearch.action.index.IndexRequest;
|
11
12
|
import org.elasticsearch.client.Client;
|
12
13
|
import org.elasticsearch.client.Requests;
|
14
|
+
import org.elasticsearch.client.transport.TransportClient;
|
13
15
|
import org.elasticsearch.common.settings.ImmutableSettings;
|
14
16
|
import org.elasticsearch.common.settings.Settings;
|
17
|
+
import org.elasticsearch.common.transport.InetSocketTransportAddress;
|
15
18
|
import org.elasticsearch.common.xcontent.XContentBuilder;
|
16
19
|
import org.elasticsearch.common.xcontent.XContentFactory;
|
17
20
|
import org.elasticsearch.node.Node;
|
@@ -40,24 +43,32 @@ import java.util.concurrent.TimeUnit;
|
|
40
43
|
public class ElasticsearchOutputPlugin
|
41
44
|
implements OutputPlugin
|
42
45
|
{
|
46
|
+
public interface NodeAddressTask
|
47
|
+
extends Task
|
48
|
+
{
|
49
|
+
@Config("host")
|
50
|
+
public String getHost();
|
51
|
+
|
52
|
+
@Config("port")
|
53
|
+
@ConfigDefault("9300")
|
54
|
+
public int getPort();
|
55
|
+
}
|
56
|
+
|
43
57
|
public interface RunnerTask
|
44
58
|
extends Task
|
45
59
|
{
|
46
|
-
@Config("
|
47
|
-
|
48
|
-
public String getClusterName();
|
60
|
+
@Config("nodes")
|
61
|
+
public List<NodeAddressTask> getNodes();
|
49
62
|
|
50
63
|
@Config("index_name")
|
51
|
-
@ConfigDefault("embulk")
|
52
64
|
public String getIndex();
|
53
65
|
|
54
66
|
@Config("index_type")
|
55
|
-
@ConfigDefault("embulk")
|
56
67
|
public String getIndexType();
|
57
68
|
|
58
|
-
@Config("
|
69
|
+
@Config("doc_id_column")
|
59
70
|
@ConfigDefault("null")
|
60
|
-
public Optional<String>
|
71
|
+
public Optional<String> getIdColumn();
|
61
72
|
|
62
73
|
@Config("bulk_actions")
|
63
74
|
@ConfigDefault("1000")
|
@@ -83,9 +94,7 @@ public class ElasticsearchOutputPlugin
|
|
83
94
|
{
|
84
95
|
final RunnerTask task = config.loadConfig(RunnerTask.class);
|
85
96
|
|
86
|
-
try (
|
87
|
-
try (Client client = createClient(task, node)) {
|
88
|
-
}
|
97
|
+
try (Client client = createClient(task)) {
|
89
98
|
}
|
90
99
|
|
91
100
|
try {
|
@@ -113,21 +122,18 @@ public class ElasticsearchOutputPlugin
|
|
113
122
|
List<CommitReport> successCommitReports)
|
114
123
|
{ }
|
115
124
|
|
116
|
-
private
|
125
|
+
private Client createClient(final RunnerTask task)
|
117
126
|
{
|
118
127
|
// @see http://www.elasticsearch.org/guide/en/elasticsearch/client/java-api/current/client.html
|
119
128
|
Settings settings = ImmutableSettings.settingsBuilder()
|
120
129
|
.classLoader(Settings.class.getClassLoader())
|
121
130
|
.build();
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
private Client createClient(final RunnerTask task, final Node node)
|
129
|
-
{
|
130
|
-
return node.client();
|
131
|
+
TransportClient client = new TransportClient(settings);
|
132
|
+
List<NodeAddressTask> nodes = task.getNodes();
|
133
|
+
for (NodeAddressTask node : nodes) {
|
134
|
+
client.addTransportAddress(new InetSocketTransportAddress(node.getHost(), node.getPort()));
|
135
|
+
}
|
136
|
+
return client;
|
131
137
|
}
|
132
138
|
|
133
139
|
private BulkProcessor newBulkProcessor(final RunnerTask task, final Client client)
|
@@ -176,10 +182,9 @@ public class ElasticsearchOutputPlugin
|
|
176
182
|
{
|
177
183
|
final RunnerTask task = taskSource.loadTask(RunnerTask.class);
|
178
184
|
|
179
|
-
|
180
|
-
Client client = createClient(task, node);
|
185
|
+
Client client = createClient(task);
|
181
186
|
BulkProcessor bulkProcessor = newBulkProcessor(task, client);
|
182
|
-
ElasticsearchPageOutput pageOutput = new ElasticsearchPageOutput(task,
|
187
|
+
ElasticsearchPageOutput pageOutput = new ElasticsearchPageOutput(task, client, bulkProcessor);
|
183
188
|
pageOutput.open(schema);
|
184
189
|
return pageOutput;
|
185
190
|
}
|
@@ -188,7 +193,6 @@ public class ElasticsearchOutputPlugin
|
|
188
193
|
{
|
189
194
|
private Logger log;
|
190
195
|
|
191
|
-
private Node node;
|
192
196
|
private Client client;
|
193
197
|
private BulkProcessor bulkProcessor;
|
194
198
|
|
@@ -196,20 +200,18 @@ public class ElasticsearchOutputPlugin
|
|
196
200
|
|
197
201
|
private final String index;
|
198
202
|
private final String indexType;
|
199
|
-
private final String
|
203
|
+
private final String docIdColumn;
|
200
204
|
|
201
|
-
ElasticsearchPageOutput(RunnerTask task,
|
202
|
-
BulkProcessor bulkProcessor)
|
205
|
+
ElasticsearchPageOutput(RunnerTask task, Client client, BulkProcessor bulkProcessor)
|
203
206
|
{
|
204
207
|
this.log = Exec.getLogger(getClass());
|
205
208
|
|
206
|
-
this.node = node;
|
207
209
|
this.client = client;
|
208
210
|
this.bulkProcessor = bulkProcessor;
|
209
211
|
|
210
212
|
this.index = task.getIndex();
|
211
213
|
this.indexType = task.getIndexType();
|
212
|
-
this.
|
214
|
+
this.docIdColumn = task.getIdColumn().orNull();
|
213
215
|
}
|
214
216
|
|
215
217
|
void open(final Schema schema)
|
@@ -295,7 +297,7 @@ public class ElasticsearchOutputPlugin
|
|
295
297
|
|
296
298
|
private IndexRequest newIndexRequest()
|
297
299
|
{
|
298
|
-
return Requests.indexRequest(index).type(indexType).id(
|
300
|
+
return Requests.indexRequest(index).type(indexType).id(docIdColumn);
|
299
301
|
}
|
300
302
|
|
301
303
|
@Override
|
@@ -326,11 +328,6 @@ public class ElasticsearchOutputPlugin
|
|
326
328
|
client.close(); // ElasticsearchException
|
327
329
|
client = null;
|
328
330
|
}
|
329
|
-
|
330
|
-
if (node != null) {
|
331
|
-
node.close();
|
332
|
-
node = null;
|
333
|
-
}
|
334
331
|
}
|
335
332
|
|
336
333
|
@Override
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Muga Nishizawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -73,7 +73,7 @@ files:
|
|
73
73
|
- src/main/resources/META-INF/services/org.embulk.spi.Extension
|
74
74
|
- src/test/java/org/embulk/output/TestElasticsearchOutputPlugin.java
|
75
75
|
- classpath/elasticsearch-1.4.2.jar
|
76
|
-
- classpath/embulk-output-elasticsearch-0.1.
|
76
|
+
- classpath/embulk-output-elasticsearch-0.1.2.jar
|
77
77
|
- classpath/lucene-analyzers-common-4.10.2.jar
|
78
78
|
- classpath/lucene-core-4.10.2.jar
|
79
79
|
- classpath/lucene-grouping-4.10.2.jar
|