embulk-input-mongodb 0.3.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +48 -0
- data/build.gradle +1 -1
- data/classpath/embulk-input-mongodb-0.4.0.jar +0 -0
- data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java +130 -4
- data/src/main/java/org/embulk/input/mongodb/ValueCodec.java +35 -5
- data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java +167 -1
- metadata +3 -3
- data/classpath/embulk-input-mongodb-0.3.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc232a73722d1157d53b5d4de1374cadd11cd234
|
4
|
+
data.tar.gz: 7792d6114a467b793fcacfaa9a64df7bfe1e5aa3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1af752c57c9fc04d2d65e9665d6c11c739be8c2805c07b6bd65fad7a244685bf78eaf8da6a5a84e59d48e1fd18db671690789ace2d9cd7c6d62d2e1afcb5c390
|
7
|
+
data.tar.gz: 2cb4e589f0e8d524414b6a51135b4931ca91514f811b036c5d1452e2933045a4a86d7c3a615ce31eef85d8c23e66c1ba78d8ad952d9a5dd324e9c15682933038
|
data/README.md
CHANGED
@@ -28,6 +28,8 @@ This plugin only works with embulk >= 0.8.8.
|
|
28
28
|
- **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
|
29
29
|
- **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
|
30
30
|
- **sort**: ordering of results (string, optional)
|
31
|
+
- **incremental_field** list of field name (list, optional, can't use with sort option)
|
32
|
+
- **last_record** (hash, optional) last loaded record for incremental load
|
31
33
|
- **stop_on_invalid_record** Stop bulk load transaction if a document includes invalid record (such as unsupported object type) (boolean, optional, default: false)
|
32
34
|
- **json_column_name**: column name used in outputs (string, optional, default: "json")
|
33
35
|
|
@@ -54,6 +56,52 @@ in:
|
|
54
56
|
sort: '{ "field1": 1 }'
|
55
57
|
```
|
56
58
|
|
59
|
+
### Incremental loading
|
60
|
+
|
61
|
+
```yaml
|
62
|
+
in:
|
63
|
+
type: mongodb
|
64
|
+
uri: mongodb://myuser:mypassword@localhost:27017/my_database
|
65
|
+
collection: "my_collection"
|
66
|
+
query: '{ field1: { $gt: 3 } }'
|
67
|
+
projection: '{ "_id": 1, "field1": 1, "field2": 1 }'
|
68
|
+
incremental_field:
|
69
|
+
- "field2"
|
70
|
+
last_record: {"field2": 13215}
|
71
|
+
```
|
72
|
+
|
73
|
+
Plugin will create new query and sort value.
|
74
|
+
You can't use `incremental_field` option with `sort` option at the same time.
|
75
|
+
|
76
|
+
```
|
77
|
+
query { field1: { $gt: 3 }, field2: { $gt: 13215}}
|
78
|
+
sort {"field2", 1} # field2 ascending
|
79
|
+
```
|
80
|
+
|
81
|
+
You have to specify last_record with special characters when field type is `ObjectId` or `DateTime`.
|
82
|
+
|
83
|
+
```yaml
|
84
|
+
# ObjectId field
|
85
|
+
in:
|
86
|
+
type: mongodb
|
87
|
+
incremental_field:
|
88
|
+
- "_id"
|
89
|
+
last_record: {"_id": {"$oid": "5739b2261c21e58edfe39716"}}
|
90
|
+
|
91
|
+
# DateTime field
|
92
|
+
in:
|
93
|
+
type: mongodb
|
94
|
+
incremental_field:
|
95
|
+
- "time_field"
|
96
|
+
last_record: {"time_field": {"$date": "2015-01-25T13:23:15.000Z"}}
|
97
|
+
```
|
98
|
+
|
99
|
+
#### Run Incremental load
|
100
|
+
|
101
|
+
```
|
102
|
+
$ embulk run /path/to/config.yml -c config-diff.yml
|
103
|
+
```
|
104
|
+
|
57
105
|
### Advanced usage with filter plugins
|
58
106
|
|
59
107
|
```yaml
|
data/build.gradle
CHANGED
Binary file
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.input.mongodb;
|
2
2
|
|
3
|
+
import com.fasterxml.jackson.databind.ObjectMapper;
|
3
4
|
import com.google.common.base.Optional;
|
4
5
|
import com.google.common.base.Throwables;
|
5
6
|
import com.mongodb.MongoClient;
|
@@ -19,6 +20,9 @@ import org.embulk.config.ConfigDiff;
|
|
19
20
|
import org.embulk.config.ConfigException;
|
20
21
|
import org.embulk.config.ConfigInject;
|
21
22
|
import org.embulk.config.ConfigSource;
|
23
|
+
import org.embulk.config.DataSource;
|
24
|
+
import org.embulk.config.DataSourceImpl;
|
25
|
+
import org.embulk.config.ModelManager;
|
22
26
|
import org.embulk.config.Task;
|
23
27
|
import org.embulk.config.TaskReport;
|
24
28
|
import org.embulk.config.TaskSource;
|
@@ -36,8 +40,12 @@ import org.slf4j.Logger;
|
|
36
40
|
|
37
41
|
import javax.validation.constraints.Min;
|
38
42
|
|
43
|
+
import java.io.IOException;
|
39
44
|
import java.net.UnknownHostException;
|
45
|
+
import java.util.HashMap;
|
46
|
+
import java.util.LinkedHashMap;
|
40
47
|
import java.util.List;
|
48
|
+
import java.util.Map;
|
41
49
|
|
42
50
|
public class MongodbInputPlugin
|
43
51
|
implements InputPlugin
|
@@ -63,10 +71,12 @@ public class MongodbInputPlugin
|
|
63
71
|
@Config("query")
|
64
72
|
@ConfigDefault("\"{}\"")
|
65
73
|
String getQuery();
|
74
|
+
void setQuery(String query);
|
66
75
|
|
67
76
|
@Config("sort")
|
68
77
|
@ConfigDefault("\"{}\"")
|
69
78
|
String getSort();
|
79
|
+
void setSort(String sort);
|
70
80
|
|
71
81
|
@Config("id_field_name")
|
72
82
|
@ConfigDefault("\"_id\"")
|
@@ -85,6 +95,14 @@ public class MongodbInputPlugin
|
|
85
95
|
@ConfigDefault("\"record\"")
|
86
96
|
String getJsonColumnName();
|
87
97
|
|
98
|
+
@Config("incremental_field")
|
99
|
+
@ConfigDefault("null")
|
100
|
+
Optional<List<String>> getIncrementalField();
|
101
|
+
|
102
|
+
@Config("last_record")
|
103
|
+
@ConfigDefault("null")
|
104
|
+
Optional<Map<String, Object>> getLastRecord();
|
105
|
+
|
88
106
|
@ConfigInject
|
89
107
|
BufferAllocator getBufferAllocator();
|
90
108
|
}
|
@@ -99,6 +117,13 @@ public class MongodbInputPlugin
|
|
99
117
|
if (task.getFields().isPresent()) {
|
100
118
|
throw new ConfigException("field option was deprecated so setting will be ignored");
|
101
119
|
}
|
120
|
+
if (task.getIncrementalField().isPresent() && !task.getSort().equals("{}")) {
|
121
|
+
throw new ConfigException("both of sort and incremental_load can't be used together");
|
122
|
+
}
|
123
|
+
|
124
|
+
Map<String, String> newCondition = buildIncrementalCondition(task);
|
125
|
+
task.setQuery(newCondition.get("query"));
|
126
|
+
task.setSort(newCondition.get("sort"));
|
102
127
|
|
103
128
|
validateJsonField("projection", task.getProjection());
|
104
129
|
validateJsonField("query", task.getQuery());
|
@@ -120,8 +145,14 @@ public class MongodbInputPlugin
|
|
120
145
|
Schema schema, int taskCount,
|
121
146
|
InputPlugin.Control control)
|
122
147
|
{
|
123
|
-
control.run(taskSource, schema, taskCount);
|
124
|
-
|
148
|
+
List<TaskReport> report = control.run(taskSource, schema, taskCount);
|
149
|
+
|
150
|
+
ConfigDiff configDiff = Exec.newConfigDiff();
|
151
|
+
if (report.size() > 0 && report.get(0).has("last_record")) {
|
152
|
+
configDiff.set("last_record", report.get(0).get(Map.class, "last_record"));
|
153
|
+
}
|
154
|
+
|
155
|
+
return configDiff;
|
125
156
|
}
|
126
157
|
|
127
158
|
@Override
|
@@ -142,13 +173,14 @@ public class MongodbInputPlugin
|
|
142
173
|
PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
|
143
174
|
final Column column = pageBuilder.getSchema().getColumns().get(0);
|
144
175
|
|
176
|
+
ValueCodec valueCodec = new ValueCodec(task.getStopOnInvalidRecord(), task);
|
145
177
|
MongoCollection<Value> collection;
|
146
178
|
try {
|
147
179
|
MongoDatabase db = connect(task);
|
148
180
|
|
149
181
|
CodecRegistry registry = CodecRegistries.fromRegistries(
|
150
182
|
MongoClient.getDefaultCodecRegistry(),
|
151
|
-
CodecRegistries.fromCodecs(
|
183
|
+
CodecRegistries.fromCodecs(valueCodec)
|
152
184
|
);
|
153
185
|
collection = db.getCollection(task.getCollection(), Value.class)
|
154
186
|
.withCodecRegistry(registry);
|
@@ -181,7 +213,44 @@ public class MongodbInputPlugin
|
|
181
213
|
|
182
214
|
pageBuilder.finish();
|
183
215
|
|
184
|
-
|
216
|
+
TaskReport report = Exec.newTaskReport();
|
217
|
+
|
218
|
+
if (valueCodec.getLastRecord() != null) {
|
219
|
+
DataSource lastRecord = new DataSourceImpl(Exec.getInjector().getInstance(ModelManager.class));
|
220
|
+
for (String k : valueCodec.getLastRecord().keySet()) {
|
221
|
+
String value = valueCodec.getLastRecord().get(k).toString();
|
222
|
+
Map<String, String> types = valueCodec.getLastRecordType();
|
223
|
+
HashMap<String, String> innerValue = new HashMap<>();
|
224
|
+
switch(types.get(k)) {
|
225
|
+
case "OBJECT_ID":
|
226
|
+
innerValue.put("$oid", value);
|
227
|
+
lastRecord.set(k, innerValue);
|
228
|
+
break;
|
229
|
+
case "DATE_TIME":
|
230
|
+
innerValue.put("$date", value);
|
231
|
+
lastRecord.set(k, innerValue);
|
232
|
+
break;
|
233
|
+
case "INT32":
|
234
|
+
case "INT64":
|
235
|
+
case "TIMESTAMP":
|
236
|
+
lastRecord.set(k, Integer.valueOf(value));
|
237
|
+
break;
|
238
|
+
case "BOOLEAN":
|
239
|
+
lastRecord.set(k, Boolean.valueOf(value));
|
240
|
+
break;
|
241
|
+
case "DOUBLE":
|
242
|
+
lastRecord.set(k, Double.valueOf(value));
|
243
|
+
break;
|
244
|
+
case "DOCUMENT":
|
245
|
+
case "ARRAY":
|
246
|
+
throw new ConfigException(String.format("Unsupported type '%s' was given for 'last_record' [%s]", types.get(k), value));
|
247
|
+
default:
|
248
|
+
lastRecord.set(k, value);
|
249
|
+
}
|
250
|
+
}
|
251
|
+
report.setNested("last_record", lastRecord);
|
252
|
+
}
|
253
|
+
return report;
|
185
254
|
}
|
186
255
|
|
187
256
|
@Override
|
@@ -201,6 +270,63 @@ public class MongodbInputPlugin
|
|
201
270
|
return db;
|
202
271
|
}
|
203
272
|
|
273
|
+
private Map<String, String> buildIncrementalCondition(PluginTask task)
|
274
|
+
{
|
275
|
+
Map<String, String> result = new HashMap<>();
|
276
|
+
String query = task.getQuery();
|
277
|
+
String sort = task.getSort();
|
278
|
+
result.put("query", query);
|
279
|
+
result.put("sort", sort);
|
280
|
+
|
281
|
+
Optional<List<String>> incrementalField = task.getIncrementalField();
|
282
|
+
Optional<Map<String, Object>> lastRecord = task.getLastRecord();
|
283
|
+
if (!incrementalField.isPresent()) {
|
284
|
+
return result;
|
285
|
+
}
|
286
|
+
|
287
|
+
Map<String, Object> newQuery = new LinkedHashMap<>();
|
288
|
+
Map<String, Integer> newSort = new LinkedHashMap<>();
|
289
|
+
ObjectMapper mapper = new ObjectMapper();
|
290
|
+
|
291
|
+
try {
|
292
|
+
@SuppressWarnings("unchecked")
|
293
|
+
Map<String, Object> queryJson = mapper.readValue(query, Map.class);
|
294
|
+
for (String k : queryJson.keySet()) {
|
295
|
+
newQuery.put(k, queryJson.get(k));
|
296
|
+
}
|
297
|
+
|
298
|
+
if (lastRecord.isPresent()) {
|
299
|
+
for (String k : lastRecord.get().keySet()) {
|
300
|
+
Map<String, Object> v = new HashMap<>();
|
301
|
+
Object record = lastRecord.get().get(k);
|
302
|
+
|
303
|
+
if (newQuery.containsKey(k)) {
|
304
|
+
throw new ConfigException("Field declaration was duplicated between 'incremental_field' and 'query' options");
|
305
|
+
}
|
306
|
+
|
307
|
+
v.put("$gt", record);
|
308
|
+
newQuery.put(k, v);
|
309
|
+
}
|
310
|
+
String newQueryString = mapper.writeValueAsString(newQuery);
|
311
|
+
log.info(String.format("New query value was generated for incremental load: '%s'", newQueryString));
|
312
|
+
result.put("query", newQueryString);
|
313
|
+
}
|
314
|
+
|
315
|
+
for (String k : incrementalField.get()) {
|
316
|
+
newSort.put(k, 1);
|
317
|
+
}
|
318
|
+
|
319
|
+
String newSortString = mapper.writeValueAsString(newSort);
|
320
|
+
log.info(String.format("New sort value was generated for incremental load: '%s'", newSortString));
|
321
|
+
result.put("sort", newSortString);
|
322
|
+
|
323
|
+
return result;
|
324
|
+
}
|
325
|
+
catch (JSONParseException | IOException ex) {
|
326
|
+
throw new ConfigException("Could not generate new query for incremental load.");
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
204
330
|
private void validateJsonField(String name, String jsonString)
|
205
331
|
{
|
206
332
|
try {
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.input.mongodb;
|
2
2
|
|
3
|
+
import com.google.common.base.Optional;
|
3
4
|
import org.bson.BsonReader;
|
4
5
|
import org.bson.BsonType;
|
5
6
|
import org.bson.BsonWriter;
|
@@ -14,12 +15,19 @@ import org.slf4j.Logger;
|
|
14
15
|
import java.text.SimpleDateFormat;
|
15
16
|
import java.util.ArrayList;
|
16
17
|
import java.util.Date;
|
18
|
+
import java.util.HashMap;
|
17
19
|
import java.util.LinkedHashMap;
|
18
20
|
import java.util.List;
|
19
21
|
import java.util.Map;
|
20
22
|
import java.util.TimeZone;
|
21
|
-
|
22
|
-
import static org.msgpack.value.ValueFactory
|
23
|
+
import static org.msgpack.value.ValueFactory.newArray;
|
24
|
+
import static org.msgpack.value.ValueFactory.newBinary;
|
25
|
+
import static org.msgpack.value.ValueFactory.newBoolean;
|
26
|
+
import static org.msgpack.value.ValueFactory.newFloat;
|
27
|
+
import static org.msgpack.value.ValueFactory.newInteger;
|
28
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
29
|
+
import static org.msgpack.value.ValueFactory.newNil;
|
30
|
+
import static org.msgpack.value.ValueFactory.newString;
|
23
31
|
|
24
32
|
public class ValueCodec implements Codec<Value>
|
25
33
|
{
|
@@ -27,6 +35,9 @@ public class ValueCodec implements Codec<Value>
|
|
27
35
|
private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
|
28
36
|
private final boolean stopOnInvalidRecord;
|
29
37
|
private final MongodbInputPlugin.PluginTask task;
|
38
|
+
private final Optional<List<String>> incrementalField;
|
39
|
+
private Map<String, Object> lastRecord;
|
40
|
+
private Map<String, String> lastRecordType;
|
30
41
|
|
31
42
|
public ValueCodec(boolean stopOnInvalidRecord, MongodbInputPlugin.PluginTask task)
|
32
43
|
{
|
@@ -34,6 +45,9 @@ public class ValueCodec implements Codec<Value>
|
|
34
45
|
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
|
35
46
|
this.stopOnInvalidRecord = stopOnInvalidRecord;
|
36
47
|
this.task = task;
|
48
|
+
this.incrementalField = task.getIncrementalField();
|
49
|
+
this.lastRecord = new HashMap<>();
|
50
|
+
this.lastRecordType = new HashMap<>();
|
37
51
|
}
|
38
52
|
|
39
53
|
@Override
|
@@ -49,12 +63,18 @@ public class ValueCodec implements Codec<Value>
|
|
49
63
|
|
50
64
|
reader.readStartDocument();
|
51
65
|
while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
|
52
|
-
String
|
66
|
+
String originalFieldName = reader.readName();
|
53
67
|
BsonType type = reader.getCurrentBsonType();
|
54
|
-
fieldName = normalize(
|
68
|
+
String fieldName = normalize(originalFieldName);
|
55
69
|
|
70
|
+
Value value;
|
56
71
|
try {
|
57
|
-
|
72
|
+
value = readValue(reader, decoderContext);
|
73
|
+
kvs.put(newString(fieldName), value);
|
74
|
+
if (incrementalField.isPresent() && incrementalField.get().contains(originalFieldName)) {
|
75
|
+
this.lastRecord.put(originalFieldName, value);
|
76
|
+
this.lastRecordType.put(originalFieldName, type.toString());
|
77
|
+
}
|
58
78
|
}
|
59
79
|
catch (UnknownTypeFoundException ex) {
|
60
80
|
reader.skipValue();
|
@@ -141,6 +161,16 @@ public class ValueCodec implements Codec<Value>
|
|
141
161
|
return key;
|
142
162
|
}
|
143
163
|
|
164
|
+
public Map<String, Object> getLastRecord()
|
165
|
+
{
|
166
|
+
return this.lastRecord;
|
167
|
+
}
|
168
|
+
|
169
|
+
public Map<String, String> getLastRecordType()
|
170
|
+
{
|
171
|
+
return this.lastRecordType;
|
172
|
+
}
|
173
|
+
|
144
174
|
public static class UnknownTypeFoundException extends DataException
|
145
175
|
{
|
146
176
|
UnknownTypeFoundException(String message)
|
@@ -2,6 +2,7 @@ package org.embulk.input.mongodb;
|
|
2
2
|
|
3
3
|
import com.fasterxml.jackson.databind.JsonNode;
|
4
4
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
|
+
import com.google.common.base.Optional;
|
5
6
|
import com.google.common.collect.ImmutableList;
|
6
7
|
import com.google.common.collect.Lists;
|
7
8
|
import com.mongodb.client.MongoCollection;
|
@@ -15,6 +16,7 @@ import org.bson.BsonTimestamp;
|
|
15
16
|
import org.bson.Document;
|
16
17
|
import org.bson.types.Symbol;
|
17
18
|
import org.embulk.EmbulkTestRuntime;
|
19
|
+
import org.embulk.config.ConfigDiff;
|
18
20
|
import org.embulk.config.ConfigException;
|
19
21
|
import org.embulk.config.ConfigSource;
|
20
22
|
import org.embulk.config.TaskReport;
|
@@ -39,7 +41,9 @@ import java.text.DateFormat;
|
|
39
41
|
import java.text.SimpleDateFormat;
|
40
42
|
import java.util.ArrayList;
|
41
43
|
import java.util.Arrays;
|
44
|
+
import java.util.HashMap;
|
42
45
|
import java.util.List;
|
46
|
+
import java.util.Map;
|
43
47
|
import java.util.TimeZone;
|
44
48
|
|
45
49
|
import static org.hamcrest.CoreMatchers.is;
|
@@ -93,6 +97,8 @@ public class TestMongodbInputPlugin
|
|
93
97
|
assertEquals("{}", task.getSort());
|
94
98
|
assertEquals((long) 10000, (long) task.getBatchSize());
|
95
99
|
assertEquals("record", task.getJsonColumnName());
|
100
|
+
assertEquals(Optional.absent(), task.getIncrementalField());
|
101
|
+
assertEquals(Optional.absent(), task.getLastRecord());
|
96
102
|
}
|
97
103
|
|
98
104
|
@Test(expected = ConfigException.class)
|
@@ -125,6 +131,31 @@ public class TestMongodbInputPlugin
|
|
125
131
|
plugin.transaction(config, new Control());
|
126
132
|
}
|
127
133
|
|
134
|
+
@Test(expected = ConfigException.class)
|
135
|
+
public void checkInvalidOptionCombination()
|
136
|
+
{
|
137
|
+
ConfigSource config = Exec.newConfigSource()
|
138
|
+
.set("uri", MONGO_URI)
|
139
|
+
.set("collection", MONGO_COLLECTION)
|
140
|
+
.set("sort", "{ \"field1\": 1 }")
|
141
|
+
.set("incremental_field", Optional.of(Arrays.asList("account")));
|
142
|
+
|
143
|
+
plugin.transaction(config, new Control());
|
144
|
+
}
|
145
|
+
|
146
|
+
@Test(expected = ConfigException.class)
|
147
|
+
public void checkInvalidQueryOption()
|
148
|
+
{
|
149
|
+
ConfigSource config = Exec.newConfigSource()
|
150
|
+
.set("uri", MONGO_URI)
|
151
|
+
.set("collection", MONGO_COLLECTION)
|
152
|
+
.set("query", "{\"key\":invalid_value}")
|
153
|
+
.set("last_record", 0)
|
154
|
+
.set("incremental_field", Optional.of(Arrays.asList("account")));
|
155
|
+
|
156
|
+
plugin.transaction(config, new Control());
|
157
|
+
}
|
158
|
+
|
128
159
|
@Test
|
129
160
|
public void testResume()
|
130
161
|
{
|
@@ -167,6 +198,44 @@ public class TestMongodbInputPlugin
|
|
167
198
|
assertValidRecords(getFieldSchema(), output);
|
168
199
|
}
|
169
200
|
|
201
|
+
@Test
|
202
|
+
public void testRunWithIncrementalLoad() throws Exception
|
203
|
+
{
|
204
|
+
ConfigSource config = Exec.newConfigSource()
|
205
|
+
.set("uri", MONGO_URI)
|
206
|
+
.set("collection", MONGO_COLLECTION)
|
207
|
+
.set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")));
|
208
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
209
|
+
|
210
|
+
dropCollection(task, MONGO_COLLECTION);
|
211
|
+
createCollection(task, MONGO_COLLECTION);
|
212
|
+
insertDocument(task, createValidDocuments());
|
213
|
+
|
214
|
+
ConfigDiff diff = plugin.transaction(config, new Control());
|
215
|
+
ConfigDiff lastRecord = diff.getNested("last_record");
|
216
|
+
|
217
|
+
assertEquals("32864", lastRecord.get(String.class, "int32_field"));
|
218
|
+
assertEquals("1.23", lastRecord.get(String.class, "double_field"));
|
219
|
+
assertEquals("{$date=2015-01-27T10:23:49.000Z}", lastRecord.get(Map.class, "datetime_field").toString());
|
220
|
+
assertEquals("true", lastRecord.get(String.class, "boolean_field"));
|
221
|
+
}
|
222
|
+
|
223
|
+
@Test(expected = ConfigException.class)
|
224
|
+
public void testRunWithIncrementalLoadUnsupportedType() throws Exception
|
225
|
+
{
|
226
|
+
ConfigSource config = Exec.newConfigSource()
|
227
|
+
.set("uri", MONGO_URI)
|
228
|
+
.set("collection", MONGO_COLLECTION)
|
229
|
+
.set("incremental_field", Optional.of(Arrays.asList("document_field")));
|
230
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
231
|
+
|
232
|
+
dropCollection(task, MONGO_COLLECTION);
|
233
|
+
createCollection(task, MONGO_COLLECTION);
|
234
|
+
insertDocument(task, createValidDocuments());
|
235
|
+
|
236
|
+
plugin.transaction(config, new Control());
|
237
|
+
}
|
238
|
+
|
170
239
|
@Test(expected = ValueCodec.UnknownTypeFoundException.class)
|
171
240
|
public void testRunWithUnsupportedType() throws Exception
|
172
241
|
{
|
@@ -229,6 +298,102 @@ public class TestMongodbInputPlugin
|
|
229
298
|
}
|
230
299
|
}
|
231
300
|
|
301
|
+
@Test
|
302
|
+
@SuppressWarnings("unchecked")
|
303
|
+
public void testBuildIncrementalCondition() throws Exception
|
304
|
+
{
|
305
|
+
PluginTask task = config().loadConfig(PluginTask.class);
|
306
|
+
dropCollection(task, MONGO_COLLECTION);
|
307
|
+
createCollection(task, MONGO_COLLECTION);
|
308
|
+
insertDocument(task, createValidDocuments());
|
309
|
+
|
310
|
+
Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
|
311
|
+
method.setAccessible(true);
|
312
|
+
|
313
|
+
ConfigSource config = Exec.newConfigSource()
|
314
|
+
.set("uri", MONGO_URI)
|
315
|
+
.set("collection", MONGO_COLLECTION)
|
316
|
+
.set("incremental_field", Optional.of(Arrays.asList("account")));
|
317
|
+
task = config.loadConfig(PluginTask.class);
|
318
|
+
Map<String, String> actual = (Map<String, String>) method.invoke(plugin, task);
|
319
|
+
Map<String, String> expected = new HashMap<>();
|
320
|
+
expected.put("query", "{}");
|
321
|
+
expected.put("sort", "{\"account\":1}");
|
322
|
+
assertEquals(expected, actual);
|
323
|
+
|
324
|
+
Map<String, Object> lastRecord = new HashMap<>();
|
325
|
+
Map<String, String> innerRecord = new HashMap<>();
|
326
|
+
innerRecord.put("$oid", "abc");
|
327
|
+
lastRecord.put("_id", innerRecord);
|
328
|
+
lastRecord.put("int32_field", 15000);
|
329
|
+
innerRecord = new HashMap<>();
|
330
|
+
innerRecord.put("$date", "2015-01-27T19:23:49Z");
|
331
|
+
lastRecord.put("datetime_field", innerRecord);
|
332
|
+
config = Exec.newConfigSource()
|
333
|
+
.set("uri", MONGO_URI)
|
334
|
+
.set("collection", MONGO_COLLECTION)
|
335
|
+
.set("query", "{\"double_field\":{\"$gte\": 1.23}}")
|
336
|
+
.set("incremental_field", Optional.of(Arrays.asList("_id", "int32_field", "datetime_field")))
|
337
|
+
.set("last_record", Optional.of(lastRecord));
|
338
|
+
task = config.loadConfig(PluginTask.class);
|
339
|
+
actual = (Map<String, String>) method.invoke(plugin, task);
|
340
|
+
expected.put("query", "{\"double_field\":{\"$gte\":1.23},\"int32_field\":{\"$gt\":15000},\"_id\":{\"$gt\":{\"$oid\":\"abc\"}},\"datetime_field\":{\"$gt\":{\"$date\":\"2015-01-27T19:23:49Z\"}}}");
|
341
|
+
expected.put("sort", "{\"_id\":1,\"int32_field\":1,\"datetime_field\":1}");
|
342
|
+
assertEquals(expected, actual);
|
343
|
+
}
|
344
|
+
|
345
|
+
@Test
|
346
|
+
public void testBuildIncrementalConditionFieldDuplicated() throws Exception
|
347
|
+
{
|
348
|
+
Map<String, Object> lastRecord = new HashMap<>();
|
349
|
+
lastRecord.put("double_field", "0");
|
350
|
+
|
351
|
+
ConfigSource config = Exec.newConfigSource()
|
352
|
+
.set("uri", MONGO_URI)
|
353
|
+
.set("collection", MONGO_COLLECTION)
|
354
|
+
.set("query", "{\"double_field\":{\"$gte\": 1.23}}")
|
355
|
+
.set("incremental_field", Optional.of(Arrays.asList("double_field")))
|
356
|
+
.set("last_record", Optional.of(lastRecord));
|
357
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
358
|
+
dropCollection(task, MONGO_COLLECTION);
|
359
|
+
createCollection(task, MONGO_COLLECTION);
|
360
|
+
insertDocument(task, createValidDocuments());
|
361
|
+
|
362
|
+
Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
|
363
|
+
method.setAccessible(true);
|
364
|
+
try {
|
365
|
+
method.invoke(plugin, task); // field declaration was duplicated between query and incremental_field
|
366
|
+
}
|
367
|
+
catch (Exception ex) {
|
368
|
+
assertEquals(ConfigException.class, ex.getCause().getClass());
|
369
|
+
}
|
370
|
+
}
|
371
|
+
|
372
|
+
@Test
|
373
|
+
public void testBuildIncrementalConditionFieldRequired() throws Exception
|
374
|
+
{
|
375
|
+
Map<String, Object> lastRecord = new HashMap<>();
|
376
|
+
lastRecord.put("double_field", "0");
|
377
|
+
|
378
|
+
ConfigSource config = Exec.newConfigSource()
|
379
|
+
.set("uri", MONGO_URI)
|
380
|
+
.set("collection", MONGO_COLLECTION)
|
381
|
+
.set("incremental_field", Optional.of(Arrays.asList("invalid_field")))
|
382
|
+
.set("last_record", Optional.of(lastRecord));
|
383
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
384
|
+
dropCollection(task, MONGO_COLLECTION);
|
385
|
+
createCollection(task, MONGO_COLLECTION);
|
386
|
+
|
387
|
+
Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
|
388
|
+
method.setAccessible(true);
|
389
|
+
try {
|
390
|
+
method.invoke(plugin, task); // field declaration was not set at incremental_field
|
391
|
+
}
|
392
|
+
catch (Exception ex) {
|
393
|
+
assertEquals(ConfigException.class, ex.getCause().getClass());
|
394
|
+
}
|
395
|
+
}
|
396
|
+
|
232
397
|
static List<TaskReport> emptyTaskReports(int taskCount)
|
233
398
|
{
|
234
399
|
ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
@@ -376,7 +541,8 @@ public class TestMongodbInputPlugin
|
|
376
541
|
collection.insertMany(documents);
|
377
542
|
}
|
378
543
|
|
379
|
-
private DateFormat getUTCDateFormat()
|
544
|
+
private DateFormat getUTCDateFormat()
|
545
|
+
{
|
380
546
|
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
|
381
547
|
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
|
382
548
|
return dateFormat;
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-mongodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuyuki Honda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -67,7 +67,7 @@ files:
|
|
67
67
|
- src/test/resources/id_field_name.yml
|
68
68
|
- src/test/resources/id_field_name_expected.csv
|
69
69
|
- src/test/resources/my_collection.jsonl
|
70
|
-
- classpath/embulk-input-mongodb-0.
|
70
|
+
- classpath/embulk-input-mongodb-0.4.0.jar
|
71
71
|
- classpath/mongo-java-driver-3.2.2.jar
|
72
72
|
homepage: https://github.com/hakobera/embulk-input-mongodb
|
73
73
|
licenses:
|
Binary file
|