embulk-input-mongodb 0.3.2 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9e8662a212da54dab9c41190514709db11ee5db3
4
- data.tar.gz: e914845e6bdbf89e1bdf4f8c23f968f316cbd130
3
+ metadata.gz: fc232a73722d1157d53b5d4de1374cadd11cd234
4
+ data.tar.gz: 7792d6114a467b793fcacfaa9a64df7bfe1e5aa3
5
5
  SHA512:
6
- metadata.gz: 6031e390a789b9feefb3918eefdd051ac23ad7e95594d56d65dfbf6af034b9611437979cbc30f9bfb9fe9fe49d16323171a455b4ac4d7903c4b361353099182f
7
- data.tar.gz: 2a2bd692dd0506b46036ebb10e50af514d1a251f72fc74d853995e7a1d28cb3ca68036fb26faab045862cc85ed25ec75344b761f60251b9220fb10aa5f37ad53
6
+ metadata.gz: 1af752c57c9fc04d2d65e9665d6c11c739be8c2805c07b6bd65fad7a244685bf78eaf8da6a5a84e59d48e1fd18db671690789ace2d9cd7c6d62d2e1afcb5c390
7
+ data.tar.gz: 2cb4e589f0e8d524414b6a51135b4931ca91514f811b036c5d1452e2933045a4a86d7c3a615ce31eef85d8c23e66c1ba78d8ad952d9a5dd324e9c15682933038
data/README.md CHANGED
@@ -28,6 +28,8 @@ This plugin only works with embulk >= 0.8.8.
28
28
  - **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
29
29
  - **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
30
30
  - **sort**: ordering of results (string, optional)
31
+ - **incremental_field** list of field name (list, optional, can't use with sort option)
32
+ - **last_record** (hash, optional) last loaded record for incremental load
31
33
  - **stop_on_invalid_record** Stop bulk load transaction if a document includes invalid record (such as unsupported object type) (boolean, optional, default: false)
32
34
  - **json_column_name**: column name used in outputs (string, optional, default: "json")
33
35
 
@@ -54,6 +56,52 @@ in:
54
56
  sort: '{ "field1": 1 }'
55
57
  ```
56
58
 
59
+ ### Incremental loading
60
+
61
+ ```yaml
62
+ in:
63
+ type: mongodb
64
+ uri: mongodb://myuser:mypassword@localhost:27017/my_database
65
+ collection: "my_collection"
66
+ query: '{ field1: { $gt: 3 } }'
67
+ projection: '{ "_id": 1, "field1": 1, "field2": 1 }'
68
+ incremental_field:
69
+ - "field2"
70
+ last_record: {"field2": 13215}
71
+ ```
72
+
73
+ Plugin will create new query and sort value.
74
+ You can't use `incremental_field` option with `sort` option at the same time.
75
+
76
+ ```
77
+ query { field1: { $gt: 3 }, field2: { $gt: 13215}}
78
+ sort {"field2", 1} # field2 ascending
79
+ ```
80
+
81
+ You have to specify last_record with special characters when field type is `ObjectId` or `DateTime`.
82
+
83
+ ```yaml
84
+ # ObjectId field
85
+ in:
86
+ type: mongodb
87
+ incremental_field:
88
+ - "_id"
89
+ last_record: {"_id": {"$oid": "5739b2261c21e58edfe39716"}}
90
+
91
+ # DateTime field
92
+ in:
93
+ type: mongodb
94
+ incremental_field:
95
+ - "time_field"
96
+ last_record: {"time_field": {"$date": "2015-01-25T13:23:15.000Z"}}
97
+ ```
98
+
99
+ #### Run Incremental load
100
+
101
+ ```
102
+ $ embulk run /path/to/config.yml -c config-diff.yml
103
+ ```
104
+
57
105
  ### Advanced usage with filter plugins
58
106
 
59
107
  ```yaml
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  provided
18
18
  }
19
19
 
20
- version = "0.3.2"
20
+ version = "0.4.0"
21
21
 
22
22
  sourceCompatibility = 1.7
23
23
  targetCompatibility = 1.7
@@ -1,5 +1,6 @@
1
1
  package org.embulk.input.mongodb;
2
2
 
3
+ import com.fasterxml.jackson.databind.ObjectMapper;
3
4
  import com.google.common.base.Optional;
4
5
  import com.google.common.base.Throwables;
5
6
  import com.mongodb.MongoClient;
@@ -19,6 +20,9 @@ import org.embulk.config.ConfigDiff;
19
20
  import org.embulk.config.ConfigException;
20
21
  import org.embulk.config.ConfigInject;
21
22
  import org.embulk.config.ConfigSource;
23
+ import org.embulk.config.DataSource;
24
+ import org.embulk.config.DataSourceImpl;
25
+ import org.embulk.config.ModelManager;
22
26
  import org.embulk.config.Task;
23
27
  import org.embulk.config.TaskReport;
24
28
  import org.embulk.config.TaskSource;
@@ -36,8 +40,12 @@ import org.slf4j.Logger;
36
40
 
37
41
  import javax.validation.constraints.Min;
38
42
 
43
+ import java.io.IOException;
39
44
  import java.net.UnknownHostException;
45
+ import java.util.HashMap;
46
+ import java.util.LinkedHashMap;
40
47
  import java.util.List;
48
+ import java.util.Map;
41
49
 
42
50
  public class MongodbInputPlugin
43
51
  implements InputPlugin
@@ -63,10 +71,12 @@ public class MongodbInputPlugin
63
71
  @Config("query")
64
72
  @ConfigDefault("\"{}\"")
65
73
  String getQuery();
74
+ void setQuery(String query);
66
75
 
67
76
  @Config("sort")
68
77
  @ConfigDefault("\"{}\"")
69
78
  String getSort();
79
+ void setSort(String sort);
70
80
 
71
81
  @Config("id_field_name")
72
82
  @ConfigDefault("\"_id\"")
@@ -85,6 +95,14 @@ public class MongodbInputPlugin
85
95
  @ConfigDefault("\"record\"")
86
96
  String getJsonColumnName();
87
97
 
98
+ @Config("incremental_field")
99
+ @ConfigDefault("null")
100
+ Optional<List<String>> getIncrementalField();
101
+
102
+ @Config("last_record")
103
+ @ConfigDefault("null")
104
+ Optional<Map<String, Object>> getLastRecord();
105
+
88
106
  @ConfigInject
89
107
  BufferAllocator getBufferAllocator();
90
108
  }
@@ -99,6 +117,13 @@ public class MongodbInputPlugin
99
117
  if (task.getFields().isPresent()) {
100
118
  throw new ConfigException("field option was deprecated so setting will be ignored");
101
119
  }
120
+ if (task.getIncrementalField().isPresent() && !task.getSort().equals("{}")) {
121
+ throw new ConfigException("both of sort and incremental_load can't be used together");
122
+ }
123
+
124
+ Map<String, String> newCondition = buildIncrementalCondition(task);
125
+ task.setQuery(newCondition.get("query"));
126
+ task.setSort(newCondition.get("sort"));
102
127
 
103
128
  validateJsonField("projection", task.getProjection());
104
129
  validateJsonField("query", task.getQuery());
@@ -120,8 +145,14 @@ public class MongodbInputPlugin
120
145
  Schema schema, int taskCount,
121
146
  InputPlugin.Control control)
122
147
  {
123
- control.run(taskSource, schema, taskCount);
124
- return Exec.newConfigDiff();
148
+ List<TaskReport> report = control.run(taskSource, schema, taskCount);
149
+
150
+ ConfigDiff configDiff = Exec.newConfigDiff();
151
+ if (report.size() > 0 && report.get(0).has("last_record")) {
152
+ configDiff.set("last_record", report.get(0).get(Map.class, "last_record"));
153
+ }
154
+
155
+ return configDiff;
125
156
  }
126
157
 
127
158
  @Override
@@ -142,13 +173,14 @@ public class MongodbInputPlugin
142
173
  PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
143
174
  final Column column = pageBuilder.getSchema().getColumns().get(0);
144
175
 
176
+ ValueCodec valueCodec = new ValueCodec(task.getStopOnInvalidRecord(), task);
145
177
  MongoCollection<Value> collection;
146
178
  try {
147
179
  MongoDatabase db = connect(task);
148
180
 
149
181
  CodecRegistry registry = CodecRegistries.fromRegistries(
150
182
  MongoClient.getDefaultCodecRegistry(),
151
- CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord(), task))
183
+ CodecRegistries.fromCodecs(valueCodec)
152
184
  );
153
185
  collection = db.getCollection(task.getCollection(), Value.class)
154
186
  .withCodecRegistry(registry);
@@ -181,7 +213,44 @@ public class MongodbInputPlugin
181
213
 
182
214
  pageBuilder.finish();
183
215
 
184
- return Exec.newTaskReport();
216
+ TaskReport report = Exec.newTaskReport();
217
+
218
+ if (valueCodec.getLastRecord() != null) {
219
+ DataSource lastRecord = new DataSourceImpl(Exec.getInjector().getInstance(ModelManager.class));
220
+ for (String k : valueCodec.getLastRecord().keySet()) {
221
+ String value = valueCodec.getLastRecord().get(k).toString();
222
+ Map<String, String> types = valueCodec.getLastRecordType();
223
+ HashMap<String, String> innerValue = new HashMap<>();
224
+ switch(types.get(k)) {
225
+ case "OBJECT_ID":
226
+ innerValue.put("$oid", value);
227
+ lastRecord.set(k, innerValue);
228
+ break;
229
+ case "DATE_TIME":
230
+ innerValue.put("$date", value);
231
+ lastRecord.set(k, innerValue);
232
+ break;
233
+ case "INT32":
234
+ case "INT64":
235
+ case "TIMESTAMP":
236
+ lastRecord.set(k, Integer.valueOf(value));
237
+ break;
238
+ case "BOOLEAN":
239
+ lastRecord.set(k, Boolean.valueOf(value));
240
+ break;
241
+ case "DOUBLE":
242
+ lastRecord.set(k, Double.valueOf(value));
243
+ break;
244
+ case "DOCUMENT":
245
+ case "ARRAY":
246
+ throw new ConfigException(String.format("Unsupported type '%s' was given for 'last_record' [%s]", types.get(k), value));
247
+ default:
248
+ lastRecord.set(k, value);
249
+ }
250
+ }
251
+ report.setNested("last_record", lastRecord);
252
+ }
253
+ return report;
185
254
  }
186
255
 
187
256
  @Override
@@ -201,6 +270,63 @@ public class MongodbInputPlugin
201
270
  return db;
202
271
  }
203
272
 
273
+ private Map<String, String> buildIncrementalCondition(PluginTask task)
274
+ {
275
+ Map<String, String> result = new HashMap<>();
276
+ String query = task.getQuery();
277
+ String sort = task.getSort();
278
+ result.put("query", query);
279
+ result.put("sort", sort);
280
+
281
+ Optional<List<String>> incrementalField = task.getIncrementalField();
282
+ Optional<Map<String, Object>> lastRecord = task.getLastRecord();
283
+ if (!incrementalField.isPresent()) {
284
+ return result;
285
+ }
286
+
287
+ Map<String, Object> newQuery = new LinkedHashMap<>();
288
+ Map<String, Integer> newSort = new LinkedHashMap<>();
289
+ ObjectMapper mapper = new ObjectMapper();
290
+
291
+ try {
292
+ @SuppressWarnings("unchecked")
293
+ Map<String, Object> queryJson = mapper.readValue(query, Map.class);
294
+ for (String k : queryJson.keySet()) {
295
+ newQuery.put(k, queryJson.get(k));
296
+ }
297
+
298
+ if (lastRecord.isPresent()) {
299
+ for (String k : lastRecord.get().keySet()) {
300
+ Map<String, Object> v = new HashMap<>();
301
+ Object record = lastRecord.get().get(k);
302
+
303
+ if (newQuery.containsKey(k)) {
304
+ throw new ConfigException("Field declaration was duplicated between 'incremental_field' and 'query' options");
305
+ }
306
+
307
+ v.put("$gt", record);
308
+ newQuery.put(k, v);
309
+ }
310
+ String newQueryString = mapper.writeValueAsString(newQuery);
311
+ log.info(String.format("New query value was generated for incremental load: '%s'", newQueryString));
312
+ result.put("query", newQueryString);
313
+ }
314
+
315
+ for (String k : incrementalField.get()) {
316
+ newSort.put(k, 1);
317
+ }
318
+
319
+ String newSortString = mapper.writeValueAsString(newSort);
320
+ log.info(String.format("New sort value was generated for incremental load: '%s'", newSortString));
321
+ result.put("sort", newSortString);
322
+
323
+ return result;
324
+ }
325
+ catch (JSONParseException | IOException ex) {
326
+ throw new ConfigException("Could not generate new query for incremental load.");
327
+ }
328
+ }
329
+
204
330
  private void validateJsonField(String name, String jsonString)
205
331
  {
206
332
  try {
@@ -1,5 +1,6 @@
1
1
  package org.embulk.input.mongodb;
2
2
 
3
+ import com.google.common.base.Optional;
3
4
  import org.bson.BsonReader;
4
5
  import org.bson.BsonType;
5
6
  import org.bson.BsonWriter;
@@ -14,12 +15,19 @@ import org.slf4j.Logger;
14
15
  import java.text.SimpleDateFormat;
15
16
  import java.util.ArrayList;
16
17
  import java.util.Date;
18
+ import java.util.HashMap;
17
19
  import java.util.LinkedHashMap;
18
20
  import java.util.List;
19
21
  import java.util.Map;
20
22
  import java.util.TimeZone;
21
-
22
- import static org.msgpack.value.ValueFactory.*;
23
+ import static org.msgpack.value.ValueFactory.newArray;
24
+ import static org.msgpack.value.ValueFactory.newBinary;
25
+ import static org.msgpack.value.ValueFactory.newBoolean;
26
+ import static org.msgpack.value.ValueFactory.newFloat;
27
+ import static org.msgpack.value.ValueFactory.newInteger;
28
+ import static org.msgpack.value.ValueFactory.newMap;
29
+ import static org.msgpack.value.ValueFactory.newNil;
30
+ import static org.msgpack.value.ValueFactory.newString;
23
31
 
24
32
  public class ValueCodec implements Codec<Value>
25
33
  {
@@ -27,6 +35,9 @@ public class ValueCodec implements Codec<Value>
27
35
  private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
28
36
  private final boolean stopOnInvalidRecord;
29
37
  private final MongodbInputPlugin.PluginTask task;
38
+ private final Optional<List<String>> incrementalField;
39
+ private Map<String, Object> lastRecord;
40
+ private Map<String, String> lastRecordType;
30
41
 
31
42
  public ValueCodec(boolean stopOnInvalidRecord, MongodbInputPlugin.PluginTask task)
32
43
  {
@@ -34,6 +45,9 @@ public class ValueCodec implements Codec<Value>
34
45
  formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
35
46
  this.stopOnInvalidRecord = stopOnInvalidRecord;
36
47
  this.task = task;
48
+ this.incrementalField = task.getIncrementalField();
49
+ this.lastRecord = new HashMap<>();
50
+ this.lastRecordType = new HashMap<>();
37
51
  }
38
52
 
39
53
  @Override
@@ -49,12 +63,18 @@ public class ValueCodec implements Codec<Value>
49
63
 
50
64
  reader.readStartDocument();
51
65
  while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
52
- String fieldName = reader.readName();
66
+ String originalFieldName = reader.readName();
53
67
  BsonType type = reader.getCurrentBsonType();
54
- fieldName = normalize(fieldName);
68
+ String fieldName = normalize(originalFieldName);
55
69
 
70
+ Value value;
56
71
  try {
57
- kvs.put(newString(fieldName), readValue(reader, decoderContext));
72
+ value = readValue(reader, decoderContext);
73
+ kvs.put(newString(fieldName), value);
74
+ if (incrementalField.isPresent() && incrementalField.get().contains(originalFieldName)) {
75
+ this.lastRecord.put(originalFieldName, value);
76
+ this.lastRecordType.put(originalFieldName, type.toString());
77
+ }
58
78
  }
59
79
  catch (UnknownTypeFoundException ex) {
60
80
  reader.skipValue();
@@ -141,6 +161,16 @@ public class ValueCodec implements Codec<Value>
141
161
  return key;
142
162
  }
143
163
 
164
+ public Map<String, Object> getLastRecord()
165
+ {
166
+ return this.lastRecord;
167
+ }
168
+
169
+ public Map<String, String> getLastRecordType()
170
+ {
171
+ return this.lastRecordType;
172
+ }
173
+
144
174
  public static class UnknownTypeFoundException extends DataException
145
175
  {
146
176
  UnknownTypeFoundException(String message)
@@ -2,6 +2,7 @@ package org.embulk.input.mongodb;
2
2
 
3
3
  import com.fasterxml.jackson.databind.JsonNode;
4
4
  import com.fasterxml.jackson.databind.ObjectMapper;
5
+ import com.google.common.base.Optional;
5
6
  import com.google.common.collect.ImmutableList;
6
7
  import com.google.common.collect.Lists;
7
8
  import com.mongodb.client.MongoCollection;
@@ -15,6 +16,7 @@ import org.bson.BsonTimestamp;
15
16
  import org.bson.Document;
16
17
  import org.bson.types.Symbol;
17
18
  import org.embulk.EmbulkTestRuntime;
19
+ import org.embulk.config.ConfigDiff;
18
20
  import org.embulk.config.ConfigException;
19
21
  import org.embulk.config.ConfigSource;
20
22
  import org.embulk.config.TaskReport;
@@ -39,7 +41,9 @@ import java.text.DateFormat;
39
41
  import java.text.SimpleDateFormat;
40
42
  import java.util.ArrayList;
41
43
  import java.util.Arrays;
44
+ import java.util.HashMap;
42
45
  import java.util.List;
46
+ import java.util.Map;
43
47
  import java.util.TimeZone;
44
48
 
45
49
  import static org.hamcrest.CoreMatchers.is;
@@ -93,6 +97,8 @@ public class TestMongodbInputPlugin
93
97
  assertEquals("{}", task.getSort());
94
98
  assertEquals((long) 10000, (long) task.getBatchSize());
95
99
  assertEquals("record", task.getJsonColumnName());
100
+ assertEquals(Optional.absent(), task.getIncrementalField());
101
+ assertEquals(Optional.absent(), task.getLastRecord());
96
102
  }
97
103
 
98
104
  @Test(expected = ConfigException.class)
@@ -125,6 +131,31 @@ public class TestMongodbInputPlugin
125
131
  plugin.transaction(config, new Control());
126
132
  }
127
133
 
134
+ @Test(expected = ConfigException.class)
135
+ public void checkInvalidOptionCombination()
136
+ {
137
+ ConfigSource config = Exec.newConfigSource()
138
+ .set("uri", MONGO_URI)
139
+ .set("collection", MONGO_COLLECTION)
140
+ .set("sort", "{ \"field1\": 1 }")
141
+ .set("incremental_field", Optional.of(Arrays.asList("account")));
142
+
143
+ plugin.transaction(config, new Control());
144
+ }
145
+
146
+ @Test(expected = ConfigException.class)
147
+ public void checkInvalidQueryOption()
148
+ {
149
+ ConfigSource config = Exec.newConfigSource()
150
+ .set("uri", MONGO_URI)
151
+ .set("collection", MONGO_COLLECTION)
152
+ .set("query", "{\"key\":invalid_value}")
153
+ .set("last_record", 0)
154
+ .set("incremental_field", Optional.of(Arrays.asList("account")));
155
+
156
+ plugin.transaction(config, new Control());
157
+ }
158
+
128
159
  @Test
129
160
  public void testResume()
130
161
  {
@@ -167,6 +198,44 @@ public class TestMongodbInputPlugin
167
198
  assertValidRecords(getFieldSchema(), output);
168
199
  }
169
200
 
201
+ @Test
202
+ public void testRunWithIncrementalLoad() throws Exception
203
+ {
204
+ ConfigSource config = Exec.newConfigSource()
205
+ .set("uri", MONGO_URI)
206
+ .set("collection", MONGO_COLLECTION)
207
+ .set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")));
208
+ PluginTask task = config.loadConfig(PluginTask.class);
209
+
210
+ dropCollection(task, MONGO_COLLECTION);
211
+ createCollection(task, MONGO_COLLECTION);
212
+ insertDocument(task, createValidDocuments());
213
+
214
+ ConfigDiff diff = plugin.transaction(config, new Control());
215
+ ConfigDiff lastRecord = diff.getNested("last_record");
216
+
217
+ assertEquals("32864", lastRecord.get(String.class, "int32_field"));
218
+ assertEquals("1.23", lastRecord.get(String.class, "double_field"));
219
+ assertEquals("{$date=2015-01-27T10:23:49.000Z}", lastRecord.get(Map.class, "datetime_field").toString());
220
+ assertEquals("true", lastRecord.get(String.class, "boolean_field"));
221
+ }
222
+
223
+ @Test(expected = ConfigException.class)
224
+ public void testRunWithIncrementalLoadUnsupportedType() throws Exception
225
+ {
226
+ ConfigSource config = Exec.newConfigSource()
227
+ .set("uri", MONGO_URI)
228
+ .set("collection", MONGO_COLLECTION)
229
+ .set("incremental_field", Optional.of(Arrays.asList("document_field")));
230
+ PluginTask task = config.loadConfig(PluginTask.class);
231
+
232
+ dropCollection(task, MONGO_COLLECTION);
233
+ createCollection(task, MONGO_COLLECTION);
234
+ insertDocument(task, createValidDocuments());
235
+
236
+ plugin.transaction(config, new Control());
237
+ }
238
+
170
239
  @Test(expected = ValueCodec.UnknownTypeFoundException.class)
171
240
  public void testRunWithUnsupportedType() throws Exception
172
241
  {
@@ -229,6 +298,102 @@ public class TestMongodbInputPlugin
229
298
  }
230
299
  }
231
300
 
301
+ @Test
302
+ @SuppressWarnings("unchecked")
303
+ public void testBuildIncrementalCondition() throws Exception
304
+ {
305
+ PluginTask task = config().loadConfig(PluginTask.class);
306
+ dropCollection(task, MONGO_COLLECTION);
307
+ createCollection(task, MONGO_COLLECTION);
308
+ insertDocument(task, createValidDocuments());
309
+
310
+ Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
311
+ method.setAccessible(true);
312
+
313
+ ConfigSource config = Exec.newConfigSource()
314
+ .set("uri", MONGO_URI)
315
+ .set("collection", MONGO_COLLECTION)
316
+ .set("incremental_field", Optional.of(Arrays.asList("account")));
317
+ task = config.loadConfig(PluginTask.class);
318
+ Map<String, String> actual = (Map<String, String>) method.invoke(plugin, task);
319
+ Map<String, String> expected = new HashMap<>();
320
+ expected.put("query", "{}");
321
+ expected.put("sort", "{\"account\":1}");
322
+ assertEquals(expected, actual);
323
+
324
+ Map<String, Object> lastRecord = new HashMap<>();
325
+ Map<String, String> innerRecord = new HashMap<>();
326
+ innerRecord.put("$oid", "abc");
327
+ lastRecord.put("_id", innerRecord);
328
+ lastRecord.put("int32_field", 15000);
329
+ innerRecord = new HashMap<>();
330
+ innerRecord.put("$date", "2015-01-27T19:23:49Z");
331
+ lastRecord.put("datetime_field", innerRecord);
332
+ config = Exec.newConfigSource()
333
+ .set("uri", MONGO_URI)
334
+ .set("collection", MONGO_COLLECTION)
335
+ .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
336
+ .set("incremental_field", Optional.of(Arrays.asList("_id", "int32_field", "datetime_field")))
337
+ .set("last_record", Optional.of(lastRecord));
338
+ task = config.loadConfig(PluginTask.class);
339
+ actual = (Map<String, String>) method.invoke(plugin, task);
340
+ expected.put("query", "{\"double_field\":{\"$gte\":1.23},\"int32_field\":{\"$gt\":15000},\"_id\":{\"$gt\":{\"$oid\":\"abc\"}},\"datetime_field\":{\"$gt\":{\"$date\":\"2015-01-27T19:23:49Z\"}}}");
341
+ expected.put("sort", "{\"_id\":1,\"int32_field\":1,\"datetime_field\":1}");
342
+ assertEquals(expected, actual);
343
+ }
344
+
345
+ @Test
346
+ public void testBuildIncrementalConditionFieldDuplicated() throws Exception
347
+ {
348
+ Map<String, Object> lastRecord = new HashMap<>();
349
+ lastRecord.put("double_field", "0");
350
+
351
+ ConfigSource config = Exec.newConfigSource()
352
+ .set("uri", MONGO_URI)
353
+ .set("collection", MONGO_COLLECTION)
354
+ .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
355
+ .set("incremental_field", Optional.of(Arrays.asList("double_field")))
356
+ .set("last_record", Optional.of(lastRecord));
357
+ PluginTask task = config.loadConfig(PluginTask.class);
358
+ dropCollection(task, MONGO_COLLECTION);
359
+ createCollection(task, MONGO_COLLECTION);
360
+ insertDocument(task, createValidDocuments());
361
+
362
+ Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
363
+ method.setAccessible(true);
364
+ try {
365
+ method.invoke(plugin, task); // field declaration was duplicated between query and incremental_field
366
+ }
367
+ catch (Exception ex) {
368
+ assertEquals(ConfigException.class, ex.getCause().getClass());
369
+ }
370
+ }
371
+
372
+ @Test
373
+ public void testBuildIncrementalConditionFieldRequired() throws Exception
374
+ {
375
+ Map<String, Object> lastRecord = new HashMap<>();
376
+ lastRecord.put("double_field", "0");
377
+
378
+ ConfigSource config = Exec.newConfigSource()
379
+ .set("uri", MONGO_URI)
380
+ .set("collection", MONGO_COLLECTION)
381
+ .set("incremental_field", Optional.of(Arrays.asList("invalid_field")))
382
+ .set("last_record", Optional.of(lastRecord));
383
+ PluginTask task = config.loadConfig(PluginTask.class);
384
+ dropCollection(task, MONGO_COLLECTION);
385
+ createCollection(task, MONGO_COLLECTION);
386
+
387
+ Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
388
+ method.setAccessible(true);
389
+ try {
390
+ method.invoke(plugin, task); // field declaration was not set at incremental_field
391
+ }
392
+ catch (Exception ex) {
393
+ assertEquals(ConfigException.class, ex.getCause().getClass());
394
+ }
395
+ }
396
+
232
397
  static List<TaskReport> emptyTaskReports(int taskCount)
233
398
  {
234
399
  ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
@@ -376,7 +541,8 @@ public class TestMongodbInputPlugin
376
541
  collection.insertMany(documents);
377
542
  }
378
543
 
379
- private DateFormat getUTCDateFormat() {
544
+ private DateFormat getUTCDateFormat()
545
+ {
380
546
  DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
381
547
  dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
382
548
  return dateFormat;
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.2
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuyuki Honda
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-06-23 00:00:00.000000000 Z
11
+ date: 2016-07-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ files:
67
67
  - src/test/resources/id_field_name.yml
68
68
  - src/test/resources/id_field_name_expected.csv
69
69
  - src/test/resources/my_collection.jsonl
70
- - classpath/embulk-input-mongodb-0.3.2.jar
70
+ - classpath/embulk-input-mongodb-0.4.0.jar
71
71
  - classpath/mongo-java-driver-3.2.2.jar
72
72
  homepage: https://github.com/hakobera/embulk-input-mongodb
73
73
  licenses: