embulk-input-mongodb 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1c0c287fc6caae40f480044e097b35c22748be2d
4
- data.tar.gz: 31c40a09bfd9aa1d3333105a49b02807c7fefb5b
3
+ metadata.gz: ad8c05dd58ee2ce347781dd2d519aed6a9f08d9a
4
+ data.tar.gz: bd43fe1ba4a5a18d3f577130a4c8a82e3e96184c
5
5
  SHA512:
6
- metadata.gz: 013109760ca0b9040ea5367384d61c283801c97c833c1da0b52625ee8927dadefe71c6cd0f15a4115e4184b4fbdf40faed93eb2406fc3385ed4fb85eef5c3150
7
- data.tar.gz: 2e1e7a1dcac758ede878293a37c9a95a532cb177964c9208d05f64f97c9ad3b1040ec5f9ce37abb25ddc53af07d379abbe24b9edf6c67f2b12e3474c300c2530
6
+ metadata.gz: 5d0597d3ac72e926e6ceda10e15f07fbaf95c8594f24b7fd9facb65c5beade7843b1aa4e8d0d6cbad3891b4d00f5a2d7e3d4f857ea8ba3533a880a05d600a896
7
+ data.tar.gz: 44acde315c5bda2b634ec10af13b88121c413ca96b6549ccc1feb3ff6adc6806107d8fb8bc5cc8e336bd19957324f75f174d114db97706afdc833d32a7c8d346
data/.gitignore CHANGED
@@ -1,4 +1,6 @@
1
1
  *~
2
+ /db/
3
+ /out/
2
4
  /pkg/
3
5
  /tmp/
4
6
  *.gemspec
@@ -2,17 +2,18 @@ language: java
2
2
 
3
3
  jdk:
4
4
  - oraclejdk8
5
- - oraclejdk7
6
- - openjdk7
5
+
6
+ services:
7
+ - docker
7
8
 
8
9
  env:
9
10
  global:
10
- - MONGO_DATABASE=my_database
11
+ - MONGO_DATABASE=mydb
11
12
  - MONGO_COLLECTION=my_collection
12
- - MONGO_URI=mongodb://localhost:27017/my_database
13
+ - MONGO_URI=mongodb://localhost:27017/mydb
14
+ - DOCKER_COMPOSE_VERSION=1.22.0
13
15
 
14
16
  sudo: required
15
- dist: precise
16
17
 
17
18
  before_cache:
18
19
  - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
@@ -22,36 +23,28 @@ cache:
22
23
  - $HOME/.gradle/caches/
23
24
  - $HOME/.gradle/wrapper/
24
25
 
25
- # Work around fix for buffer overflow error on OpenJDK7
26
- # ref: https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165131913
27
26
  before_install:
28
- - cat /etc/hosts # optionally check the content *before*
29
- - sudo hostname "$(hostname | cut -c1-63)"
30
- - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts > /tmp/hosts
31
- - sudo mv /tmp/hosts /etc/hosts
32
- - cat /etc/hosts # optionally check the content *after*
27
+ - sudo rm /usr/local/bin/docker-compose
28
+ - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
29
+ - chmod +x docker-compose
30
+ - sudo mv docker-compose /usr/local/bin
33
31
 
34
32
  install:
35
- - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
36
- - echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
37
- - sudo apt-get update
38
- - sudo apt-get install -y --force-yes mongodb-org
39
- - mongod -version
40
33
  - curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.bintray.com/embulk/maven/embulk-0.8.8.jar"
41
34
  - chmod +x ~/.embulk/bin/embulk
42
35
  - export PATH="$HOME/.embulk/bin:$PATH"
43
36
  - embulk --version
44
37
 
45
38
  before_script:
46
- - echo "Wait mongodb wakeup"
47
- - sleep 15
48
39
  - mkdir -p ./tmp
49
40
  - date
41
+ - docker-compose up -d
42
+ - docker-compose ps
50
43
 
51
44
  script:
52
45
  - ./gradlew check
53
46
  - ./gradlew package
54
- - mongoimport --host 127.0.0.1 --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
47
+ - mongoimport --host 127.0.0.1 -u mongo_user -p dbpass --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
55
48
  - |
56
49
  for target in basic full id_field_name
57
50
  do
data/README.md CHANGED
@@ -21,6 +21,8 @@ This plugin only works with embulk >= 0.8.8.
21
21
  - **uri**: [MongoDB connection string URI](http://docs.mongodb.org/manual/reference/connection-string/) (e.g. 'mongodb://localhost:27017/mydb') (string, required)
22
22
  - use separated URI parameters
23
23
  - **hosts**: list of hosts. `hosts` are pairs of host(string, required) and port(integer, optional, default: 27017)
24
+ - **auth_method**: Auth method. One of `scram-sha-1`, `mongodb-cr`, `auto` (string, optional, default: null)
25
+ - **auth_source**: Auth source. The database name where the user is defined (string, optional, default: null)
24
26
  - **user**: (string, optional)
25
27
  - **password**: (string, optional)
26
28
  - **database**: (string, required)
@@ -37,6 +39,7 @@ This plugin only works with embulk >= 0.8.8.
37
39
  - **query**: A JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
38
40
  - **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
39
41
  - **sort**: Ordering of results (string, optional)
42
+ - **aggregation**: Aggregation query (string, optional) See [Aggregation query](#aggregation-query) for more detail.
40
43
  - **batch_size**: Limits the number of objects returned in one [batch](http://api.mongodb.com/java/current/com/mongodb/DBCursor.html#batchSize-int-) (integer, optional, default: 10000)
41
44
  - **incremental_field** List of field name (list, optional, can't use with sort option)
42
45
  - **last_record** Last loaded record for incremental load (hash, optional)
@@ -45,6 +48,37 @@ This plugin only works with embulk >= 0.8.8.
45
48
 
46
49
  ## Example
47
50
 
51
+ ### Authentication
52
+
53
+ #### Use separated URI prameters
54
+
55
+ ```yaml
56
+ in:
57
+ type: mongodb
58
+ hosts:
59
+ - {host: localhost, port:27017}
60
+ user: myuser
61
+ password: mypassword
62
+ database: my_database
63
+ auth_method: scram-sha-1
64
+ auth_source: auth_db
65
+ collection: "my_collection
66
+ ```
67
+
68
+ If you set `auth_method: auto`, The client will negotiate the best mechanism based on the version of the server that the client is authenticating to.
69
+
70
+ If the server version is 3.0 or higher, the driver will authenticate using the SCRAM-SHA-1 mechanism.
71
+
72
+ Otherwise, the driver will authenticate using the MONGODB_CR mechanism.
73
+
74
+ #### Use URI String
75
+
76
+ ```yaml
77
+ in:
78
+ type: mongodb
79
+ uri: mongodb://myuser:mypassword@localhost:27017/my_database?authMechanism=SCRAM-SHA-1&authSource=another_database
80
+ ```
81
+
48
82
  ### Exporting all objects
49
83
 
50
84
  #### Specify with MongoDB connection string URI.
@@ -128,6 +162,20 @@ in:
128
162
  $ embulk run /path/to/config.yml -c config-diff.yml
129
163
  ```
130
164
 
165
+ ### Aggregation query
166
+
167
+ This plugin supports aggregation query. You can write complex query like below.
168
+
169
+ `aggregation` option can't be used with `sort`, `limit`, `skip`, `query` option. Incremental load also doesn't work with aggregation query.
170
+
171
+ ```yaml
172
+ in:
173
+ type: mongodb
174
+ aggregation: { $match: {"int32_field":{"$gte":5 },} }
175
+ ```
176
+
177
+ See also [Aggregation — MongoDB Manual](https://docs.mongodb.com/manual/aggregation/) and [Aggregation Pipeline Stages — MongoDB Manual](https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/)
178
+
131
179
  ### Advanced usage with filter plugins
132
180
 
133
181
  ```yaml
@@ -173,44 +221,21 @@ $ ./gradlew gem
173
221
 
174
222
  ## Test
175
223
 
176
- ```
177
- $ ./gradlew test # -t to watch change of files and rebuild continuously
178
- ```
224
+ Firstly install Docker and Docker compose then `docker-compose up -d`,
225
+ so that an MongoDB server will be locally launched then you can run tests with `./gradlew test`.
179
226
 
180
- To run unit tests, we need to configure the following environment variables.
227
+ ```sh
228
+ $ docker-compose up -d
229
+ Creating embulk-input-mongodb_server ... done
230
+ Creating mongo-express ... done
231
+ Creating mongoClientTemp ... done
181
232
 
182
- When environment variables are not set, skip almost test cases.
233
+ $ docker-compose ps
234
+ Name Command State Ports
235
+ ------------------------------------------------------------------------------------------------------------------------------
236
+ embulk-input-mongodb_server docker-entrypoint.sh mongod Up 0.0.0.0:27017->27017/tcp, 0.0.0.0:27018->27018/tcp
237
+ mongo-express tini -- /docker-entrypoint ... Up 0.0.0.0:8081->8081/tcp
238
+ mongoClientTemp docker-entrypoint.sh mongo ... Restarting
183
239
 
184
- ```
185
- MONGO_URI
186
- MONGO_COLLECTION
187
- ```
188
-
189
- If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
190
- ```xml
191
- $ vi ~/Library/LaunchAgents/environment.plist
192
- <?xml version="1.0" encoding="UTF-8"?>
193
- <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
194
- <plist version="1.0">
195
- <dict>
196
- <key>Label</key>
197
- <string>my.startup</string>
198
- <key>ProgramArguments</key>
199
- <array>
200
- <string>sh</string>
201
- <string>-c</string>
202
- <string>
203
- launchctl setenv MONGO_URI mongodb://myuser:mypassword@localhost:27017/my_database
204
- launchctl setenv MONGO_COLLECTION my_collection
205
- </string>
206
- </array>
207
- <key>RunAtLoad</key>
208
- <true/>
209
- </dict>
210
- </plist>
211
-
212
- $ launchctl load ~/Library/LaunchAgents/environment.plist
213
- $ launchctl getenv MONGO_URI //try to get value.
214
-
215
- Then start your applications.
240
+ $ ./gradlew test # -t to watch change of files and rebuild continuously
216
241
  ```
@@ -17,15 +17,15 @@ configurations {
17
17
  provided
18
18
  }
19
19
 
20
- version = "0.6.1"
20
+ version = "0.7.0"
21
21
 
22
- sourceCompatibility = 1.7
23
- targetCompatibility = 1.7
22
+ sourceCompatibility = 1.8
23
+ targetCompatibility = 1.8
24
24
 
25
25
  dependencies {
26
26
  compile "org.embulk:embulk-core:0.8.8"
27
27
  provided "org.embulk:embulk-core:0.8.8"
28
- compile "org.mongodb:mongo-java-driver:3.6.1"
28
+ compile "org.mongodb:mongo-java-driver:3.8.1"
29
29
 
30
30
  testCompile "junit:junit:4.+"
31
31
  testCompile "org.embulk:embulk-core:0.8.8:tests"
@@ -0,0 +1,44 @@
1
+ version: '3.1'
2
+ services:
3
+ mongodb:
4
+ container_name: embulk-input-mongodb_server
5
+ image: mongo:3.6
6
+ restart: always
7
+ ports:
8
+ - 27017:27017
9
+ - 27018:27018
10
+ environment:
11
+ MONGO_INITDB_ROOT_USERNAME: admin
12
+ MONGO_INITDB_ROOT_PASSWORD: tiger
13
+ volumes:
14
+ - mongodb-data:/data/db
15
+ - mongodb-configdb:/data/configdb
16
+
17
+ mongo-express:
18
+ container_name: mongo-express
19
+ image: mongo-express
20
+ restart: always
21
+ ports:
22
+ - 8081:8081
23
+ depends_on:
24
+ - mongodb
25
+ environment:
26
+ ME_CONFIG_MONGODB_ADMINUSERNAME: admin
27
+ ME_CONFIG_MONGODB_ADMINPASSWORD: tiger
28
+ ME_CONFIG_MONGODB_SERVER: mongodb
29
+
30
+ mongoClientTemp:
31
+ container_name: mongoClientTemp
32
+ image: mongo:3.6
33
+ depends_on:
34
+ - mongodb
35
+ # Sleep to wait MongoDB wake up on Travis CI
36
+ command: >
37
+ /bin/bash -c
38
+ "sleep 15 &&
39
+ mongo --host mongodb -u admin -p tiger admin --eval \"db.getSiblingDB('mydb').createUser({user:'mongo_user', pwd:'dbpass', roles:[{role:'readWrite',db:'mydb'}]});\""
40
+ volumes:
41
+ mongodb-data:
42
+ driver: local
43
+ mongodb-configdb:
44
+ driver: local
@@ -0,0 +1,36 @@
1
+ package org.embulk.input.mongodb;
2
+
3
+ import com.fasterxml.jackson.annotation.JsonCreator;
4
+ import com.fasterxml.jackson.annotation.JsonValue;
5
+ import org.embulk.config.ConfigException;
6
+
7
+ import java.util.Locale;
8
+
9
+ public enum AuthMethod
10
+ {
11
+ AUTO,
12
+ SCRAM_SHA_1,
13
+ MONGODB_CR;
14
+
15
+ @JsonValue
16
+ @Override
17
+ public String toString()
18
+ {
19
+ return name().toLowerCase(Locale.ENGLISH);
20
+ }
21
+
22
+ @JsonCreator
23
+ public static AuthMethod fromString(String value)
24
+ {
25
+ switch (value.replace("_", "-")) {
26
+ case "scram-sha-1":
27
+ return SCRAM_SHA_1;
28
+ case "mongodb-cr":
29
+ return MONGODB_CR;
30
+ case "auto":
31
+ return AUTO;
32
+ default:
33
+ throw new ConfigException(String.format("Unknown auth_method '%s'. Supported auth_method are scram-sha-1, mongodb-cr, auto", value));
34
+ }
35
+ }
36
+ }
@@ -12,6 +12,7 @@ import com.mongodb.ServerAddress;
12
12
  import com.mongodb.client.MongoCollection;
13
13
  import com.mongodb.client.MongoCursor;
14
14
  import com.mongodb.client.MongoDatabase;
15
+ import org.bson.Document;
15
16
  import org.bson.codecs.configuration.CodecRegistries;
16
17
  import org.bson.codecs.configuration.CodecRegistry;
17
18
  import org.bson.conversions.Bson;
@@ -64,6 +65,24 @@ public class MongodbInputPlugin
64
65
  throw new ConfigException("both of skip and incremental_load can't be used together");
65
66
  }
66
67
 
68
+ if (task.getAggregation().isPresent()) {
69
+ if (task.getIncrementalField().isPresent()) {
70
+ throw new ConfigException("both of aggregation and incremental_load can't be used together");
71
+ }
72
+ if (!task.getSort().equals("{}")) {
73
+ throw new ConfigException("both of sort and aggregation can't be used together");
74
+ }
75
+ if (task.getLimit().isPresent()) {
76
+ throw new ConfigException("both of limit and aggregation can't be used together");
77
+ }
78
+ if (task.getSkip().isPresent()) {
79
+ throw new ConfigException("both of skip and aggregation can't be used together");
80
+ }
81
+ if (!task.getQuery().equals("{}")) {
82
+ throw new ConfigException("both of query and aggregation can't be used together");
83
+ }
84
+ }
85
+
67
86
  Map<String, String> newCondition = buildIncrementalCondition(task);
68
87
  task.setQuery(newCondition.get("query"));
69
88
  task.setSort(newCondition.get("sort"));
@@ -71,6 +90,9 @@ public class MongodbInputPlugin
71
90
  validateJsonField("projection", task.getProjection());
72
91
  validateJsonField("query", task.getQuery());
73
92
  validateJsonField("sort", task.getSort());
93
+ if (task.getAggregation().isPresent()) {
94
+ validateJsonField("aggrigation", task.getAggregation().get());
95
+ }
74
96
 
75
97
  // Connect once to throw ConfigException in earlier stage of excecution
76
98
  try {
@@ -146,20 +168,35 @@ public class MongodbInputPlugin
146
168
  log.trace("skip: {}", task.getSkip());
147
169
  }
148
170
 
149
- try (MongoCursor<Value> cursor = collection
150
- .find(query)
151
- .projection(projection)
152
- .sort(sort)
153
- .batchSize(task.getBatchSize())
154
- .limit(task.getLimit().or(0))
155
- .skip(task.getSkip().or(0))
156
- .iterator()) {
157
- while (cursor.hasNext()) {
158
- pageBuilder.setJson(column, cursor.next());
159
- pageBuilder.addRecord();
171
+ if (task.getAggregation().isPresent()) {
172
+ Bson aggregationString = Document.parse(task.getAggregation().get());
173
+ List<Bson> aggregation = Arrays.asList(aggregationString);
174
+ try (MongoCursor<Value> cursor = collection
175
+ .aggregate(aggregation).iterator()) {
176
+ while (cursor.hasNext()) {
177
+ pageBuilder.setJson(column, cursor.next());
178
+ pageBuilder.addRecord();
179
+ }
180
+ } catch (MongoException ex) {
181
+ Throwables.propagate(ex);
182
+ }
183
+ }
184
+ else {
185
+ try (MongoCursor<Value> cursor = collection
186
+ .find(query)
187
+ .projection(projection)
188
+ .sort(sort)
189
+ .batchSize(task.getBatchSize())
190
+ .limit(task.getLimit().or(0))
191
+ .skip(task.getSkip().or(0))
192
+ .iterator()) {
193
+ while (cursor.hasNext()) {
194
+ pageBuilder.setJson(column, cursor.next());
195
+ pageBuilder.addRecord();
196
+ }
197
+ } catch (MongoException ex) {
198
+ Throwables.propagate(ex);
160
199
  }
161
- } catch (MongoException ex) {
162
- Throwables.propagate(ex);
163
200
  }
164
201
 
165
202
  pageBuilder.finish();
@@ -259,18 +296,48 @@ public class MongodbInputPlugin
259
296
  }
260
297
 
261
298
  if (task.getUser().isPresent()) {
262
- MongoCredential credential = MongoCredential.createCredential(
263
- task.getUser().get(),
264
- task.getDatabase().get(),
265
- task.getPassword().get().toCharArray()
266
- );
267
- return new MongoClient(addresses, Arrays.asList(credential));
299
+ return new MongoClient(addresses, Arrays.asList(createCredential(task)));
268
300
  }
269
301
  else {
270
302
  return new MongoClient(addresses);
271
303
  }
272
304
  }
273
305
 
306
+ // @see http://mongodb.github.io/mongo-java-driver/3.0/driver-async/reference/connecting/authenticating/
307
+ private MongoCredential createCredential(PluginTask task)
308
+ {
309
+ MongoCredential credential;
310
+ String authSource = task.getAuthSource().isPresent() ? task.getAuthSource().get() : task.getDatabase().get();
311
+ AuthMethod authMethod = task.getAuthMethod().isPresent() ? task.getAuthMethod().get() : AuthMethod.AUTO;
312
+ switch (authMethod) {
313
+ case SCRAM_SHA_1:
314
+ credential = MongoCredential.createScramSha1Credential(
315
+ task.getUser().get(),
316
+ authSource,
317
+ task.getPassword().get().toCharArray());
318
+ break;
319
+ case MONGODB_CR:
320
+ credential = MongoCredential.createMongoCRCredential(
321
+ task.getUser().get(),
322
+ authSource,
323
+ task.getPassword().get().toCharArray());
324
+ break;
325
+ case AUTO:
326
+ default:
327
+ /* The client will negotiate the best mechanism based on the
328
+ * version of the server that the client is authenticating to.
329
+ * If the server version is 3.0 or higher, the driver will authenticate using the SCRAM-SHA-1 mechanism.
330
+ * Otherwise, the driver will authenticate using the MONGODB_CR mechanism.
331
+ */
332
+ credential = MongoCredential.createCredential(
333
+ task.getUser().get(),
334
+ authSource,
335
+ task.getPassword().get().toCharArray()
336
+ );
337
+ }
338
+ return credential;
339
+ }
340
+
274
341
  private Map<String, String> buildIncrementalCondition(PluginTask task)
275
342
  {
276
343
  Map<String, String> result = new HashMap<>();
@@ -331,7 +398,7 @@ public class MongodbInputPlugin
331
398
  private void validateJsonField(String name, String jsonString)
332
399
  {
333
400
  try {
334
- BasicDBObject.parse(jsonString);
401
+ Document.parse(jsonString);
335
402
  }
336
403
  catch (JsonParseException ex) {
337
404
  throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));
@@ -25,6 +25,14 @@ public interface PluginTask
25
25
  @ConfigDefault("null")
26
26
  Optional<List<HostTask>> getHosts();
27
27
 
28
+ @Config("auth_method")
29
+ @ConfigDefault("null")
30
+ Optional<AuthMethod> getAuthMethod();
31
+
32
+ @Config("auth_source")
33
+ @ConfigDefault("null")
34
+ Optional<String> getAuthSource();
35
+
28
36
  @Config("user")
29
37
  @ConfigDefault("null")
30
38
  Optional<String> getUser();
@@ -53,6 +61,10 @@ public interface PluginTask
53
61
  String getQuery();
54
62
  void setQuery(String query);
55
63
 
64
+ @Config("aggregation")
65
+ @ConfigDefault("null")
66
+ Optional<String> getAggregation();
67
+
56
68
  @Config("sort")
57
69
  @ConfigDefault("\"{}\"")
58
70
  String getSort();
@@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList;
7
7
  import com.google.common.collect.ImmutableMap;
8
8
  import com.google.common.collect.Lists;
9
9
  import com.mongodb.MongoClientURI;
10
+ import com.mongodb.MongoCredential;
10
11
  import com.mongodb.client.MongoCollection;
11
12
  import com.mongodb.client.MongoDatabase;
12
13
  import org.bson.BsonBinary;
@@ -31,7 +32,6 @@ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
31
32
  import org.embulk.spi.type.Types;
32
33
  import org.embulk.spi.util.Pages;
33
34
  import org.junit.Before;
34
- import org.junit.BeforeClass;
35
35
  import org.junit.Rule;
36
36
  import org.junit.Test;
37
37
  import org.junit.rules.ExpectedException;
@@ -53,8 +53,8 @@ import static org.junit.Assert.assertThat;
53
53
 
54
54
  public class TestMongodbInputPlugin
55
55
  {
56
- private static String MONGO_URI;
57
- private static String MONGO_COLLECTION;
56
+ private final String mongoUri = "mongodb://mongo_user:dbpass@localhost:27017/mydb";
57
+ private final String mongoCollection = "my_collection";
58
58
 
59
59
  @Rule
60
60
  public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
@@ -66,20 +66,8 @@ public class TestMongodbInputPlugin
66
66
  private MongodbInputPlugin plugin;
67
67
  private MockPageOutput output;
68
68
 
69
- /*
70
- * This test case requires environment variables
71
- * MONGO_URI
72
- * MONGO_COLLECTION
73
- */
74
- @BeforeClass
75
- public static void initializeConstant()
76
- {
77
- MONGO_URI = System.getenv("MONGO_URI");
78
- MONGO_COLLECTION = System.getenv("MONGO_COLLECTION");
79
- }
80
-
81
69
  @Before
82
- public void createResources() throws Exception
70
+ public void createResources()
83
71
  {
84
72
  config = config();
85
73
  plugin = new MongodbInputPlugin();
@@ -90,8 +78,8 @@ public class TestMongodbInputPlugin
90
78
  public void checkDefaultValues()
91
79
  {
92
80
  ConfigSource config = Exec.newConfigSource()
93
- .set("uri", MONGO_URI)
94
- .set("collection", MONGO_COLLECTION);
81
+ .set("uri", mongoUri)
82
+ .set("collection", mongoCollection);
95
83
 
96
84
  PluginTask task = config.loadConfig(PluginTask.class);
97
85
  assertEquals("{}", task.getQuery());
@@ -109,7 +97,7 @@ public class TestMongodbInputPlugin
109
97
  {
110
98
  ConfigSource config = Exec.newConfigSource()
111
99
  .set("uri", null)
112
- .set("collection", MONGO_COLLECTION);
100
+ .set("collection", mongoCollection);
113
101
 
114
102
  plugin.transaction(config, new Control());
115
103
  }
@@ -119,7 +107,7 @@ public class TestMongodbInputPlugin
119
107
  {
120
108
  ConfigSource config = Exec.newConfigSource()
121
109
  .set("uri", "mongodb://mongouser:password@non-exists.example.com:23490/test")
122
- .set("collection", MONGO_COLLECTION);
110
+ .set("collection", mongoCollection);
123
111
 
124
112
  plugin.transaction(config, new Control());
125
113
  }
@@ -128,7 +116,7 @@ public class TestMongodbInputPlugin
128
116
  public void checkDefaultValuesCollectionIsNull()
129
117
  {
130
118
  ConfigSource config = Exec.newConfigSource()
131
- .set("uri", MONGO_URI)
119
+ .set("uri", mongoUri)
132
120
  .set("collection", null);
133
121
 
134
122
  plugin.transaction(config, new Control());
@@ -138,8 +126,8 @@ public class TestMongodbInputPlugin
138
126
  public void checkSortCannotUseWithIncremental()
139
127
  {
140
128
  ConfigSource config = Exec.newConfigSource()
141
- .set("uri", MONGO_URI)
142
- .set("collection", MONGO_COLLECTION)
129
+ .set("uri", mongoUri)
130
+ .set("collection", mongoCollection)
143
131
  .set("sort", "{ \"field1\": 1 }")
144
132
  .set("incremental_field", Optional.of(Arrays.asList("account")));
145
133
 
@@ -150,8 +138,8 @@ public class TestMongodbInputPlugin
150
138
  public void checkSkipCannotUseWithIncremental()
151
139
  {
152
140
  ConfigSource config = Exec.newConfigSource()
153
- .set("uri", MONGO_URI)
154
- .set("collection", MONGO_COLLECTION)
141
+ .set("uri", mongoUri)
142
+ .set("collection", mongoCollection)
155
143
  .set("skip", 10)
156
144
  .set("incremental_field", Optional.of(Arrays.asList("account")));
157
145
 
@@ -162,8 +150,8 @@ public class TestMongodbInputPlugin
162
150
  public void checkInvalidQueryOption()
163
151
  {
164
152
  ConfigSource config = Exec.newConfigSource()
165
- .set("uri", MONGO_URI)
166
- .set("collection", MONGO_COLLECTION)
153
+ .set("uri", mongoUri)
154
+ .set("collection", mongoCollection)
167
155
  .set("query", "{\"key\":invalid_value}")
168
156
  .set("last_record", 0)
169
157
  .set("incremental_field", Optional.of(Arrays.asList("account")));
@@ -171,6 +159,63 @@ public class TestMongodbInputPlugin
171
159
  plugin.transaction(config, new Control());
172
160
  }
173
161
 
162
+ @Test(expected = ConfigException.class)
163
+ public void checkAggregationWithOtherOption()
164
+ {
165
+ ConfigSource config = Exec.newConfigSource()
166
+ .set("uri", mongoUri)
167
+ .set("collection", mongoCollection)
168
+ .set("query", "{\"key\":\"valid_value\"}")
169
+ .set("aggregation", "{$match: { account: { $gt: 32864}}}")
170
+ .set("incremental_field", Optional.of(Arrays.asList("account")));
171
+
172
+ plugin.transaction(config, new Control());
173
+ }
174
+
175
+ @Test
176
+ public void testCreateCredentialsSha1() throws Exception
177
+ {
178
+ PluginTask task = configForAuth().deepCopy()
179
+ .set("auth_method", "scram-sha-1")
180
+ .set("database", "db")
181
+ .loadConfig(PluginTask.class);
182
+
183
+ Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
184
+ createCredential.setAccessible(true);
185
+ MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
186
+ assertThat("SCRAM-SHA-1", is(credential.getMechanism()));
187
+ assertThat("db", is(credential.getSource()));
188
+ }
189
+
190
+ @Test
191
+ public void testCreateCredentialsSha1WithAuthSource() throws Exception
192
+ {
193
+ PluginTask task = configForAuth().deepCopy()
194
+ .set("auth_method", "scram-sha-1")
195
+ .set("database", "db")
196
+ .set("auth_source", "authdb")
197
+ .loadConfig(PluginTask.class);
198
+
199
+ Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
200
+ createCredential.setAccessible(true);
201
+ MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
202
+ assertThat("SCRAM-SHA-1", is(credential.getMechanism()));
203
+ assertThat("authdb", is(credential.getSource()));
204
+ }
205
+
206
+ @Test
207
+ public void testCreateCredentialsCr() throws Exception
208
+ {
209
+ PluginTask task = configForAuth().deepCopy()
210
+ .set("auth_method", "mongodb-cr")
211
+ .loadConfig(PluginTask.class);
212
+
213
+ Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
214
+ createCredential.setAccessible(true);
215
+ MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
216
+ assertThat("MONGODB-CR", is(credential.getMechanism()));
217
+ }
218
+
174
219
  @Test
175
220
  public void testResume()
176
221
  {
@@ -205,8 +250,8 @@ public class TestMongodbInputPlugin
205
250
  {
206
251
  PluginTask task = config.loadConfig(PluginTask.class);
207
252
 
208
- dropCollection(task, MONGO_COLLECTION);
209
- createCollection(task, MONGO_COLLECTION);
253
+ dropCollection(task, mongoCollection);
254
+ createCollection(task, mongoCollection);
210
255
  insertDocument(task, createValidDocuments());
211
256
 
212
257
  plugin.transaction(config, new Control());
@@ -217,13 +262,13 @@ public class TestMongodbInputPlugin
217
262
  public void testRunWithLimit() throws Exception
218
263
  {
219
264
  ConfigSource config = Exec.newConfigSource()
220
- .set("uri", MONGO_URI)
221
- .set("collection", MONGO_COLLECTION)
265
+ .set("uri", mongoUri)
266
+ .set("collection", mongoCollection)
222
267
  .set("limit", 1);
223
268
  PluginTask task = config.loadConfig(PluginTask.class);
224
269
 
225
- dropCollection(task, MONGO_COLLECTION);
226
- createCollection(task, MONGO_COLLECTION);
270
+ dropCollection(task, mongoCollection);
271
+ createCollection(task, mongoCollection);
227
272
  insertDocument(task, createValidDocuments());
228
273
 
229
274
  plugin.transaction(config, new Control());
@@ -234,14 +279,14 @@ public class TestMongodbInputPlugin
234
279
  public void testRunWithLimitSkip() throws Exception
235
280
  {
236
281
  ConfigSource config = Exec.newConfigSource()
237
- .set("uri", MONGO_URI)
238
- .set("collection", MONGO_COLLECTION)
282
+ .set("uri", mongoUri)
283
+ .set("collection", mongoCollection)
239
284
  .set("limit", 3)
240
285
  .set("skip", 1);
241
286
  PluginTask task = config.loadConfig(PluginTask.class);
242
287
 
243
- dropCollection(task, MONGO_COLLECTION);
244
- createCollection(task, MONGO_COLLECTION);
288
+ dropCollection(task, mongoCollection);
289
+ createCollection(task, mongoCollection);
245
290
  insertDocument(task, createValidDocuments());
246
291
 
247
292
  plugin.transaction(config, new Control());
@@ -251,7 +296,7 @@ public class TestMongodbInputPlugin
251
296
  @Test
252
297
  public void testRunWithConnectionParams() throws Exception
253
298
  {
254
- MongoClientURI uri = new MongoClientURI(MONGO_URI);
299
+ MongoClientURI uri = new MongoClientURI(mongoUri);
255
300
  String host = uri.getHosts().get(0);
256
301
  Integer port = (host.split(":")[1] != null) ? Integer.valueOf(host.split(":")[1]) : 27017;
257
302
  ConfigSource config = Exec.newConfigSource()
@@ -259,11 +304,11 @@ public class TestMongodbInputPlugin
259
304
  .set("user", uri.getUsername())
260
305
  .set("password", uri.getPassword())
261
306
  .set("database", uri.getDatabase())
262
- .set("collection", MONGO_COLLECTION);
307
+ .set("collection", mongoCollection);
263
308
  PluginTask task = config.loadConfig(PluginTask.class);
264
309
 
265
- dropCollection(task, MONGO_COLLECTION);
266
- createCollection(task, MONGO_COLLECTION);
310
+ dropCollection(task, mongoCollection);
311
+ createCollection(task, mongoCollection);
267
312
  insertDocument(task, createValidDocuments());
268
313
 
269
314
  plugin.transaction(config, new Control());
@@ -274,13 +319,13 @@ public class TestMongodbInputPlugin
274
319
  public void testRunWithIncrementalLoad() throws Exception
275
320
  {
276
321
  ConfigSource config = Exec.newConfigSource()
277
- .set("uri", MONGO_URI)
278
- .set("collection", MONGO_COLLECTION)
322
+ .set("uri", mongoUri)
323
+ .set("collection", mongoCollection)
279
324
  .set("incremental_field", Optional.of(Arrays.asList("int32_field")));
280
325
  PluginTask task = config.loadConfig(PluginTask.class);
281
326
 
282
- dropCollection(task, MONGO_COLLECTION);
283
- createCollection(task, MONGO_COLLECTION);
327
+ dropCollection(task, mongoCollection);
328
+ createCollection(task, mongoCollection);
284
329
  insertDocument(task, createValidDocuments());
285
330
 
286
331
  ConfigDiff diff = plugin.transaction(config, new Control());
@@ -293,15 +338,15 @@ public class TestMongodbInputPlugin
293
338
  public void testRunWithLimitIncrementalLoad() throws Exception
294
339
  {
295
340
  ConfigSource config = Exec.newConfigSource()
296
- .set("uri", MONGO_URI)
297
- .set("collection", MONGO_COLLECTION)
341
+ .set("uri", mongoUri)
342
+ .set("collection", mongoCollection)
298
343
  .set("id_field_name", "int32_field")
299
344
  .set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")))
300
345
  .set("limit", 1);
301
346
  PluginTask task = config.loadConfig(PluginTask.class);
302
347
 
303
- dropCollection(task, MONGO_COLLECTION);
304
- createCollection(task, MONGO_COLLECTION);
348
+ dropCollection(task, mongoCollection);
349
+ createCollection(task, mongoCollection);
305
350
  insertDocument(task, createValidDocuments());
306
351
 
307
352
  ConfigDiff diff = plugin.transaction(config, new Control());
@@ -321,8 +366,8 @@ public class TestMongodbInputPlugin
321
366
  previousLastRecord.put("datetime_field", "{$date=2015-01-27T10:23:49.000Z}");
322
367
  previousLastRecord.put("boolean_field", true);
323
368
  ConfigSource config = Exec.newConfigSource()
324
- .set("uri", MONGO_URI)
325
- .set("collection", MONGO_COLLECTION)
369
+ .set("uri", mongoUri)
370
+ .set("collection", mongoCollection)
326
371
  .set("id_field_name", "int32_field")
327
372
  .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
328
373
  .set("incremental_field", Optional.of(Arrays.asList("int32_field", "datetime_field", "boolean_field")))
@@ -330,8 +375,8 @@ public class TestMongodbInputPlugin
330
375
 
331
376
  PluginTask task = config.loadConfig(PluginTask.class);
332
377
 
333
- dropCollection(task, MONGO_COLLECTION);
334
- createCollection(task, MONGO_COLLECTION);
378
+ dropCollection(task, mongoCollection);
379
+ createCollection(task, mongoCollection);
335
380
  insertDocument(task, createValidDocuments());
336
381
 
337
382
  ConfigDiff diff = plugin.transaction(config, new Control());
@@ -346,13 +391,13 @@ public class TestMongodbInputPlugin
346
391
  public void testRunWithIncrementalLoadUnsupportedType() throws Exception
347
392
  {
348
393
  ConfigSource config = Exec.newConfigSource()
349
- .set("uri", MONGO_URI)
350
- .set("collection", MONGO_COLLECTION)
394
+ .set("uri", mongoUri)
395
+ .set("collection", mongoCollection)
351
396
  .set("incremental_field", Optional.of(Arrays.asList("document_field")));
352
397
  PluginTask task = config.loadConfig(PluginTask.class);
353
398
 
354
- dropCollection(task, MONGO_COLLECTION);
355
- createCollection(task, MONGO_COLLECTION);
399
+ dropCollection(task, mongoCollection);
400
+ createCollection(task, mongoCollection);
356
401
  insertDocument(task, createValidDocuments());
357
402
 
358
403
  plugin.transaction(config, new Control());
@@ -362,14 +407,14 @@ public class TestMongodbInputPlugin
362
407
  public void testRunWithUnsupportedType() throws Exception
363
408
  {
364
409
  ConfigSource config = Exec.newConfigSource()
365
- .set("uri", MONGO_URI)
366
- .set("collection", MONGO_COLLECTION)
410
+ .set("uri", mongoUri)
411
+ .set("collection", mongoCollection)
367
412
  .set("stop_on_invalid_record", true);
368
413
 
369
414
  PluginTask task = config.loadConfig(PluginTask.class);
370
415
 
371
- dropCollection(task, MONGO_COLLECTION);
372
- createCollection(task, MONGO_COLLECTION);
416
+ dropCollection(task, mongoCollection);
417
+ createCollection(task, mongoCollection);
373
418
 
374
419
  List<Document> documents = new ArrayList<>();
375
420
  documents.add(
@@ -380,6 +425,25 @@ public class TestMongodbInputPlugin
380
425
  plugin.transaction(config, new Control());
381
426
  }
382
427
 
428
+ @Test
429
+ public void testRunWithAggregation() throws Exception
430
+ {
431
+ ConfigSource config = Exec.newConfigSource()
432
+ .set("uri", mongoUri)
433
+ .set("collection", mongoCollection)
434
+ .set("id_field_name", "int32_field")
435
+ .set("aggregation", "{ $match: {\"int32_field\":{\"$gte\":5 },} }");
436
+
437
+ PluginTask task = config.loadConfig(PluginTask.class);
438
+
439
+ dropCollection(task, mongoCollection);
440
+ createCollection(task, mongoCollection);
441
+ insertDocument(task, createValidDocuments());
442
+
443
+ plugin.transaction(config, new Control());
444
+ assertValidRecordsForAggregation(getFieldSchema(), output);
445
+ }
446
+
383
447
  @Test
384
448
  public void testNormalize() throws Exception
385
449
  {
@@ -425,16 +489,16 @@ public class TestMongodbInputPlugin
425
489
  public void testBuildIncrementalCondition() throws Exception
426
490
  {
427
491
  PluginTask task = config().loadConfig(PluginTask.class);
428
- dropCollection(task, MONGO_COLLECTION);
429
- createCollection(task, MONGO_COLLECTION);
492
+ dropCollection(task, mongoCollection);
493
+ createCollection(task, mongoCollection);
430
494
  insertDocument(task, createValidDocuments());
431
495
 
432
496
  Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
433
497
  method.setAccessible(true);
434
498
 
435
499
  ConfigSource config = Exec.newConfigSource()
436
- .set("uri", MONGO_URI)
437
- .set("collection", MONGO_COLLECTION)
500
+ .set("uri", mongoUri)
501
+ .set("collection", mongoCollection)
438
502
  .set("incremental_field", Optional.of(Arrays.asList("account")));
439
503
  task = config.loadConfig(PluginTask.class);
440
504
  Map<String, String> actual = (Map<String, String>) method.invoke(plugin, task);
@@ -452,8 +516,8 @@ public class TestMongodbInputPlugin
452
516
  innerRecord.put("$date", "2015-01-27T19:23:49Z");
453
517
  lastRecord.put("datetime_field", innerRecord);
454
518
  config = Exec.newConfigSource()
455
- .set("uri", MONGO_URI)
456
- .set("collection", MONGO_COLLECTION)
519
+ .set("uri", mongoUri)
520
+ .set("collection", mongoCollection)
457
521
  .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
458
522
  .set("incremental_field", Optional.of(Arrays.asList("_id", "int32_field", "datetime_field")))
459
523
  .set("last_record", Optional.of(lastRecord));
@@ -471,14 +535,14 @@ public class TestMongodbInputPlugin
471
535
  lastRecord.put("double_field", "0");
472
536
 
473
537
  ConfigSource config = Exec.newConfigSource()
474
- .set("uri", MONGO_URI)
475
- .set("collection", MONGO_COLLECTION)
538
+ .set("uri", mongoUri)
539
+ .set("collection", mongoCollection)
476
540
  .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
477
541
  .set("incremental_field", Optional.of(Arrays.asList("double_field")))
478
542
  .set("last_record", Optional.of(lastRecord));
479
543
  PluginTask task = config.loadConfig(PluginTask.class);
480
- dropCollection(task, MONGO_COLLECTION);
481
- createCollection(task, MONGO_COLLECTION);
544
+ dropCollection(task, mongoCollection);
545
+ createCollection(task, mongoCollection);
482
546
  insertDocument(task, createValidDocuments());
483
547
 
484
548
  Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
@@ -498,13 +562,13 @@ public class TestMongodbInputPlugin
498
562
  lastRecord.put("double_field", "0");
499
563
 
500
564
  ConfigSource config = Exec.newConfigSource()
501
- .set("uri", MONGO_URI)
502
- .set("collection", MONGO_COLLECTION)
565
+ .set("uri", mongoUri)
566
+ .set("collection", mongoCollection)
503
567
  .set("incremental_field", Optional.of(Arrays.asList("invalid_field")))
504
568
  .set("last_record", Optional.of(lastRecord));
505
569
  PluginTask task = config.loadConfig(PluginTask.class);
506
- dropCollection(task, MONGO_COLLECTION);
507
- createCollection(task, MONGO_COLLECTION);
570
+ dropCollection(task, mongoCollection);
571
+ createCollection(task, mongoCollection);
508
572
 
509
573
  Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
510
574
  method.setAccessible(true);
@@ -542,8 +606,17 @@ public class TestMongodbInputPlugin
542
606
  private ConfigSource config()
543
607
  {
544
608
  return Exec.newConfigSource()
545
- .set("uri", MONGO_URI)
546
- .set("collection", MONGO_COLLECTION);
609
+ .set("uri", mongoUri)
610
+ .set("collection", mongoCollection);
611
+ }
612
+
613
+ private ConfigSource configForAuth()
614
+ {
615
+ return Exec.newConfigSource()
616
+ .set("database", "db")
617
+ .set("collection", mongoCollection)
618
+ .set("user", "abcde")
619
+ .set("password", "passw0rd");
547
620
  }
548
621
 
549
622
  private List<Document> createValidDocuments() throws Exception
@@ -604,6 +677,11 @@ public class TestMongodbInputPlugin
604
677
  assertValidRecords(schema, output, 5, 0);
605
678
  }
606
679
 
680
+ private void assertValidRecordsForAggregation(Schema schema, MockPageOutput output) throws Exception
681
+ {
682
+ assertValidRecords(schema, output, 1, 4);
683
+ }
684
+
607
685
  private void assertValidRecords(Schema schema, MockPageOutput output, int limit, int skip) throws Exception
608
686
  {
609
687
  int maxRecordSize = 5;
@@ -1,6 +1,6 @@
1
1
  in:
2
2
  type: mongodb
3
- uri: mongodb://localhost:27017/my_database
3
+ uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
4
4
  collection: "my_collection"
5
5
  projection: '{ "_id": 0, "name": 1, "rank": 1 }'
6
6
  sort: '{ rank: 1 }'
@@ -1,6 +1,6 @@
1
1
  in:
2
2
  type: mongodb
3
- uri: mongodb://localhost:27017/my_database
3
+ uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
4
4
  collection: "my_collection"
5
5
  json_column_name: "json"
6
6
  query: '{ rank: { $gte: 3 } }'
@@ -1,6 +1,6 @@
1
1
  in:
2
2
  type: mongodb
3
- uri: mongodb://localhost:27017/my_database
3
+ uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
4
4
  collection: "my_collection"
5
5
  json_column_name: "json"
6
6
  query: '{ rank: { $gte: 3 } }'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-mongodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kazuyuki Honda
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-05-08 00:00:00.000000000 Z
11
+ date: 2018-12-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -53,11 +53,13 @@ files:
53
53
  - build.gradle
54
54
  - config/checkstyle/checkstyle.xml
55
55
  - config/checkstyle/default.xml
56
+ - docker-compose.yml
56
57
  - gradle/wrapper/gradle-wrapper.jar
57
58
  - gradle/wrapper/gradle-wrapper.properties
58
59
  - gradlew
59
60
  - gradlew.bat
60
61
  - lib/embulk/input/mongodb.rb
62
+ - src/main/java/org/embulk/input/mongodb/AuthMethod.java
61
63
  - src/main/java/org/embulk/input/mongodb/HostTask.java
62
64
  - src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java
63
65
  - src/main/java/org/embulk/input/mongodb/PluginTask.java
@@ -70,8 +72,8 @@ files:
70
72
  - src/test/resources/id_field_name.yml
71
73
  - src/test/resources/id_field_name_expected.csv
72
74
  - src/test/resources/my_collection.jsonl
73
- - classpath/embulk-input-mongodb-0.6.1.jar
74
- - classpath/mongo-java-driver-3.6.1.jar
75
+ - classpath/embulk-input-mongodb-0.7.0.jar
76
+ - classpath/mongo-java-driver-3.8.1.jar
75
77
  homepage: https://github.com/hakobera/embulk-input-mongodb
76
78
  licenses:
77
79
  - MIT