embulk-input-mongodb 0.6.1 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c0c287fc6caae40f480044e097b35c22748be2d
-  data.tar.gz: 31c40a09bfd9aa1d3333105a49b02807c7fefb5b
+  metadata.gz: ad8c05dd58ee2ce347781dd2d519aed6a9f08d9a
+  data.tar.gz: bd43fe1ba4a5a18d3f577130a4c8a82e3e96184c
 SHA512:
-  metadata.gz: 013109760ca0b9040ea5367384d61c283801c97c833c1da0b52625ee8927dadefe71c6cd0f15a4115e4184b4fbdf40faed93eb2406fc3385ed4fb85eef5c3150
-  data.tar.gz: 2e1e7a1dcac758ede878293a37c9a95a532cb177964c9208d05f64f97c9ad3b1040ec5f9ce37abb25ddc53af07d379abbe24b9edf6c67f2b12e3474c300c2530
+  metadata.gz: 5d0597d3ac72e926e6ceda10e15f07fbaf95c8594f24b7fd9facb65c5beade7843b1aa4e8d0d6cbad3891b4d00f5a2d7e3d4f857ea8ba3533a880a05d600a896
+  data.tar.gz: 44acde315c5bda2b634ec10af13b88121c413ca96b6549ccc1feb3ff6adc6806107d8fb8bc5cc8e336bd19957324f75f174d114db97706afdc833d32a7c8d346

data/.gitignore

@@ -1,4 +1,6 @@
 *~
+/db/
+/out/
 /pkg/
 /tmp/
 *.gemspec

data/.travis.yml

@@ -2,17 +2,18 @@ language: java
 
 jdk:
   - oraclejdk8
-  - oraclejdk7
-  - openjdk7
+
+services:
+  - docker
 
 env:
   global:
-    - MONGO_DATABASE=my_database
+    - MONGO_DATABASE=mydb
     - MONGO_COLLECTION=my_collection
-    - MONGO_URI=mongodb://localhost:27017/my_database
+    - MONGO_URI=mongodb://localhost:27017/mydb
+    - DOCKER_COMPOSE_VERSION=1.22.0
 
 sudo: required
-dist: precise
 
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock
@@ -22,36 +23,28 @@ cache:
     - $HOME/.gradle/caches/
     - $HOME/.gradle/wrapper/
 
-# Work around fix for buffer overflow error on OpenJDK7
-# ref: https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165131913
 before_install:
-  - cat /etc/hosts # optionally check the content *before*
-  - sudo hostname "$(hostname | cut -c1-63)"
-  - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts > /tmp/hosts
-  - sudo mv /tmp/hosts /etc/hosts
-  - cat /etc/hosts # optionally check the content *after*
+  - sudo rm /usr/local/bin/docker-compose
+  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - chmod +x docker-compose
+  - sudo mv docker-compose /usr/local/bin
 
 install:
-  - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-  - echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
-  - sudo apt-get update
-  - sudo apt-get install -y --force-yes mongodb-org
-  - mongod -version
   - curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.bintray.com/embulk/maven/embulk-0.8.8.jar"
   - chmod +x ~/.embulk/bin/embulk
   - export PATH="$HOME/.embulk/bin:$PATH"
   - embulk --version
 
 before_script:
-  - echo "Wait mongodb wakeup"
-  - sleep 15
   - mkdir -p ./tmp
   - date
+  - docker-compose up -d
+  - docker-compose ps
 
 script:
   - ./gradlew check
   - ./gradlew package
-  - mongoimport --host 127.0.0.1 --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
+  - mongoimport --host 127.0.0.1 -u mongo_user -p dbpass --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
   - |
     for target in basic full id_field_name
     do

data/README.md

@@ -21,6 +21,8 @@ This plugin only works with embulk >= 0.8.8.
     - **uri**: [MongoDB connection string URI](http://docs.mongodb.org/manual/reference/connection-string/) (e.g. 'mongodb://localhost:27017/mydb') (string, required)
   - use separated URI parameters
     - **hosts**: list of hosts. `hosts` are pairs of host(string, required) and port(integer, optional, default: 27017)
+    - **auth_method**: Auth method. One of `scram-sha-1`, `mongodb-cr`, `auto` (string, optional, default: null)
+    - **auth_source**: Auth source. The database name where the user is defined (string, optional, default: null)
     - **user**: (string, optional)
     - **password**:  (string, optional)
     - **database**:  (string, required)
@@ -37,6 +39,7 @@ This plugin only works with embulk >= 0.8.8.
 - **query**: A JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
 - **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
 - **sort**: Ordering of results (string, optional)
+- **aggregation**: Aggregation query (string, optional) See [Aggregation query](#aggregation-query) for more detail.
 - **batch_size**: Limits the number of objects returned in one [batch](http://api.mongodb.com/java/current/com/mongodb/DBCursor.html#batchSize-int-) (integer, optional, default: 10000)
 - **incremental_field** List of field name (list, optional, can't use with sort option)
 - **last_record** Last loaded record for incremental load (hash, optional)
@@ -45,6 +48,37 @@ This plugin only works with embulk >= 0.8.8.
 
 ## Example
 
+### Authentication
+
+#### Use separated URI prameters
+
+```yaml
+in:
+  type: mongodb
+  hosts:
+  - {host: localhost, port:27017}
+  user:  myuser
+  password: mypassword
+  database: my_database
+  auth_method: scram-sha-1
+  auth_source: auth_db
+  collection: "my_collection
+```
+
+If you set `auth_method: auto`, The client will negotiate the best mechanism based on the version of the server that the client is authenticating to.
+
+If the server version is 3.0 or higher, the driver will authenticate using the SCRAM-SHA-1 mechanism.
+
+Otherwise, the driver will authenticate using the MONGODB_CR mechanism. 
+
+#### Use URI String
+
+```yaml
+in:
+  type: mongodb
+  uri: mongodb://myuser:mypassword@localhost:27017/my_database?authMechanism=SCRAM-SHA-1&authSource=another_database
+```
+
 ### Exporting all objects
 
 #### Specify with MongoDB connection string URI.
@@ -128,6 +162,20 @@ in:
 $ embulk run /path/to/config.yml -c config-diff.yml
 ```
 
+### Aggregation query
+
+This plugin supports aggregation query. You can write complex query like below.
+
+`aggregation` option can't be used with `sort`, `limit`, `skip`, `query` option. Incremental load also doesn't work with aggregation query.
+
+```yaml
+in:
+  type: mongodb
+  aggregation: { $match: {"int32_field":{"$gte":5 },} }
+```
+
+See also [Aggregation — MongoDB Manual](https://docs.mongodb.com/manual/aggregation/) and [Aggregation Pipeline Stages — MongoDB Manual](https://docs.mongodb.com/manual/reference/operator/aggregation-pipeline/)
+
 ### Advanced usage with filter plugins
 
 ```yaml
@@ -173,44 +221,21 @@ $ ./gradlew gem
 
 ## Test
 
-```
-$ ./gradlew test  # -t to watch change of files and rebuild continuously
-```
+Firstly install Docker and Docker compose then `docker-compose up -d`,
+so that an MongoDB server will be locally launched then you can run tests with `./gradlew test`.
 
-To run unit tests, we need to configure the following environment variables.
+```sh
+$ docker-compose up -d
+Creating embulk-input-mongodb_server ... done
+Creating mongo-express               ... done
+Creating mongoClientTemp             ... done
 
-When environment variables are not set, skip almost test cases.
+$ docker-compose ps
+           Name                          Command                 State                            Ports
+------------------------------------------------------------------------------------------------------------------------------
+embulk-input-mongodb_server   docker-entrypoint.sh mongod      Up           0.0.0.0:27017->27017/tcp, 0.0.0.0:27018->27018/tcp
+mongo-express                 tini -- /docker-entrypoint ...   Up           0.0.0.0:8081->8081/tcp
+mongoClientTemp               docker-entrypoint.sh mongo ...   Restarting
 
-```
-MONGO_URI
-MONGO_COLLECTION
-```
-
-If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
-```xml
-$ vi ~/Library/LaunchAgents/environment.plist
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-  <key>Label</key>
-  <string>my.startup</string>
-  <key>ProgramArguments</key>
-  <array>
-    <string>sh</string>
-    <string>-c</string>
-    <string>
-      launchctl setenv MONGO_URI mongodb://myuser:mypassword@localhost:27017/my_database
-      launchctl setenv MONGO_COLLECTION my_collection
-    </string>
-  </array>
-  <key>RunAtLoad</key>
-  <true/>
-</dict>
-</plist>
-
-$ launchctl load ~/Library/LaunchAgents/environment.plist
-$ launchctl getenv MONGO_URI //try to get value.
-
-Then start your applications.
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
 ```

data/build.gradle

@@ -17,15 +17,15 @@ configurations {
     provided
 }
 
-version = "0.6.1"
+version = "0.7.0"
 
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.8"
     provided "org.embulk:embulk-core:0.8.8"
-    compile "org.mongodb:mongo-java-driver:3.6.1"
+    compile "org.mongodb:mongo-java-driver:3.8.1"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.8:tests"

data/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.1'
+services:
+  mongodb:
+    container_name: embulk-input-mongodb_server
+    image: mongo:3.6
+    restart: always
+    ports:
+    - 27017:27017
+    - 27018:27018
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: admin
+      MONGO_INITDB_ROOT_PASSWORD: tiger
+    volumes:
+    - mongodb-data:/data/db
+    - mongodb-configdb:/data/configdb
+
+  mongo-express:
+    container_name: mongo-express
+    image: mongo-express
+    restart: always
+    ports:
+    - 8081:8081
+    depends_on:
+      - mongodb
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: admin
+      ME_CONFIG_MONGODB_ADMINPASSWORD: tiger
+      ME_CONFIG_MONGODB_SERVER: mongodb
+
+  mongoClientTemp:
+    container_name: mongoClientTemp
+    image: mongo:3.6
+    depends_on:
+    - mongodb
+    # Sleep to wait MongoDB wake up on Travis CI
+    command: >
+      /bin/bash -c
+      "sleep 15 &&
+      mongo --host mongodb -u admin -p tiger admin --eval \"db.getSiblingDB('mydb').createUser({user:'mongo_user', pwd:'dbpass', roles:[{role:'readWrite',db:'mydb'}]});\""
+volumes:
+  mongodb-data:
+    driver: local
+  mongodb-configdb:
+    driver: local

data/src/main/java/org/embulk/input/mongodb/AuthMethod.java

@@ -0,0 +1,36 @@
+package org.embulk.input.mongodb;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.embulk.config.ConfigException;
+
+import java.util.Locale;
+
+public enum AuthMethod
+{
+    AUTO,
+    SCRAM_SHA_1,
+    MONGODB_CR;
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+    @JsonCreator
+    public static AuthMethod fromString(String value)
+    {
+        switch (value.replace("_", "-")) {
+            case "scram-sha-1":
+                return SCRAM_SHA_1;
+            case "mongodb-cr":
+                return MONGODB_CR;
+            case "auto":
+                return AUTO;
+            default:
+                throw new ConfigException(String.format("Unknown auth_method '%s'. Supported auth_method are scram-sha-1, mongodb-cr, auto", value));
+        }
+    }
+}

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -12,6 +12,7 @@ import com.mongodb.ServerAddress;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
+import org.bson.Document;
 import org.bson.codecs.configuration.CodecRegistries;
 import org.bson.codecs.configuration.CodecRegistry;
 import org.bson.conversions.Bson;
@@ -64,6 +65,24 @@ public class MongodbInputPlugin
             throw new ConfigException("both of skip and incremental_load can't be used together");
         }
 
+        if (task.getAggregation().isPresent()) {
+            if (task.getIncrementalField().isPresent()) {
+                throw new ConfigException("both of aggregation and incremental_load can't be used together");
+            }
+            if (!task.getSort().equals("{}")) {
+                throw new ConfigException("both of sort and aggregation can't be used together");
+            }
+            if (task.getLimit().isPresent()) {
+                throw new ConfigException("both of limit and aggregation can't be used together");
+            }
+            if (task.getSkip().isPresent()) {
+                throw new ConfigException("both of skip and aggregation can't be used together");
+            }
+            if (!task.getQuery().equals("{}")) {
+                throw new ConfigException("both of query and aggregation can't be used together");
+            }
+        }
+
         Map<String, String> newCondition = buildIncrementalCondition(task);
         task.setQuery(newCondition.get("query"));
         task.setSort(newCondition.get("sort"));
@@ -71,6 +90,9 @@ public class MongodbInputPlugin
         validateJsonField("projection", task.getProjection());
         validateJsonField("query", task.getQuery());
         validateJsonField("sort", task.getSort());
+        if (task.getAggregation().isPresent()) {
+            validateJsonField("aggrigation", task.getAggregation().get());
+        }
 
         // Connect once to throw ConfigException in earlier stage of excecution
         try {
@@ -146,20 +168,35 @@ public class MongodbInputPlugin
             log.trace("skip: {}", task.getSkip());
         }
 
-        try (MongoCursor<Value> cursor = collection
-                .find(query)
-                .projection(projection)
-                .sort(sort)
-                .batchSize(task.getBatchSize())
-                .limit(task.getLimit().or(0))
-                .skip(task.getSkip().or(0))
-                .iterator()) {
-            while (cursor.hasNext()) {
-                pageBuilder.setJson(column, cursor.next());
-                pageBuilder.addRecord();
+        if (task.getAggregation().isPresent()) {
+            Bson aggregationString = Document.parse(task.getAggregation().get());
+            List<Bson> aggregation = Arrays.asList(aggregationString);
+            try (MongoCursor<Value> cursor = collection
+                    .aggregate(aggregation).iterator()) {
+                while (cursor.hasNext()) {
+                    pageBuilder.setJson(column, cursor.next());
+                    pageBuilder.addRecord();
+                }
+            } catch (MongoException ex) {
+                Throwables.propagate(ex);
+            }
+        }
+        else {
+            try (MongoCursor<Value> cursor = collection
+                    .find(query)
+                    .projection(projection)
+                    .sort(sort)
+                    .batchSize(task.getBatchSize())
+                    .limit(task.getLimit().or(0))
+                    .skip(task.getSkip().or(0))
+                    .iterator()) {
+                while (cursor.hasNext()) {
+                    pageBuilder.setJson(column, cursor.next());
+                    pageBuilder.addRecord();
+                }
+            } catch (MongoException ex) {
+                Throwables.propagate(ex);
             }
-        } catch (MongoException ex) {
-            Throwables.propagate(ex);
         }
 
         pageBuilder.finish();
@@ -259,18 +296,48 @@ public class MongodbInputPlugin
         }
 
         if (task.getUser().isPresent()) {
-            MongoCredential credential = MongoCredential.createCredential(
-                    task.getUser().get(),
-                    task.getDatabase().get(),
-                    task.getPassword().get().toCharArray()
-            );
-            return new MongoClient(addresses, Arrays.asList(credential));
+            return new MongoClient(addresses, Arrays.asList(createCredential(task)));
         }
         else {
             return new MongoClient(addresses);
         }
     }
 
+    // @see http://mongodb.github.io/mongo-java-driver/3.0/driver-async/reference/connecting/authenticating/
+    private MongoCredential createCredential(PluginTask task)
+    {
+        MongoCredential credential;
+        String authSource = task.getAuthSource().isPresent() ? task.getAuthSource().get() : task.getDatabase().get();
+        AuthMethod authMethod = task.getAuthMethod().isPresent() ? task.getAuthMethod().get() : AuthMethod.AUTO;
+        switch (authMethod) {
+            case SCRAM_SHA_1:
+                credential = MongoCredential.createScramSha1Credential(
+                        task.getUser().get(),
+                        authSource,
+                        task.getPassword().get().toCharArray());
+                break;
+            case MONGODB_CR:
+                credential = MongoCredential.createMongoCRCredential(
+                        task.getUser().get(),
+                        authSource,
+                        task.getPassword().get().toCharArray());
+                break;
+            case AUTO:
+            default:
+                /* The client will negotiate the best mechanism based on the
+                 * version of the server that the client is authenticating to.
+                 * If the server version is 3.0 or higher, the driver will authenticate using the SCRAM-SHA-1 mechanism.
+                 * Otherwise, the driver will authenticate using the MONGODB_CR mechanism.
+                 */
+                credential = MongoCredential.createCredential(
+                        task.getUser().get(),
+                        authSource,
+                        task.getPassword().get().toCharArray()
+                );
+        }
+        return credential;
+    }
+
     private Map<String, String> buildIncrementalCondition(PluginTask task)
     {
         Map<String, String> result = new HashMap<>();
@@ -331,7 +398,7 @@ public class MongodbInputPlugin
     private void validateJsonField(String name, String jsonString)
     {
         try {
-            BasicDBObject.parse(jsonString);
+            Document.parse(jsonString);
         }
         catch (JsonParseException ex) {
             throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));

data/src/main/java/org/embulk/input/mongodb/PluginTask.java

@@ -25,6 +25,14 @@ public interface PluginTask
     @ConfigDefault("null")
     Optional<List<HostTask>> getHosts();
 
+    @Config("auth_method")
+    @ConfigDefault("null")
+    Optional<AuthMethod> getAuthMethod();
+
+    @Config("auth_source")
+    @ConfigDefault("null")
+    Optional<String> getAuthSource();
+
     @Config("user")
     @ConfigDefault("null")
     Optional<String> getUser();
@@ -53,6 +61,10 @@ public interface PluginTask
     String getQuery();
     void setQuery(String query);
 
+    @Config("aggregation")
+    @ConfigDefault("null")
+    Optional<String> getAggregation();
+
     @Config("sort")
     @ConfigDefault("\"{}\"")
     String getSort();

data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java

@@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import com.mongodb.MongoClientURI;
+import com.mongodb.MongoCredential;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
 import org.bson.BsonBinary;
@@ -31,7 +32,6 @@ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
 import org.embulk.spi.type.Types;
 import org.embulk.spi.util.Pages;
 import org.junit.Before;
-import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -53,8 +53,8 @@ import static org.junit.Assert.assertThat;
 
 public class TestMongodbInputPlugin
 {
-    private static String MONGO_URI;
-    private static String MONGO_COLLECTION;
+    private final String mongoUri = "mongodb://mongo_user:dbpass@localhost:27017/mydb";
+    private final String mongoCollection = "my_collection";
 
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
@@ -66,20 +66,8 @@ public class TestMongodbInputPlugin
     private MongodbInputPlugin plugin;
     private MockPageOutput output;
 
-    /*
-     * This test case requires environment variables
-     *   MONGO_URI
-     *   MONGO_COLLECTION
-     */
-    @BeforeClass
-    public static void initializeConstant()
-    {
-        MONGO_URI = System.getenv("MONGO_URI");
-        MONGO_COLLECTION = System.getenv("MONGO_COLLECTION");
-    }
-
     @Before
-    public void createResources() throws Exception
+    public void createResources()
     {
         config = config();
         plugin = new MongodbInputPlugin();
@@ -90,8 +78,8 @@ public class TestMongodbInputPlugin
     public void checkDefaultValues()
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION);
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection);
 
         PluginTask task = config.loadConfig(PluginTask.class);
         assertEquals("{}", task.getQuery());
@@ -109,7 +97,7 @@ public class TestMongodbInputPlugin
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("uri", null)
-                .set("collection", MONGO_COLLECTION);
+                .set("collection", mongoCollection);
 
         plugin.transaction(config, new Control());
     }
@@ -119,7 +107,7 @@ public class TestMongodbInputPlugin
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("uri", "mongodb://mongouser:password@non-exists.example.com:23490/test")
-                .set("collection", MONGO_COLLECTION);
+                .set("collection", mongoCollection);
 
         plugin.transaction(config, new Control());
     }
@@ -128,7 +116,7 @@ public class TestMongodbInputPlugin
     public void checkDefaultValuesCollectionIsNull()
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
+                .set("uri", mongoUri)
                 .set("collection", null);
 
         plugin.transaction(config, new Control());
@@ -138,8 +126,8 @@ public class TestMongodbInputPlugin
     public void checkSortCannotUseWithIncremental()
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("sort", "{ \"field1\": 1 }")
                 .set("incremental_field", Optional.of(Arrays.asList("account")));
 
@@ -150,8 +138,8 @@ public class TestMongodbInputPlugin
     public void checkSkipCannotUseWithIncremental()
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("skip", 10)
                 .set("incremental_field", Optional.of(Arrays.asList("account")));
 
@@ -162,8 +150,8 @@ public class TestMongodbInputPlugin
     public void checkInvalidQueryOption()
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("query", "{\"key\":invalid_value}")
                 .set("last_record", 0)
                 .set("incremental_field", Optional.of(Arrays.asList("account")));
@@ -171,6 +159,63 @@ public class TestMongodbInputPlugin
         plugin.transaction(config, new Control());
     }
 
+    @Test(expected = ConfigException.class)
+    public void checkAggregationWithOtherOption()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
+                .set("query", "{\"key\":\"valid_value\"}")
+                .set("aggregation", "{$match: { account: { $gt: 32864}}}")
+                .set("incremental_field", Optional.of(Arrays.asList("account")));
+
+        plugin.transaction(config, new Control());
+    }
+
+    @Test
+    public void testCreateCredentialsSha1() throws Exception
+    {
+        PluginTask task = configForAuth().deepCopy()
+                .set("auth_method", "scram-sha-1")
+                .set("database", "db")
+                .loadConfig(PluginTask.class);
+
+        Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
+        createCredential.setAccessible(true);
+        MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
+        assertThat("SCRAM-SHA-1", is(credential.getMechanism()));
+        assertThat("db", is(credential.getSource()));
+    }
+
+    @Test
+    public void testCreateCredentialsSha1WithAuthSource() throws Exception
+    {
+        PluginTask task = configForAuth().deepCopy()
+                .set("auth_method", "scram-sha-1")
+                .set("database", "db")
+                .set("auth_source", "authdb")
+                .loadConfig(PluginTask.class);
+
+        Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
+        createCredential.setAccessible(true);
+        MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
+        assertThat("SCRAM-SHA-1", is(credential.getMechanism()));
+        assertThat("authdb", is(credential.getSource()));
+    }
+
+    @Test
+    public void testCreateCredentialsCr() throws Exception
+    {
+        PluginTask task = configForAuth().deepCopy()
+                .set("auth_method", "mongodb-cr")
+                .loadConfig(PluginTask.class);
+
+        Method createCredential = MongodbInputPlugin.class.getDeclaredMethod("createCredential", PluginTask.class);
+        createCredential.setAccessible(true);
+        MongoCredential credential = (MongoCredential) createCredential.invoke(plugin, task);
+        assertThat("MONGODB-CR", is(credential.getMechanism()));
+    }
+
     @Test
     public void testResume()
     {
@@ -205,8 +250,8 @@ public class TestMongodbInputPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         plugin.transaction(config, new Control());
@@ -217,13 +262,13 @@ public class TestMongodbInputPlugin
     public void testRunWithLimit() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("limit", 1);
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         plugin.transaction(config, new Control());
@@ -234,14 +279,14 @@ public class TestMongodbInputPlugin
     public void testRunWithLimitSkip() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("limit", 3)
                 .set("skip", 1);
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         plugin.transaction(config, new Control());
@@ -251,7 +296,7 @@ public class TestMongodbInputPlugin
     @Test
     public void testRunWithConnectionParams() throws Exception
     {
-        MongoClientURI uri = new MongoClientURI(MONGO_URI);
+        MongoClientURI uri = new MongoClientURI(mongoUri);
         String host = uri.getHosts().get(0);
         Integer port = (host.split(":")[1] != null) ? Integer.valueOf(host.split(":")[1]) : 27017;
         ConfigSource config = Exec.newConfigSource()
@@ -259,11 +304,11 @@ public class TestMongodbInputPlugin
                 .set("user", uri.getUsername())
                 .set("password", uri.getPassword())
                 .set("database", uri.getDatabase())
-                .set("collection", MONGO_COLLECTION);
+                .set("collection", mongoCollection);
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         plugin.transaction(config, new Control());
@@ -274,13 +319,13 @@ public class TestMongodbInputPlugin
     public void testRunWithIncrementalLoad() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("incremental_field", Optional.of(Arrays.asList("int32_field")));
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         ConfigDiff diff = plugin.transaction(config, new Control());
@@ -293,15 +338,15 @@ public class TestMongodbInputPlugin
     public void testRunWithLimitIncrementalLoad() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("id_field_name", "int32_field")
                 .set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")))
                 .set("limit", 1);
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         ConfigDiff diff = plugin.transaction(config, new Control());
@@ -321,8 +366,8 @@ public class TestMongodbInputPlugin
         previousLastRecord.put("datetime_field", "{$date=2015-01-27T10:23:49.000Z}");
         previousLastRecord.put("boolean_field", true);
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("id_field_name", "int32_field")
                 .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
                 .set("incremental_field", Optional.of(Arrays.asList("int32_field", "datetime_field", "boolean_field")))
@@ -330,8 +375,8 @@ public class TestMongodbInputPlugin
 
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         ConfigDiff diff = plugin.transaction(config, new Control());
@@ -346,13 +391,13 @@ public class TestMongodbInputPlugin
     public void testRunWithIncrementalLoadUnsupportedType() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("incremental_field", Optional.of(Arrays.asList("document_field")));
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         plugin.transaction(config, new Control());
@@ -362,14 +407,14 @@ public class TestMongodbInputPlugin
     public void testRunWithUnsupportedType() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("stop_on_invalid_record", true);
 
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
 
         List<Document> documents = new ArrayList<>();
         documents.add(
@@ -380,6 +425,25 @@ public class TestMongodbInputPlugin
         plugin.transaction(config, new Control());
     }
 
+    @Test
+    public void testRunWithAggregation() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
+                .set("id_field_name", "int32_field")
+                .set("aggregation", "{ $match: {\"int32_field\":{\"$gte\":5 },} }");
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
+        insertDocument(task, createValidDocuments());
+
+        plugin.transaction(config, new Control());
+        assertValidRecordsForAggregation(getFieldSchema(), output);
+    }
+
     @Test
     public void testNormalize() throws Exception
     {
@@ -425,16 +489,16 @@ public class TestMongodbInputPlugin
     public void testBuildIncrementalCondition() throws Exception
     {
         PluginTask task = config().loadConfig(PluginTask.class);
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
         method.setAccessible(true);
 
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("incremental_field", Optional.of(Arrays.asList("account")));
         task = config.loadConfig(PluginTask.class);
         Map<String, String> actual = (Map<String, String>) method.invoke(plugin, task);
@@ -452,8 +516,8 @@ public class TestMongodbInputPlugin
         innerRecord.put("$date", "2015-01-27T19:23:49Z");
         lastRecord.put("datetime_field", innerRecord);
         config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
                 .set("incremental_field", Optional.of(Arrays.asList("_id", "int32_field", "datetime_field")))
                 .set("last_record", Optional.of(lastRecord));
@@ -471,14 +535,14 @@ public class TestMongodbInputPlugin
         lastRecord.put("double_field", "0");
 
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
                 .set("incremental_field", Optional.of(Arrays.asList("double_field")))
                 .set("last_record", Optional.of(lastRecord));
         PluginTask task = config.loadConfig(PluginTask.class);
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
         insertDocument(task, createValidDocuments());
 
         Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
@@ -498,13 +562,13 @@ public class TestMongodbInputPlugin
         lastRecord.put("double_field", "0");
 
         ConfigSource config = Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION)
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection)
                 .set("incremental_field", Optional.of(Arrays.asList("invalid_field")))
                 .set("last_record", Optional.of(lastRecord));
         PluginTask task = config.loadConfig(PluginTask.class);
-        dropCollection(task, MONGO_COLLECTION);
-        createCollection(task, MONGO_COLLECTION);
+        dropCollection(task, mongoCollection);
+        createCollection(task, mongoCollection);
 
         Method method = MongodbInputPlugin.class.getDeclaredMethod("buildIncrementalCondition", PluginTask.class);
         method.setAccessible(true);
@@ -542,8 +606,17 @@ public class TestMongodbInputPlugin
     private ConfigSource config()
     {
         return Exec.newConfigSource()
-                .set("uri", MONGO_URI)
-                .set("collection", MONGO_COLLECTION);
+                .set("uri", mongoUri)
+                .set("collection", mongoCollection);
+    }
+
+    private ConfigSource configForAuth()
+    {
+        return Exec.newConfigSource()
+            .set("database", "db")
+            .set("collection", mongoCollection)
+            .set("user", "abcde")
+            .set("password", "passw0rd");
     }
 
     private List<Document> createValidDocuments() throws Exception
@@ -604,6 +677,11 @@ public class TestMongodbInputPlugin
         assertValidRecords(schema, output, 5, 0);
     }
 
+    private void assertValidRecordsForAggregation(Schema schema, MockPageOutput output) throws Exception
+    {
+        assertValidRecords(schema, output, 1, 4);
+    }
+
     private void assertValidRecords(Schema schema, MockPageOutput output, int limit, int skip) throws Exception
     {
         int maxRecordSize = 5;

data/src/test/resources/basic.yml

@@ -1,6 +1,6 @@
 in:
   type: mongodb
-  uri: mongodb://localhost:27017/my_database
+  uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
   collection: "my_collection"
   projection: '{ "_id": 0, "name": 1, "rank": 1 }'
   sort: '{ rank: 1 }'

data/src/test/resources/full.yml

@@ -1,6 +1,6 @@
 in:
   type: mongodb
-  uri: mongodb://localhost:27017/my_database
+  uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
   collection: "my_collection"
   json_column_name: "json"
   query: '{ rank: { $gte: 3 } }'

data/src/test/resources/id_field_name.yml

@@ -1,6 +1,6 @@
 in:
   type: mongodb
-  uri: mongodb://localhost:27017/my_database
+  uri: mongodb://mongo_user:dbpass@localhost:27017/mydb
   collection: "my_collection"
   json_column_name: "json"
   query: '{ rank: { $gte: 3 } }'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-08 00:00:00.000000000 Z
+date: 2018-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -53,11 +53,13 @@ files:
 - build.gradle
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- docker-compose.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/input/mongodb.rb
+- src/main/java/org/embulk/input/mongodb/AuthMethod.java
 - src/main/java/org/embulk/input/mongodb/HostTask.java
 - src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java
 - src/main/java/org/embulk/input/mongodb/PluginTask.java
@@ -70,8 +72,8 @@ files:
 - src/test/resources/id_field_name.yml
 - src/test/resources/id_field_name_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.6.1.jar
-- classpath/mongo-java-driver-3.6.1.jar
+- classpath/embulk-input-mongodb-0.7.0.jar
+- classpath/mongo-java-driver-3.8.1.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses:
 - MIT