embulk-input-mongodb 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fc232a73722d1157d53b5d4de1374cadd11cd234
-  data.tar.gz: 7792d6114a467b793fcacfaa9a64df7bfe1e5aa3
+  metadata.gz: 669b8b7010ff112526a166dd4eef85a50656e77f
+  data.tar.gz: adea0810c4f04ee5cec5ec8ba7414557a8f179a2
 SHA512:
-  metadata.gz: 1af752c57c9fc04d2d65e9665d6c11c739be8c2805c07b6bd65fad7a244685bf78eaf8da6a5a84e59d48e1fd18db671690789ace2d9cd7c6d62d2e1afcb5c390
-  data.tar.gz: 2cb4e589f0e8d524414b6a51135b4931ca91514f811b036c5d1452e2933045a4a86d7c3a615ce31eef85d8c23e66c1ba78d8ad952d9a5dd324e9c15682933038
+  metadata.gz: b8fad0bab206efc47861e6cb714c1def66cbfcb7440e042584daf454f84ef2673cdd2c8482bb3edfe3920d3bf82827c99c6cccb06d2890ea2a823be8122fffe8
+  data.tar.gz: 277157cc6382dbb98795f6e3e0ef3aaa14a31faec1f663069d9bc133f4b246a6cb73da39a2e5c5b7f80a36ae4ade7145cd01464b9c9feec896eb205dbec2ba7b

data/.travis.yml

@@ -26,7 +26,8 @@ cache:
 before_install:
   - cat /etc/hosts # optionally check the content *before*
   - sudo hostname "$(hostname | cut -c1-63)"
-  - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts
+  - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts > /tmp/hosts
+  - sudo mv /tmp/hosts /etc/hosts
   - cat /etc/hosts # optionally check the content *after*
 
 install:
@@ -42,14 +43,14 @@ install:
 
 before_script:
   - echo "Wait mongodb wakeup"
-  - sleep 10
+  - sleep 15
   - mkdir -p ./tmp
   - date
 
 script:
   - ./gradlew check
   - ./gradlew package
-  - mongoimport --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
+  - mongoimport --host 127.0.0.1 --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
   - |
     for target in basic full id_field_name
     do

data/CHANGELOG.md

@@ -0,0 +1,97 @@
+# Change Log
+
+## [v0.5.0](https://github.com/hakobera/embulk-input-mongodb/tree/v0.5.0) (2016-08-23)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.4.0...v0.5.0)
+
+**Implemented enhancements:**
+
+- Support separated URI params [\#24](https://github.com/hakobera/embulk-input-mongodb/pull/24) ([sakama](https://github.com/sakama))
+
+**Merged pull requests:**
+
+- Add --host option to mongoimport to prevent connection failure. [\#27](https://github.com/hakobera/embulk-input-mongodb/pull/27) ([hakobera](https://github.com/hakobera))
+- Add document about batch\_size config [\#25](https://github.com/hakobera/embulk-input-mongodb/pull/25) ([hakobera](https://github.com/hakobera))
+
+## [v0.4.0](https://github.com/hakobera/embulk-input-mongodb/tree/v0.4.0) (2016-07-08)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.3.2...v0.4.0)
+
+**Implemented enhancements:**
+
+- Support incremental load [\#16](https://github.com/hakobera/embulk-input-mongodb/pull/16) ([sakama](https://github.com/sakama))
+
+**Merged pull requests:**
+
+- Bumpup version to 0.4.0 [\#23](https://github.com/hakobera/embulk-input-mongodb/pull/23) ([hakobera](https://github.com/hakobera))
+
+## [v0.3.2](https://github.com/hakobera/embulk-input-mongodb/tree/v0.3.2) (2016-06-23)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.3.1...v0.3.2)
+
+**Implemented enhancements:**
+
+- Support Symbol type [\#19](https://github.com/hakobera/embulk-input-mongodb/issues/19)
+
+**Merged pull requests:**
+
+- Bumpup version to 0.3.2 [\#22](https://github.com/hakobera/embulk-input-mongodb/pull/22) ([hakobera](https://github.com/hakobera))
+- Release v0.3.1 [\#17](https://github.com/hakobera/embulk-input-mongodb/pull/17) ([hakobera](https://github.com/hakobera))
+- Support Symbol type [\#21](https://github.com/hakobera/embulk-input-mongodb/pull/21) ([hakobera](https://github.com/hakobera))
+
+## [v0.3.1](https://github.com/hakobera/embulk-input-mongodb/tree/v0.3.1) (2016-06-21)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.3.0...v0.3.1)
+
+**Implemented enhancements:**
+
+- Support id\_field\_name config [\#15](https://github.com/hakobera/embulk-input-mongodb/pull/15) ([hakobera](https://github.com/hakobera))
+
+**Closed issues:**
+
+- readName can only be called when State is NAME, not when State is VALUE. [\#18](https://github.com/hakobera/embulk-input-mongodb/issues/18)
+
+## [v0.3.0](https://github.com/hakobera/embulk-input-mongodb/tree/v0.3.0) (2016-06-13)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.2.0...v0.3.0)
+
+**Merged pull requests:**
+
+- Add checkstyle plugin and format code [\#14](https://github.com/hakobera/embulk-input-mongodb/pull/14) ([sakama](https://github.com/sakama))
+- Bumpup version to 0.3.0 [\#13](https://github.com/hakobera/embulk-input-mongodb/pull/13) ([hakobera](https://github.com/hakobera))
+- Add unit test for existing code, \#9 and \#10 [\#11](https://github.com/hakobera/embulk-input-mongodb/pull/11) ([sakama](https://github.com/sakama))
+- Change schema convert logic [\#10](https://github.com/hakobera/embulk-input-mongodb/pull/10) ([sakama](https://github.com/sakama))
+
+## [v0.2.0](https://github.com/hakobera/embulk-input-mongodb/tree/v0.2.0) (2016-05-30)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.1.2...v0.2.0)
+
+**Implemented enhancements:**
+
+- Support JSON type [\#6](https://github.com/hakobera/embulk-input-mongodb/issues/6)
+
+**Merged pull requests:**
+
+- Prepare release for v0.2.0 [\#12](https://github.com/hakobera/embulk-input-mongodb/pull/12) ([hakobera](https://github.com/hakobera))
+- Add minimum limit to batch\_size and changed not to allow null [\#9](https://github.com/hakobera/embulk-input-mongodb/pull/9) ([sakama](https://github.com/sakama))
+- Refactor some codes [\#8](https://github.com/hakobera/embulk-input-mongodb/pull/8) ([sakama](https://github.com/sakama))
+- Fix example config on readme [\#5](https://github.com/hakobera/embulk-input-mongodb/pull/5) ([hakobera](https://github.com/hakobera))
+- Support JSON type [\#7](https://github.com/hakobera/embulk-input-mongodb/pull/7) ([hakobera](https://github.com/hakobera))
+
+## [v0.1.2](https://github.com/hakobera/embulk-input-mongodb/tree/v0.1.2) (2015-09-10)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.1.1...v0.1.2)
+
+**Merged pull requests:**
+
+- Change build option for JDK7 [\#4](https://github.com/hakobera/embulk-input-mongodb/pull/4) ([hakobera](https://github.com/hakobera))
+- Set homepage info [\#3](https://github.com/hakobera/embulk-input-mongodb/pull/3) ([hakobera](https://github.com/hakobera))
+
+## [v0.1.1](https://github.com/hakobera/embulk-input-mongodb/tree/v0.1.1) (2015-09-06)
+[Full Changelog](https://github.com/hakobera/embulk-input-mongodb/compare/v0.1.0...v0.1.1)
+
+## [v0.1.0](https://github.com/hakobera/embulk-input-mongodb/tree/v0.1.0) (2015-09-06)
+**Implemented enhancements:**
+
+- Fix projection logic and better logging [\#2](https://github.com/hakobera/embulk-input-mongodb/pull/2) ([hakobera](https://github.com/hakobera))
+
+**Merged pull requests:**
+
+- Run test on travis [\#1](https://github.com/hakobera/embulk-input-mongodb/pull/1) ([hakobera](https://github.com/hakobera))
+
+
+
+\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)*

data/README.md

@@ -14,7 +14,16 @@ This plugin only works with embulk >= 0.8.8.
 
 ## Configuration
 
-- **uri**: [MongoDB connection string URI](http://docs.mongodb.org/manual/reference/connection-string/) (e.g. 'mongodb://localhost:27017/mydb') (string, required)
+- Connection parameters
+  One of them is required.
+  
+  - use MongoDB connection string URI
+    - **uri**: [MongoDB connection string URI](http://docs.mongodb.org/manual/reference/connection-string/) (e.g. 'mongodb://localhost:27017/mydb') (string, required)
+  - use separated URI parameters
+    - **hosts**: list of hosts. `hosts` are pairs of host(string, required) and port(integer, optional, default: 27017)
+    - **user**: (string, optional)
+    - **password**:  (string, optional)
+    - **database**:  (string, required)
 - **collection**: source collection name (string, required)
 - **fields**: **(deprecated)** ~~hash records that has the following two fields (array, required)~~
   ~~- name: Name of the column~~
@@ -24,12 +33,13 @@ This plugin only works with embulk >= 0.8.8.
     ~~- double~~
     ~~- string~~
     ~~- timestamp~~
-- **id_field_name** (string, optional, default: "_id") Name of Object ID field name. Set if you want to change the default name `_id` 
-- **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
+- **id_field_name** Name of Object ID field name. Set if you want to change the default name `_id` (string, optional, default: "_id")
+- **query**: A JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
 - **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
-- **sort**: ordering of results (string, optional)
-- **incremental_field** list of field name (list, optional, can't use with sort option)
-- **last_record** (hash, optional) last loaded record for incremental load
+- **sort**: Ordering of results (string, optional)
+- **batch_size**: Limits the number of objects returned in one [batch](http://api.mongodb.com/java/current/com/mongodb/DBCursor.html#batchSize-int-) (integer, optional, default: 10000)
+- **incremental_field** List of field name (list, optional, can't use with sort option)
+- **last_record** Last loaded record for incremental load (hash, optional)
 - **stop_on_invalid_record** Stop bulk load transaction if a document includes invalid record (such as unsupported object type) (boolean, optional, default: false)
 - **json_column_name**: column name used in outputs (string, optional, default: "json")
 
@@ -37,6 +47,8 @@ This plugin only works with embulk >= 0.8.8.
 
 ### Exporting all objects
 
+#### Specify with MongoDB connection string URI.
+
 ```yaml
 in:
   type: mongodb
@@ -44,6 +56,20 @@ in:
   collection: "my_collection"
 ```
 
+#### Specify with separated URI parameters.
+
+```yaml
+in:
+  type: mongodb
+  hosts:
+  - {host: localhost, port: 27017}
+  - {host: example.com, port: 27017}
+  user: myuser
+  password: mypassword
+  database: my_database
+  collection: "my_collection"
+```
+
 ### Filtering documents by query and projection
 
 ```yaml
@@ -187,4 +213,4 @@ $ launchctl load ~/Library/LaunchAgents/environment.plist
 $ launchctl getenv MONGO_URI //try to get value.
 
 Then start your applications.
-```
+```

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.4.0"
+version = "0.5.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -5,7 +5,9 @@ import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
+import com.mongodb.MongoCredential;
 import com.mongodb.MongoException;
+import com.mongodb.ServerAddress;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
@@ -42,6 +44,8 @@ import javax.validation.constraints.Min;
 
 import java.io.IOException;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -50,12 +54,40 @@ import java.util.Map;
 public class MongodbInputPlugin
         implements InputPlugin
 {
+    public interface HostTask
+            extends Task
+    {
+        @Config("host")
+        String getHost();
+
+        @Config("port")
+        @ConfigDefault("27017")
+        int getPort();
+    }
+
     public interface PluginTask
             extends Task
     {
         // MongoDB connection string URI
         @Config("uri")
-        String getUri();
+        @ConfigDefault("null")
+        Optional<String> getUri();
+
+        @Config("hosts")
+        @ConfigDefault("null")
+        Optional<List<HostTask>> getHosts();
+
+        @Config("user")
+        @ConfigDefault("null")
+        Optional<String> getUser();
+
+        @Config("password")
+        @ConfigDefault("null")
+        Optional<String> getPassword();
+
+        @Config("database")
+        @ConfigDefault("null")
+        Optional<String> getDatabase();
 
         @Config("collection")
         String getCollection();
@@ -261,15 +293,56 @@ public class MongodbInputPlugin
 
     private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException
     {
-        MongoClientURI uri = new MongoClientURI(task.getUri());
-        MongoClient mongoClient = new MongoClient(uri);
+        MongoClient mongoClient;
+        String database;
+
+        if (!task.getUri().isPresent() && !task.getHosts().isPresent()) {
+            throw new ConfigException("'uri' or 'hosts' is required");
+        }
+
+        if (task.getUri().isPresent()) {
+            MongoClientURI uri = new MongoClientURI(task.getUri().get());
+            database = uri.getDatabase();
+            mongoClient = new MongoClient(uri);
+        }
+        else {
+            mongoClient = createClientFromParams(task);
+            database = task.getDatabase().get();
+        }
 
-        MongoDatabase db = mongoClient.getDatabase(uri.getDatabase());
+        MongoDatabase db = mongoClient.getDatabase(database);
         // Get collection count for throw Exception
         db.getCollection(task.getCollection()).count();
         return db;
     }
 
+    private MongoClient createClientFromParams(PluginTask task)
+    {
+        if (!task.getHosts().isPresent()) {
+            throw new ConfigException("'hosts' option's value is required but empty");
+        }
+        if (!task.getDatabase().isPresent()) {
+            throw new ConfigException("'database' option's value is required but empty");
+        }
+
+        List<ServerAddress> addresses = new ArrayList<>();
+        for (HostTask host : task.getHosts().get()) {
+            addresses.add(new ServerAddress(host.getHost(), host.getPort()));
+        }
+
+        if (task.getUser().isPresent()) {
+            MongoCredential credential = MongoCredential.createCredential(
+                    task.getUser().get(),
+                    task.getDatabase().get(),
+                    task.getPassword().get().toCharArray()
+            );
+            return new MongoClient(addresses, Arrays.asList(credential));
+        }
+        else {
+            return new MongoClient(addresses);
+        }
+    }
+
     private Map<String, String> buildIncrementalCondition(PluginTask task)
     {
         Map<String, String> result = new HashMap<>();

data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java

@@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
+import com.mongodb.MongoClientURI;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoDatabase;
 import org.bson.BsonBinary;
@@ -198,6 +200,28 @@ public class TestMongodbInputPlugin
         assertValidRecords(getFieldSchema(), output);
     }
 
+    @Test
+    public void testRunWithConnectionParams() throws Exception
+    {
+        MongoClientURI uri = new MongoClientURI(MONGO_URI);
+        String host = uri.getHosts().get(0);
+        Integer port = (host.split(":")[1] != null) ? Integer.valueOf(host.split(":")[1]) : 27017;
+        ConfigSource config = Exec.newConfigSource()
+                .set("hosts", Arrays.asList(ImmutableMap.of("host", host.split(":")[0], "port", port)))
+                .set("user", uri.getUsername())
+                .set("password", uri.getPassword())
+                .set("database", uri.getDatabase())
+                .set("collection", MONGO_COLLECTION);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        plugin.transaction(config, new Control());
+        assertValidRecords(getFieldSchema(), output);
+    }
+
     @Test
     public void testRunWithIncrementalLoad() throws Exception
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-08 00:00:00.000000000 Z
+date: 2016-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -47,6 +47,7 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .travis.yml
+- CHANGELOG.md
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -67,7 +68,7 @@ files:
 - src/test/resources/id_field_name.yml
 - src/test/resources/id_field_name_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.4.0.jar
+- classpath/embulk-input-mongodb-0.5.0.jar
 - classpath/mongo-java-driver-3.2.2.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses: