embulk-input-mongodb 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f727800501212f31289b40afce7c6df6dd55b720
-  data.tar.gz: 5aace5d182c198c5bd9cc937a6de69e69008b2ad
+  metadata.gz: eca1bf1b21b4a44bab520a7cb93f59df2c18ea82
+  data.tar.gz: 98f866d2313fe81dbf96da2fe61b9a303afcd698
 SHA512:
-  metadata.gz: d338e4b5b5c1e117f3b57f9200d1224360af298bf1104567768f9e901285066e8a6124b6811a1697ec27520bbe52f1f295e1c2397632f2462e2ffb28e5332e30
-  data.tar.gz: b017a18476c34ab56202584b63c61121e2a0a5098d13f11a25421e8b72974d979aeea3229ddff888836486ce644195fe6c22ad698f3955a64b37e82832ffc446
+  metadata.gz: 6251028e2ec5dc41523cbc43d3a06f57cde1a7b05a9a21a8098d169e6c3d0e05af74d12e325847fcbb74328f8c9cf98eb8e6ed167091cb2215bfbf2341fdb5ff
+  data.tar.gz: 7a6c238d53ab10733917a05a170a3d06aacb8cbbd8a631c87d86d5fa2a6a198e887df27b61bfebe10655eeefe48293bf0a174417435aed4926285d6ef5ba581f

data/.travis.yml

@@ -7,13 +7,21 @@ jdk:
 
 sudo: required
 
+# Work around fix for buffer overflow error on OpenJDK7
+# ref: https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165131913
+before_install:
+  - cat /etc/hosts # optionally check the content *before*
+  - sudo hostname "$(hostname | cut -c1-63)"
+  - sed -e "s/^\\(127\\.0\\.0\\.1.*\\)/\\1 $(hostname | cut -c1-63)/" /etc/hosts | sudo tee /etc/hosts
+  - cat /etc/hosts # optionally check the content *after*
+
 install:
   - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10
-  - echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.0.list
+  - echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
   - sudo apt-get update
-  - sudo apt-get install -y mongodb-org=3.0.6
+  - sudo apt-get install -y --force-yes mongodb-org
   - mongod -version
-  - curl --create-dirs -o ~/.embulk/bin/embulk -L "http://dl.embulk.org/embulk-latest.jar"
+  - curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.bintray.com/embulk/maven/embulk-0.8.8.jar"
   - chmod +x ~/.embulk/bin/embulk
   - export PATH="$HOME/.embulk/bin:$PATH"
   - embulk --version
@@ -27,5 +35,8 @@ before_script:
 
 script:
   - embulk run -L . src/test/resources/basic.yml
+  - cat tmp/basic000.00.csv
+  - cmp tmp/basic000.00.csv src/test/resources/basic_expected.csv || exit 1
   - embulk run -L . src/test/resources/full.yml
+  - cat tmp/full000.00.csv
   - cmp tmp/full000.00.csv src/test/resources/full_expected.csv || exit 1

data/README.md

@@ -4,17 +4,9 @@
 
 MongoDB input plugin for Embulk loads records from MongoDB.
 
-**CAUTION:** this plugin does not support array and object fields,
-because embulk does not supported these types yet.
-But these types will be supported, so when it supported I add support these types.
-For more detail see following issues.
-
-- https://github.com/embulk/embulk/issues/120
-- https://github.com/embulk/embulk/issues/121
-
 ## Overview
 
-This plugin only works with embulk >= 0.7.4.
+This plugin only works with embulk >= 0.8.8.
 
 * **Plugin type**: input
 * **Resume supported**: no
@@ -43,26 +35,27 @@ This plugin only works with embulk >= 0.7.4.
 ```yaml
 in:
   type: mongodb
-  uri: mongodb://myuser@mypassword:localhost:27017/my_database
+  uri: mongodb://myuser:mypassword@localhost:27017/my_database
   collection: "my_collection"
   fields:
     - { name: id, type: string }
     - { name: field1, type: long }
     - { name: field2, type: timestamp }
+    - { name: field3, type: json }
 ```
 
 ### Filter object by query and sort
 
-
 ```yaml
 in:
   type: mongodb
-  uri: mongodb://myuser@mypassword:localhost:27017/my_database
+  uri: mongodb://myuser:mypassword@localhost:27017/my_database
   collection: "my_collection"
   fields:
     - { name: id, type: string }
     - { name: field1, type: long }
     - { name: field2, type: timestamp }
+    - { name: field3, type: json }
   query: '{ field1: { $gte: 3 } }'
   sort: '{ field1: 1 }'
 ```

data/build.gradle

@@ -15,15 +15,15 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.4"
-    provided "org.embulk:embulk-core:0.7.4"
-    compile "org.mongodb:mongo-java-driver:3.0.3"
+    compile  "org.embulk:embulk-core:0.8.8"
+    provided "org.embulk:embulk-core:0.8.8"
+    compile "org.mongodb:mongo-java-driver:3.2.2"
 
     testCompile "junit:junit:4.+"
 }

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -1,7 +1,9 @@
 package org.embulk.input.mongodb;
 
+import com.google.common.base.Throwables;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
+import com.mongodb.MongoException;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
@@ -11,6 +13,7 @@ import org.bson.conversions.Bson;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
@@ -25,10 +28,13 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.type.Type;
 import org.slf4j.Logger;
 
+import javax.validation.constraints.Min;
+import java.net.UnknownHostException;
 import java.util.List;
 
 public class MongodbInputPlugin
@@ -57,10 +63,11 @@ public class MongodbInputPlugin
 
         @Config("batch_size")
         @ConfigDefault("10000")
-        Integer getBatchSize();
+        @Min(1)
+        int getBatchSize();
 
         @ConfigInject
-        public BufferAllocator getBufferAllocator();
+        BufferAllocator getBufferAllocator();
     }
 
     private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
@@ -70,6 +77,12 @@ public class MongodbInputPlugin
             InputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
+        // Connect once to throw ConfigException in earlier stage of excecution
+        try {
+            connect(task);
+        } catch (UnknownHostException | MongoException ex) {
+            throw new ConfigException(ex);
+        }
         Schema schema = task.getFields().toSchema();
         return resume(task.dump(), schema, 1, control);
     }
@@ -99,9 +112,16 @@ public class MongodbInputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
         BufferAllocator allocator = task.getBufferAllocator();
         PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
+        JsonParser jsonParser = new JsonParser();
+        List<Column> columns = pageBuilder.getSchema().getColumns();
 
-        MongoDatabase db = connect(task);
-        MongoCollection<Document> collection = db.getCollection(task.getCollection());
+        MongoCollection<Document> collection;
+        try {
+            MongoDatabase db = connect(task);
+            collection = db.getCollection(task.getCollection());
+        } catch (UnknownHostException | MongoException ex) {
+            throw new ConfigException(ex);
+        }
 
         Bson query = (Bson) JSON.parse(task.getQuery());
         Bson projection = getProjection(task);
@@ -111,25 +131,22 @@ public class MongodbInputPlugin
         log.trace("projection: {}", projection);
         log.trace("sort: {}", sort);
 
-        MongoCursor<Document> cursor = collection
-                                        .find(query)
-                                        .projection(projection)
-                                        .sort(sort)
-                                        .batchSize(task.getBatchSize())
-                                        .iterator();
-
-        try {
+        try (MongoCursor<Document> cursor = collection
+                .find(query)
+                .projection(projection)
+                .sort(sort)
+                .batchSize(task.getBatchSize())
+                .iterator()) {
             while (cursor.hasNext()) {
-                fetch(cursor, pageBuilder);
+                fetch(cursor, pageBuilder, jsonParser, columns);
             }
-        } finally {
-            cursor.close();
+        } catch (MongoException ex) {
+            Throwables.propagate(ex);
         }
 
         pageBuilder.finish();
 
-        TaskReport report = Exec.newTaskReport();
-        return report;
+        return Exec.newTaskReport();
     }
 
     @Override
@@ -138,15 +155,19 @@ public class MongodbInputPlugin
         return Exec.newConfigDiff();
     }
 
-    private MongoDatabase connect(PluginTask task) {
+    private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException {
         MongoClientURI uri = new MongoClientURI(task.getUri());
         MongoClient mongoClient = new MongoClient(uri);
-        return mongoClient.getDatabase(uri.getDatabase());
+
+        MongoDatabase db = mongoClient.getDatabase(uri.getDatabase());
+        // Get collection count for throw Exception
+        db.getCollection(task.getCollection()).count();
+        return db;
     }
 
-    private void fetch(MongoCursor<Document> cursor, PageBuilder pageBuilder) {
+    private void fetch(MongoCursor<Document> cursor, PageBuilder pageBuilder,
+                       JsonParser jsonParser, List<Column> columns) {
         Document doc = cursor.next();
-        List<Column> columns = pageBuilder.getSchema().getColumns();
         for (Column c : columns) {
             Type t = c.getType();
             String key = normalize(c.getName());
@@ -170,12 +191,17 @@ public class MongodbInputPlugin
                     break;
 
                 case "string":
-                // Enable output object like ObjectId as string, this is reason I don't use doc.getString(key).
-                pageBuilder.setString(c, doc.get(key).toString());
-                break;
+                    // Enable output object like ObjectId as string, this is reason I don't use doc.getString(key).
+                    pageBuilder.setString(c, doc.get(key).toString());
+                    break;
 
                 case "timestamp":
                     pageBuilder.setTimestamp(c, Timestamp.ofEpochMilli(doc.getDate(key).getTime()));
+                    break;
+
+                case "json":
+                    pageBuilder.setJson(c, jsonParser.parse(((Document) doc.get(key)).toJson()));
+                    break;
                 }
             }
         }

data/src/test/resources/basic.yml

@@ -5,6 +5,7 @@ in:
   fields:
     - { name: name, type: string }
     - { name: rank, type: long }
+  sort: '{ rank: 1 }'
 out:
   type: file
   path_prefix: ./tmp/basic

data/src/test/resources/basic_expected.csv

@@ -1,10 +1,10 @@
 name,rank
 obj1,1
 obj2,2
-obj4,4
 obj3,3
+obj4,4
 obj5,5
 obj6,6
 obj7,7
-obj9,9
 obj8,8
+obj9,9

data/src/test/resources/full.yml

@@ -8,6 +8,7 @@ in:
     - { name: rank, type: long }
     - { name: value, type: double }
     - { name: created_at, type: timestamp }
+    - { name: embeded, type: json }
   query: '{ rank: { $gte: 3 } }'
   sort: '{ rank: -1 }'
   batch_size: 100

data/src/test/resources/full_expected.csv

@@ -1,8 +1,8 @@
-id,name,rank,value,created_at
-55eae883689a08361045d652,obj9,9,9.9,2015-09-06 10:05:18.786000 +0000
-55eae883689a08361045d651,obj8,8,8.8,2015-09-06 10:05:28.786000 +0000
-55eae883689a08361045d650,obj7,7,7.7,2015-09-06 10:05:38.786000 +0000
-55eae883689a08361045d64f,obj6,6,6.6,2015-09-06 10:05:48.786000 +0000
-55eae883689a08361045d64e,obj5,5,5.5,2015-09-06 10:05:58.786000 +0000
-55eae883689a08361045d64d,obj4,4,4.4,2015-09-06 10:06:08.786000 +0000
-55eae883689a08361045d64c,obj3,3,3.3,2015-09-06 10:06:18.786000 +0000
+id,name,rank,value,created_at,embeded
+55eae883689a08361045d652,obj9,9,9.9,2015-09-06 10:05:18.786000 +0000,"{""key"":""value9""}"
+55eae883689a08361045d651,obj8,8,8.8,2015-09-06 10:05:28.786000 +0000,"{""key"":""value8""}"
+55eae883689a08361045d650,obj7,7,7.7,2015-09-06 10:05:38.786000 +0000,"{""key"":""value7""}"
+55eae883689a08361045d64f,obj6,6,6.6,2015-09-06 10:05:48.786000 +0000,"{""key"":""value6""}"
+55eae883689a08361045d64e,obj5,5,5.5,2015-09-06 10:05:58.786000 +0000,"{""key"":""value5""}"
+55eae883689a08361045d64d,obj4,4,4.4,2015-09-06 10:06:08.786000 +0000,"{""key"":{""inner_key"":""value4""}}"
+55eae883689a08361045d64c,obj3,3,3.3,2015-09-06 10:06:18.786000 +0000,"{""key"":[""v3-1"",""v3-2""]}"

data/src/test/resources/my_collection.jsonl

@@ -1,9 +1,9 @@
-{ "_id": "55eae883689a08361045d64a", "name": "obj1", "rank": 1, "value": 1.1, "created_at": { "$date" : 1441533998786 } }
-{ "_id": "55eae883689a08361045d64b", "name": "obj2", "rank": 2, "value": 2.2, "created_at": { "$date" : 1441533988786 } }
-{ "_id": "55eae883689a08361045d64c", "name": "obj3", "rank": 3, "value": 3.3, "created_at": { "$date" : 1441533978786 } }
-{ "_id": "55eae883689a08361045d64d", "name": "obj4", "rank": 4, "value": 4.4, "created_at": { "$date" : 1441533968786 } }
-{ "_id": "55eae883689a08361045d64e", "name": "obj5", "rank": 5, "value": 5.5, "created_at": { "$date" : 1441533958786 } }
-{ "_id": "55eae883689a08361045d64f", "name": "obj6", "rank": 6, "value": 6.6, "created_at": { "$date" : 1441533948786 } }
-{ "_id": "55eae883689a08361045d650", "name": "obj7", "rank": 7, "value": 7.7, "created_at": { "$date" : 1441533938786 } }
-{ "_id": "55eae883689a08361045d651", "name": "obj8", "rank": 8, "value": 8.8, "created_at": { "$date" : 1441533928786 } }
-{ "_id": "55eae883689a08361045d652", "name": "obj9", "rank": 9, "value": 9.9, "created_at": { "$date" : 1441533918786 } }
+{ "_id": "55eae883689a08361045d64a", "name": "obj1", "rank": 1, "value": 1.1, "created_at": { "$date" : 1441533998786 }, "embeded": { "key": "value1" } }
+{ "_id": "55eae883689a08361045d64b", "name": "obj2", "rank": 2, "value": 2.2, "created_at": { "$date" : 1441533988786 }, "embeded": { "key": "value2" } }
+{ "_id": "55eae883689a08361045d64c", "name": "obj3", "rank": 3, "value": 3.3, "created_at": { "$date" : 1441533978786 }, "embeded": { "key": ["v3-1", "v3-2"]} }
+{ "_id": "55eae883689a08361045d64d", "name": "obj4", "rank": 4, "value": 4.4, "created_at": { "$date" : 1441533968786 }, "embeded": { "key": { "inner_key": "value4" } } }
+{ "_id": "55eae883689a08361045d64e", "name": "obj5", "rank": 5, "value": 5.5, "created_at": { "$date" : 1441533958786 }, "embeded": { "key": "value5" } }
+{ "_id": "55eae883689a08361045d64f", "name": "obj6", "rank": 6, "value": 6.6, "created_at": { "$date" : 1441533948786 }, "embeded": { "key": "value6" } }
+{ "_id": "55eae883689a08361045d650", "name": "obj7", "rank": 7, "value": 7.7, "created_at": { "$date" : 1441533938786 }, "embeded": { "key": "value7" } }
+{ "_id": "55eae883689a08361045d651", "name": "obj8", "rank": 8, "value": 8.8, "created_at": { "$date" : 1441533928786 }, "embeded": { "key": "value8" } }
+{ "_id": "55eae883689a08361045d652", "name": "obj9", "rank": 9, "value": 9.9, "created_at": { "$date" : 1441533918786 }, "embeded": { "key": "value9" } }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-10 00:00:00.000000000 Z
+date: 2016-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,8 +62,8 @@ files:
 - src/test/resources/full.yml
 - src/test/resources/full_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.1.2.jar
-- classpath/mongo-java-driver-3.0.3.jar
+- classpath/embulk-input-mongodb-0.2.0.jar
+- classpath/mongo-java-driver-3.2.2.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses:
 - MIT