embulk-input-mongodb 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eca1bf1b21b4a44bab520a7cb93f59df2c18ea82
-  data.tar.gz: 98f866d2313fe81dbf96da2fe61b9a303afcd698
+  metadata.gz: fafb06875aa38ec6b9142251fd9b57b15f56dc17
+  data.tar.gz: 6aa68d44e22d4b49e9c5918095dc1fd997a11f58
 SHA512:
-  metadata.gz: 6251028e2ec5dc41523cbc43d3a06f57cde1a7b05a9a21a8098d169e6c3d0e05af74d12e325847fcbb74328f8c9cf98eb8e6ed167091cb2215bfbf2341fdb5ff
-  data.tar.gz: 7a6c238d53ab10733917a05a170a3d06aacb8cbbd8a631c87d86d5fa2a6a198e887df27b61bfebe10655eeefe48293bf0a174417435aed4926285d6ef5ba581f
+  metadata.gz: 51b725f4f59acc26978861e96ce4c9388dc53fdd6ff9cb5edb959984a3fd0e2622731d4941940ece4370f5a1b2094b4bb087fd354ea086ba161d685ac170140c
+  data.tar.gz: 1ec52180d29002a95c024a878e257ba399caee04e06dba3e7fdb74f9d130e7e207abca5744be7620f958dbd81ff74ea59be53011216fe3809ceb9539ab9d0128

data/README.md

@@ -3,61 +3,91 @@
 [![Build Status](https://travis-ci.org/hakobera/embulk-input-mongodb.svg)](https://travis-ci.org/hakobera/embulk-input-mongodb)
 
 MongoDB input plugin for Embulk loads records from MongoDB.
+This plugin loads documents as single-column records (column name is "record"). You can use filter plugins such as [embulk-filter-expand_json](https://github.com/civitaspo/embulk-filter-expand_json) or [embulk-filter-add_time](https://github.com/treasure-data/embulk-filter-add_time) to convert the json column to typed columns. [Rename filter](http://www.embulk.org/docs/built-in.html#rename-filter-plugin) is also useful to rename the typed columns.
 
 ## Overview
 
 This plugin only works with embulk >= 0.8.8.
 
 * **Plugin type**: input
-* **Resume supported**: no
-* **Cleanup supported**: no
 * **Guess supported**: no
 
 ## Configuration
 
 - **uri**: [MongoDB connection string URI](http://docs.mongodb.org/manual/reference/connection-string/) (e.g. 'mongodb://localhost:27017/mydb') (string, required)
 - **collection**: source collection name (string, required)
-- **fields**: hash records that has the following two fields (array, required)
-  - name: Name of the column
-  - type: Column types as follows
-    - boolean
-    - long
-    - double
-    - string
-    - timestamp
-- **query**: provides a JSON document as a query that optionally limits the documents returned (string, optional)
-- **sort**: specifies an ordering for exported results (string, optional)
+- **fields**: **(deprecated)** ~~hash records that has the following two fields (array, required)~~
+  ~~- name: Name of the column~~
+  ~~- type: Column types as follows~~
+    ~~- boolean~~
+    ~~- long~~
+    ~~- double~~
+    ~~- string~~
+    ~~- timestamp~~
+- **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
+- **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
+- **sort**: ordering of results (string, optional)
+- **stop_on_invalid_record** Stop bulk load transaction if a document includes invalid record (such as unsupported object type) (boolean, optional, default: false)
+- **json_column_name**: column name used in outputs (string, optional, default: "json")
 
 ## Example
 
-### Export all objects
+### Exporting all objects
 
 ```yaml
 in:
   type: mongodb
   uri: mongodb://myuser:mypassword@localhost:27017/my_database
   collection: "my_collection"
-  fields:
-    - { name: id, type: string }
-    - { name: field1, type: long }
-    - { name: field2, type: timestamp }
-    - { name: field3, type: json }
 ```
 
-### Filter object by query and sort
+### Filtering documents by query and projection
 
 ```yaml
 in:
   type: mongodb
   uri: mongodb://myuser:mypassword@localhost:27017/my_database
   collection: "my_collection"
-  fields:
-    - { name: id, type: string }
-    - { name: field1, type: long }
-    - { name: field2, type: timestamp }
-    - { name: field3, type: json }
   query: '{ field1: { $gte: 3 } }'
-  sort: '{ field1: 1 }'
+  projection: '{ "_id": 1, "field1": 1, "field2": 0 }'
+  sort: '{ "field1": 1 }'
+```
+
+### Advanced usage with filter plugins
+
+```yaml
+in:
+  type: mongodb
+  uri: mongodb://myuser:mypassword@localhost:27017/my_database
+  collection: "my_collection"
+  query: '{ "age": { $gte: 3 } }'
+  projection: '{ "_id": 1, "age": 1, "ts": 1, "firstName": 1, "lastName": 1 }'
+
+filters:
+  # convert json column into typed columns
+  - type: expand_json
+    json_column_name: record
+    expanded_columns:
+      - {name: _id, type: long}
+      - {name: ts, type: string}
+      - {name: firstName, type: string}
+      - {name: lastName, type: string}
+
+  # rename column names
+  - type: rename
+    columns:
+      _id: id
+      firstName: first_name
+      lastName: last_name
+
+  # convert string "ts" column into timestamp "time" column
+  - type: add_time
+    from_column:
+      name: ts
+      timestamp_format: "%Y-%m-%dT%H:%M:%S.%N%z"
+    to_column:
+      name: time
+      type: timestamp
 ```
 
 ## Build
@@ -65,3 +95,47 @@ in:
 ```
 $ ./gradlew gem
 ```
+
+## Test
+
+```
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
+```
+
+To run unit tests, we need to configure the following environment variables.
+
+When environment variables are not set, skip almost test cases.
+
+```
+MONGO_URI
+MONGO_COLLECTION
+```
+
+If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
+```xml
+$ vi ~/Library/LaunchAgents/environment.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>my.startup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>sh</string>
+    <string>-c</string>
+    <string>
+      launchctl setenv MONGO_URI mongodb://myuser:mypassword@localhost:27017/my_database
+      launchctl setenv MONGO_COLLECTION my_collection
+    </string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+</dict>
+</plist>
+
+$ launchctl load ~/Library/LaunchAgents/environment.plist
+$ launchctl getenv MONGO_URI //try to get value.
+
+Then start your applications.
+```

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -15,7 +16,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -26,6 +27,8 @@ dependencies {
     compile "org.mongodb:mongo-java-driver:3.2.2"
 
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.8:tests"
+    testCompile "org.embulk:embulk-standards:0.8.8"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.input.mongodb;
 
+import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
@@ -8,7 +9,9 @@ import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
 import com.mongodb.util.JSON;
-import org.bson.Document;
+import com.mongodb.util.JSONParseException;
+import org.bson.codecs.configuration.CodecRegistries;
+import org.bson.codecs.configuration.CodecRegistry;
 import org.bson.conversions.Bson;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -21,18 +24,15 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Column;
-import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Exec;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
 import org.slf4j.Logger;
-
 import javax.validation.constraints.Min;
 import java.net.UnknownHostException;
 import java.util.List;
@@ -51,7 +51,12 @@ public class MongodbInputPlugin
         String getCollection();
 
         @Config("fields")
-        SchemaConfig getFields();
+        @ConfigDefault("null")
+        Optional<SchemaConfig> getFields();
+
+        @Config("projection")
+        @ConfigDefault("\"{}\"")
+        String getProjection();
 
         @Config("query")
         @ConfigDefault("\"{}\"")
@@ -66,6 +71,14 @@ public class MongodbInputPlugin
         @Min(1)
         int getBatchSize();
 
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+
+        @Config("json_column_name")
+        @ConfigDefault("\"record\"")
+        String getJsonColumnName();
+
         @ConfigInject
         BufferAllocator getBufferAllocator();
     }
@@ -77,13 +90,21 @@ public class MongodbInputPlugin
             InputPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
+        if (task.getFields().isPresent()) {
+            throw new ConfigException("field option was deprecated so setting will be ignored");
+        }
+
+        validateJsonField("projection", task.getProjection());
+        validateJsonField("query", task.getQuery());
+        validateJsonField("sort", task.getSort());
+
         // Connect once to throw ConfigException in earlier stage of excecution
         try {
             connect(task);
         } catch (UnknownHostException | MongoException ex) {
             throw new ConfigException(ex);
         }
-        Schema schema = task.getFields().toSchema();
+        Schema schema = Schema.builder().add(task.getJsonColumnName(), Types.JSON).build();
         return resume(task.dump(), schema, 1, control);
     }
 
@@ -112,33 +133,39 @@ public class MongodbInputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
         BufferAllocator allocator = task.getBufferAllocator();
         PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
-        JsonParser jsonParser = new JsonParser();
-        List<Column> columns = pageBuilder.getSchema().getColumns();
+        final Column column = pageBuilder.getSchema().getColumns().get(0);
 
-        MongoCollection<Document> collection;
+        MongoCollection<Value> collection;
         try {
             MongoDatabase db = connect(task);
-            collection = db.getCollection(task.getCollection());
+
+            CodecRegistry registry = CodecRegistries.fromRegistries(
+                    MongoClient.getDefaultCodecRegistry(),
+                    CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord()))
+            );
+            collection = db.getCollection(task.getCollection(), Value.class)
+                    .withCodecRegistry(registry);
         } catch (UnknownHostException | MongoException ex) {
             throw new ConfigException(ex);
         }
 
         Bson query = (Bson) JSON.parse(task.getQuery());
-        Bson projection = getProjection(task);
+        Bson projection = (Bson) JSON.parse(task.getProjection());
         Bson sort = (Bson) JSON.parse(task.getSort());
 
         log.trace("query: {}", query);
         log.trace("projection: {}", projection);
         log.trace("sort: {}", sort);
 
-        try (MongoCursor<Document> cursor = collection
+        try (MongoCursor<Value> cursor = collection
                 .find(query)
                 .projection(projection)
                 .sort(sort)
                 .batchSize(task.getBatchSize())
                 .iterator()) {
             while (cursor.hasNext()) {
-                fetch(cursor, pageBuilder, jsonParser, columns);
+                pageBuilder.setJson(column, cursor.next());
+                pageBuilder.addRecord();
             }
         } catch (MongoException ex) {
             Throwables.propagate(ex);
@@ -165,73 +192,11 @@ public class MongodbInputPlugin
         return db;
     }
 
-    private void fetch(MongoCursor<Document> cursor, PageBuilder pageBuilder,
-                       JsonParser jsonParser, List<Column> columns) {
-        Document doc = cursor.next();
-        for (Column c : columns) {
-            Type t = c.getType();
-            String key = normalize(c.getName());
-
-            if (!doc.containsKey(key) || doc.get(key) == null) {
-                pageBuilder.setNull(c);
-            } else {
-                switch (t.getName()) {
-                case "boolean":
-                    pageBuilder.setBoolean(c, doc.getBoolean(key));
-                    break;
-
-                case "long":
-                    // MongoDB can contain both 'int' and 'long', but embulk only support 'long'
-                    // So enable handling both 'int' and 'long', first get value as java.lang.Number, then convert it to long
-                    pageBuilder.setLong(c, ((Number) doc.get(key)).longValue());
-                    break;
-
-                case "double":
-                    pageBuilder.setDouble(c, ((Number) doc.get(key)).doubleValue());
-                    break;
-
-                case "string":
-                    // Enable output object like ObjectId as string, this is reason I don't use doc.getString(key).
-                    pageBuilder.setString(c, doc.get(key).toString());
-                    break;
-
-                case "timestamp":
-                    pageBuilder.setTimestamp(c, Timestamp.ofEpochMilli(doc.getDate(key).getTime()));
-                    break;
-
-                case "json":
-                    pageBuilder.setJson(c, jsonParser.parse(((Document) doc.get(key)).toJson()));
-                    break;
-                }
-            }
-        }
-        pageBuilder.addRecord();
-    }
-
-    private Bson getProjection(PluginTask task) {
-        SchemaConfig fields = task.getFields();
-        StringBuilder sb = new StringBuilder("{");
-        int l = fields.getColumnCount();
-
-        for (int i = 0; i < l; i++) {
-            ColumnConfig c = fields.getColumn(i);
-            if (i != 0) {
-                sb.append(",");
-            }
-            String key = normalize(c.getName());
-            sb.append(key).append(":1");
-        }
-        sb.append("}");
-
-        return (Bson) JSON.parse(sb.toString());
-    }
-
-    private String normalize(String key) {
-        // 'id' is special alias key name of MongoDB ObjectId
-        // http://docs.mongodb.org/manual/reference/object-id/
-        if (key.equals("id")) {
-            return "_id";
+    private void validateJsonField(String name, String jsonString) {
+        try {
+            JSON.parse(jsonString);
+        } catch (JSONParseException ex) {
+            throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));
         }
-        return key;
     }
 }

data/src/main/java/org/embulk/input/mongodb/ValueCodec.java

@@ -0,0 +1,151 @@
+package org.embulk.input.mongodb;
+
+import org.bson.BsonReader;
+import org.bson.BsonType;
+import org.bson.BsonWriter;
+import org.bson.codecs.Codec;
+import org.bson.codecs.DecoderContext;
+import org.bson.codecs.EncoderContext;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import static org.msgpack.value.ValueFactory.newArray;
+import static org.msgpack.value.ValueFactory.newBinary;
+import static org.msgpack.value.ValueFactory.newBoolean;
+import static org.msgpack.value.ValueFactory.newFloat;
+import static org.msgpack.value.ValueFactory.newInteger;
+import static org.msgpack.value.ValueFactory.newMap;
+import static org.msgpack.value.ValueFactory.newNil;
+import static org.msgpack.value.ValueFactory.newString;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TimeZone;
+
+public class ValueCodec implements Codec<Value> {
+    private final SimpleDateFormat formatter;
+    private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
+    private final boolean stopOnInvalidRecord;
+
+    public ValueCodec(boolean stopOnInvalidRecord) {
+        this.formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
+        formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
+        this.stopOnInvalidRecord = stopOnInvalidRecord;
+    }
+
+    @Override
+    public void encode(final BsonWriter writer, final Value value, final EncoderContext encoderContext) {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Value decode(final BsonReader reader, final DecoderContext decoderContext) {
+        Map<Value, Value> kvs = new LinkedHashMap<>();
+
+        reader.readStartDocument();
+        boolean isTopLevelNode = false;
+        while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
+            String fieldName = reader.readName();
+            BsonType type = reader.getCurrentBsonType();
+            if (type == BsonType.OBJECT_ID) {
+                isTopLevelNode = true;
+            }
+            fieldName = normalize(fieldName, isTopLevelNode);
+
+            try {
+                kvs.put(newString(fieldName), readValue(reader, decoderContext));
+            } catch (UnknownTypeFoundException ex) {
+                reader.skipValue();
+                if (stopOnInvalidRecord) {
+                    throw ex;
+                }
+                log.warn(String.format("Skipped document because field '%s' contains unsupported object type [%s]",
+                        fieldName, type));
+            }
+        }
+        reader.readEndDocument();
+
+        return newMap(kvs);
+    }
+
+    public Value decodeArray(final BsonReader reader, final DecoderContext decoderContext) {
+        List<Value> list = new ArrayList<>();
+
+        reader.readStartArray();
+        while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
+            list.add(readValue(reader, decoderContext));
+        }
+        reader.readEndArray();
+
+        return newArray(list);
+    }
+
+    private Value readValue(BsonReader reader, DecoderContext decoderContext) {
+        switch (reader.getCurrentBsonType()) {
+            // https://docs.mongodb.com/manual/reference/bson-types/
+            // https://github.com/mongodb/mongo-java-driver/tree/master/bson/src/main/org/bson/codecs
+            case DOUBLE:
+                return newFloat(reader.readDouble());
+            case STRING:
+                return newString(reader.readString());
+            case ARRAY:
+                return decodeArray(reader, decoderContext);
+            case BINARY:
+                return newBinary(reader.readBinaryData().getData(), true);
+            case OBJECT_ID:
+                return newString(reader.readObjectId().toString());
+            case BOOLEAN:
+                return newBoolean(reader.readBoolean());
+            case DATE_TIME:
+                return newString(formatter.format(new Date(reader.readDateTime())));
+            case NULL:
+                reader.readNull();
+                return newNil();
+            case REGULAR_EXPRESSION:
+                return newString(reader.readRegularExpression().toString());
+            case JAVASCRIPT:
+                return newString(reader.readJavaScript());
+            case JAVASCRIPT_WITH_SCOPE:
+                return newString(reader.readJavaScriptWithScope());
+            case INT32:
+                return newInteger(reader.readInt32());
+            case TIMESTAMP:
+                return newInteger(reader.readTimestamp().getTime());
+            case INT64:
+                return newInteger(reader.readInt64());
+            case DOCUMENT:
+                return decode(reader, decoderContext);
+            default: // e.g. MIN_KEY, MAX_KEY, SYMBOL, DB_POINTER, UNDEFINED
+                throw new UnknownTypeFoundException(String.format("Unsupported type %s of '%s' field. Please exclude the field from 'projection:' option",
+                        reader.getCurrentBsonType(), reader.getCurrentName()));
+        }
+    }
+
+    @Override
+    public Class<Value> getEncoderClass() {
+        return Value.class;
+    }
+
+    private String normalize(String key, boolean isTopLevelNode) {
+        // 'id' is special alias key name of MongoDB ObjectId
+        // http://docs.mongodb.org/manual/reference/object-id/
+        if (key.equals("id") && isTopLevelNode) {
+            return "_id";
+        }
+        return key;
+    }
+
+    public static class UnknownTypeFoundException extends DataException
+    {
+        UnknownTypeFoundException(String message)
+        {
+            super(message);
+        }
+    }
+}

data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java

@@ -1,5 +1,339 @@
 package org.embulk.input.mongodb;
 
-public class TestMongodbInputPlugin
-{
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import org.bson.BsonBinary;
+import org.bson.BsonInt64;
+import org.bson.BsonJavaScript;
+import org.bson.BsonMaxKey;
+import org.bson.BsonRegularExpression;
+import org.bson.BsonTimestamp;
+import org.bson.Document;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.input.mongodb.MongodbInputPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.type.Types;
+import org.embulk.spi.util.Pages;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+
+public class TestMongodbInputPlugin {
+    private static String MONGO_URI;
+    private static String MONGO_COLLECTION;
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    private ConfigSource config;
+    private MongodbInputPlugin plugin;
+    private MockPageOutput output;
+
+    /*
+     * This test case requires environment variables
+     *   MONGO_URI
+     *   MONGO_COLLECTION
+     */
+    @BeforeClass
+    public static void initializeConstant() {
+        MONGO_URI = System.getenv("MONGO_URI");
+        MONGO_COLLECTION = System.getenv("MONGO_COLLECTION");
+    }
+
+    @Before
+    public void createResources() throws Exception {
+        config = config();
+        plugin = new MongodbInputPlugin();
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void checkDefaultValues() {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION);
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals("{}", task.getQuery());
+        assertEquals("{}", task.getSort());
+        assertEquals((long) 10000, (long) task.getBatchSize());
+        assertEquals("record", task.getJsonColumnName());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesUriIsNull() {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", null)
+                .set("collection", MONGO_COLLECTION);
+
+        plugin.transaction(config, new Control());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesInvalidUri()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", "mongodb://mongouser:password@non-exists.example.com:23490/test")
+                .set("collection", MONGO_COLLECTION);
+
+        plugin.transaction(config, new Control());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesCollectionIsNull() {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", null);
+
+        plugin.transaction(config, new Control());
+    }
+
+    @Test
+    public void testResume() {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        final Schema schema = getFieldSchema();
+        plugin.resume(task.dump(), schema, 0, new InputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount) {
+                return emptyTaskReports(taskCount);
+            }
+        });
+        // no errors happens
+    }
+
+    @Test
+    public void testCleanup() {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        Schema schema = getFieldSchema();
+        plugin.cleanup(task.dump(), schema, 0, Lists.<TaskReport>newArrayList()); // no errors happens
+    }
+
+    @Test
+    public void testGuess() {
+        plugin.guess(config); // no errors happens
+    }
+
+    @Test
+    public void testRun() throws Exception {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        plugin.transaction(config, new Control());
+        assertValidRecords(getFieldSchema(), output);
+    }
+
+    @Test(expected = ValueCodec.UnknownTypeFoundException.class)
+    public void testRunWithUnsupportedType() throws Exception {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("stop_on_invalid_record", true);
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+            new Document("invalid_field", new BsonMaxKey())
+        );
+        insertDocument(task, documents);
+
+        plugin.transaction(config, new Control());
+    }
+
+    @Test
+    public void testNormalize() throws Exception {
+        ValueCodec codec = new ValueCodec(true);
+
+        Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class, boolean.class);
+        normalize.setAccessible(true);
+        assertEquals("_id", normalize.invoke(codec, "id", true).toString());
+        assertEquals("_id", normalize.invoke(codec, "_id", true).toString());
+        assertEquals("f1", normalize.invoke(codec, "f1", true).toString());
+
+        assertEquals("id", normalize.invoke(codec, "id", false).toString());
+        assertEquals("_id", normalize.invoke(codec, "_id", false).toString());
+        assertEquals("f1", normalize.invoke(codec, "f1", false).toString());
+    }
+
+    @Test
+    public void testValidateJsonField() throws Exception {
+        Method validate = MongodbInputPlugin.class.getDeclaredMethod("validateJsonField", String.class, String.class);
+        validate.setAccessible(true);
+        String invalidJsonString = "{\"name\": invalid}";
+        try {
+            validate.invoke(plugin, "name", invalidJsonString);
+        } catch (InvocationTargetException ex) {
+            assertEquals(ConfigException.class, ex.getCause().getClass());
+        }
+    }
+
+    static List<TaskReport> emptyTaskReports(int taskCount) {
+        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
+        for (int i = 0; i < taskCount; i++) {
+            reports.add(Exec.newTaskReport());
+        }
+        return reports.build();
+    }
+
+    private class Control
+            implements InputPlugin.Control {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount) {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(plugin.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+
+    private ConfigSource config() {
+        return Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("last_path", "");
+    }
+
+    private List<Document> createValidDocuments() throws Exception {
+        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ENGLISH);
+
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+            new Document("double_field", 1.23)
+                    .append("string_field", "embulk")
+                    .append("array_field", Arrays.asList(1,2,3))
+                    .append("binary_field", new BsonBinary(("test").getBytes("UTF-8")))
+                    .append("boolean_field", true)
+                    .append("datetime_field", format.parse("2015-01-27T19:23:49Z"))
+                    .append("null_field", null)
+                    .append("regex_field", new BsonRegularExpression(".+?"))
+                    .append("javascript_field", new BsonJavaScript("var s = \"javascript\";"))
+                    .append("int32_field", 32864)
+                    .append("timestamp_field", new BsonTimestamp(1463991177, 4))
+                    .append("int64_field", new BsonInt64(314159265))
+                    .append("document_field", new Document("k", true))
+        );
+
+        documents.add(
+            new Document("boolean_field", false)
+                    .append("document_field", new Document("k", 1))
+        );
+
+        documents.add(new Document("document_field", new Document("k", 1.23)));
+
+        documents.add(new Document("document_field", new Document("k", "v")));
+
+        documents.add(new Document("document_field", new Document("k", format.parse("2015-02-03T08:13:45Z"))));
+
+        return documents;
+    }
+
+    private Schema getFieldSchema() {
+        ImmutableList.Builder<Column> columns = ImmutableList.builder();
+        columns.add(new Column(0, "record", Types.JSON));
+        return new Schema(columns.build());
+    }
+
+    private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception {
+        List<Object[]> records = Pages.toObjects(schema, output.pages);
+        assertEquals(5, records.size());
+
+        ObjectMapper mapper = new ObjectMapper();
+
+        {
+            JsonNode node = mapper.readTree(records.get(0)[0].toString());
+            assertThat(1.23, is(node.get("double_field").asDouble()));
+            assertEquals("embulk", node.get("string_field").asText());
+            assertEquals("[1,2,3]", node.get("array_field").toString());
+            assertEquals("test", node.get("binary_field").asText());
+            assertEquals(true, node.get("boolean_field").asBoolean());
+            assertEquals("2015-01-27T10:23:49.000Z", node.get("datetime_field").asText());
+            assertEquals("null", node.get("null_field").asText());
+            assertEquals("BsonRegularExpression{pattern='.+?', options=''}", node.get("regex_field").asText());
+            assertEquals("var s = \"javascript\";", node.get("javascript_field").asText());
+            assertEquals(32864L, node.get("int32_field").asLong());
+            assertEquals("1463991177", node.get("timestamp_field").asText());
+            assertEquals(314159265L, node.get("int64_field").asLong());
+            assertEquals("{\"k\":true}", node.get("document_field").toString());
+        }
+
+        {
+            JsonNode node = mapper.readTree(records.get(1)[0].toString());
+            assertEquals(false, node.get("boolean_field").asBoolean());
+            assertEquals("{\"k\":1}", node.get("document_field").toString());
+        }
+
+        {
+            JsonNode node = mapper.readTree(records.get(2)[0].toString());
+            assertEquals("{\"k\":1.23}", node.get("document_field").toString());
+        }
+
+        {
+            JsonNode node = mapper.readTree(records.get(3)[0].toString());
+            assertEquals("{\"k\":\"v\"}", node.get("document_field").toString());
+        }
+
+        {
+            JsonNode node = mapper.readTree(records.get(4)[0].toString());
+            assertEquals("{\"k\":\"2015-02-02T23:13:45.000Z\"}", node.get("document_field").toString());
+        }
+    }
+
+    private void createCollection(PluginTask task, String collectionName) throws Exception {
+        Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
+        method.setAccessible(true);
+        MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
+        db.createCollection(collectionName);
+    }
+
+    private void dropCollection(PluginTask task, String collectionName) throws Exception {
+        Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
+        method.setAccessible(true);
+        MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
+        MongoCollection collection = db.getCollection(collectionName);
+        collection.drop();
+    }
+
+    private void insertDocument(PluginTask task, List<Document> documents) throws Exception {
+        Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
+        method.setAccessible(true);
+        MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
+        MongoCollection collection = db.getCollection(task.getCollection());
+        collection.insertMany(documents);
+    }
 }

data/src/test/resources/basic.yml

@@ -2,9 +2,7 @@ in:
   type: mongodb
   uri: mongodb://localhost:27017/my_database
   collection: "my_collection"
-  fields:
-    - { name: name, type: string }
-    - { name: rank, type: long }
+  projection: '{ "_id": 0, "name": 1, "rank": 1 }'
   sort: '{ rank: 1 }'
 out:
   type: file

data/src/test/resources/basic_expected.csv

@@ -1,10 +1,10 @@
-name,rank
-obj1,1
-obj2,2
-obj3,3
-obj4,4
-obj5,5
-obj6,6
-obj7,7
-obj8,8
-obj9,9
+record
+"{""name"":""obj1"",""rank"":1}"
+"{""name"":""obj2"",""rank"":2}"
+"{""name"":""obj3"",""rank"":3}"
+"{""name"":""obj4"",""rank"":4}"
+"{""name"":""obj5"",""rank"":5}"
+"{""name"":""obj6"",""rank"":6}"
+"{""name"":""obj7"",""rank"":7}"
+"{""name"":""obj8"",""rank"":8}"
+"{""name"":""obj9"",""rank"":9}"

data/src/test/resources/full.yml

@@ -2,13 +2,7 @@ in:
   type: mongodb
   uri: mongodb://localhost:27017/my_database
   collection: "my_collection"
-  fields:
-    - { name: id, type: string }
-    - { name: name, type: string }
-    - { name: rank, type: long }
-    - { name: value, type: double }
-    - { name: created_at, type: timestamp }
-    - { name: embeded, type: json }
+  json_column_name: "json"
   query: '{ rank: { $gte: 3 } }'
   sort: '{ rank: -1 }'
   batch_size: 100

data/src/test/resources/full_expected.csv

@@ -1,8 +1,8 @@
-id,name,rank,value,created_at,embeded
-55eae883689a08361045d652,obj9,9,9.9,2015-09-06 10:05:18.786000 +0000,"{""key"":""value9""}"
-55eae883689a08361045d651,obj8,8,8.8,2015-09-06 10:05:28.786000 +0000,"{""key"":""value8""}"
-55eae883689a08361045d650,obj7,7,7.7,2015-09-06 10:05:38.786000 +0000,"{""key"":""value7""}"
-55eae883689a08361045d64f,obj6,6,6.6,2015-09-06 10:05:48.786000 +0000,"{""key"":""value6""}"
-55eae883689a08361045d64e,obj5,5,5.5,2015-09-06 10:05:58.786000 +0000,"{""key"":""value5""}"
-55eae883689a08361045d64d,obj4,4,4.4,2015-09-06 10:06:08.786000 +0000,"{""key"":{""inner_key"":""value4""}}"
-55eae883689a08361045d64c,obj3,3,3.3,2015-09-06 10:06:18.786000 +0000,"{""key"":[""v3-1"",""v3-2""]}"
+json
+"{""_id"":""55eae883689a08361045d652"",""name"":""obj9"",""rank"":9,""value"":9.9,""created_at"":""2015-09-06T10:05:18.786Z"",""embeded"":{""key"":""value9""}}"
+"{""_id"":""55eae883689a08361045d651"",""name"":""obj8"",""rank"":8,""value"":8.8,""created_at"":""2015-09-06T10:05:28.786Z"",""embeded"":{""key"":""value8""}}"
+"{""_id"":""55eae883689a08361045d650"",""name"":""obj7"",""rank"":7,""value"":7.7,""created_at"":""2015-09-06T10:05:38.786Z"",""embeded"":{""key"":""value7""}}"
+"{""_id"":""55eae883689a08361045d64f"",""name"":""obj6"",""rank"":6,""value"":6.6,""created_at"":""2015-09-06T10:05:48.786Z"",""embeded"":{""key"":""value6""}}"
+"{""_id"":""55eae883689a08361045d64e"",""name"":""obj5"",""rank"":5,""value"":5.5,""created_at"":""2015-09-06T10:05:58.786Z"",""embeded"":{""key"":""value5""}}"
+"{""_id"":""55eae883689a08361045d64d"",""name"":""obj4"",""rank"":4,""value"":4.4,""created_at"":""2015-09-06T10:06:08.786Z"",""embeded"":{""key"":{""inner_key"":""value4""}}}"
+"{""_id"":""55eae883689a08361045d64c"",""name"":""obj3"",""rank"":3,""value"":3.3,""created_at"":""2015-09-06T10:06:18.786Z"",""embeded"":{""key"":[""v3-1"",""v3-2""]}}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-30 00:00:00.000000000 Z
+date: 2016-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,13 +56,14 @@ files:
 - gradlew.bat
 - lib/embulk/input/mongodb.rb
 - src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java
+- src/main/java/org/embulk/input/mongodb/ValueCodec.java
 - src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java
 - src/test/resources/basic.yml
 - src/test/resources/basic_expected.csv
 - src/test/resources/full.yml
 - src/test/resources/full_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.2.0.jar
+- classpath/embulk-input-mongodb-0.3.0.jar
 - classpath/mongo-java-driver-3.2.2.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses: