embulk-input-mongodb 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a7ad81777e97ec36dcc249d0c6173f959db0bad
-  data.tar.gz: 3ab60b863f590fe28b3c822461b64acca465bb83
+  metadata.gz: 1c0c287fc6caae40f480044e097b35c22748be2d
+  data.tar.gz: 31c40a09bfd9aa1d3333105a49b02807c7fefb5b
 SHA512:
-  metadata.gz: c50855225aecb8680f9a89f9d84e6522b765a8b829338597a2f7bc377644b9ca20cb3c10633ca766e708e38aba19312bd60ea8119066501c5a9267aa21f73824
-  data.tar.gz: 4d6c3e3a77cf787c6c6e1361aee1c879c2e72d1aaa15cd8b1c9e44eb426732bf580625e2b5e622309a394767d0af6ad5a9ac07150f9b12730f51156060e3b3e7
+  metadata.gz: 013109760ca0b9040ea5367384d61c283801c97c833c1da0b52625ee8927dadefe71c6cd0f15a4115e4184b4fbdf40faed93eb2406fc3385ed4fb85eef5c3150
+  data.tar.gz: 2e1e7a1dcac758ede878293a37c9a95a532cb177964c9208d05f64f97c9ad3b1040ec5f9ce37abb25ddc53af07d379abbe24b9edf6c67f2b12e3474c300c2530

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.6.0"
+version = "0.6.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Sat Dec 23 16:12:16 JST 2017
+#Tue May 08 13:56:53 JST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

data/src/main/java/org/embulk/input/mongodb/HostTask.java

@@ -0,0 +1,16 @@
+package org.embulk.input.mongodb;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+
+public interface HostTask
+        extends Task
+{
+    @Config("host")
+    String getHost();
+
+    @Config("port")
+    @ConfigDefault("27017")
+    int getPort();
+}

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.input.mongodb;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
+import com.mongodb.BasicDBObject;
 import com.mongodb.MongoClient;
 import com.mongodb.MongoClientURI;
 import com.mongodb.MongoCredential;
@@ -11,21 +12,16 @@ import com.mongodb.ServerAddress;
 import com.mongodb.client.MongoCollection;
 import com.mongodb.client.MongoCursor;
 import com.mongodb.client.MongoDatabase;
-import com.mongodb.util.JSON;
-import com.mongodb.util.JSONParseException;
 import org.bson.codecs.configuration.CodecRegistries;
 import org.bson.codecs.configuration.CodecRegistry;
 import org.bson.conversions.Bson;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
+import org.bson.json.JsonParseException;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.DataSource;
 import org.embulk.config.DataSourceImpl;
 import org.embulk.config.ModelManager;
-import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
@@ -35,13 +31,10 @@ import org.embulk.spi.InputPlugin;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.type.Types;
 import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
-import javax.validation.constraints.Min;
-
 import java.io.IOException;
 import java.net.UnknownHostException;
 import java.util.ArrayList;
@@ -54,99 +47,6 @@ import java.util.Map;
 public class MongodbInputPlugin
         implements InputPlugin
 {
-    public interface HostTask
-            extends Task
-    {
-        @Config("host")
-        String getHost();
-
-        @Config("port")
-        @ConfigDefault("27017")
-        int getPort();
-    }
-
-    public interface PluginTask
-            extends Task
-    {
-        // MongoDB connection string URI
-        @Config("uri")
-        @ConfigDefault("null")
-        Optional<String> getUri();
-
-        @Config("hosts")
-        @ConfigDefault("null")
-        Optional<List<HostTask>> getHosts();
-
-        @Config("user")
-        @ConfigDefault("null")
-        Optional<String> getUser();
-
-        @Config("password")
-        @ConfigDefault("null")
-        Optional<String> getPassword();
-
-        @Config("database")
-        @ConfigDefault("null")
-        Optional<String> getDatabase();
-
-        @Config("collection")
-        String getCollection();
-
-        @Config("fields")
-        @ConfigDefault("null")
-        Optional<SchemaConfig> getFields();
-
-        @Config("projection")
-        @ConfigDefault("\"{}\"")
-        String getProjection();
-
-        @Config("query")
-        @ConfigDefault("\"{}\"")
-        String getQuery();
-        void setQuery(String query);
-
-        @Config("sort")
-        @ConfigDefault("\"{}\"")
-        String getSort();
-        void setSort(String sort);
-
-        @Config("limit")
-        @ConfigDefault("null")
-        Optional<Integer> getLimit();
-
-        @Config("skip")
-        @ConfigDefault("null")
-        Optional<Integer> getSkip();
-
-        @Config("id_field_name")
-        @ConfigDefault("\"_id\"")
-        String getIdFieldName();
-
-        @Config("batch_size")
-        @ConfigDefault("10000")
-        @Min(1)
-        int getBatchSize();
-
-        @Config("stop_on_invalid_record")
-        @ConfigDefault("false")
-        boolean getStopOnInvalidRecord();
-
-        @Config("json_column_name")
-        @ConfigDefault("\"record\"")
-        String getJsonColumnName();
-
-        @Config("incremental_field")
-        @ConfigDefault("null")
-        Optional<List<String>> getIncrementalField();
-
-        @Config("last_record")
-        @ConfigDefault("null")
-        Optional<Map<String, Object>> getLastRecord();
-
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
     private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
 
     @Override
@@ -232,9 +132,9 @@ public class MongodbInputPlugin
             throw new ConfigException(ex);
         }
 
-        Bson query = (Bson) JSON.parse(task.getQuery());
-        Bson projection = (Bson) JSON.parse(task.getProjection());
-        Bson sort = (Bson) JSON.parse(task.getSort());
+        Bson query = BasicDBObject.parse(task.getQuery());
+        Bson projection = BasicDBObject.parse(task.getProjection());
+        Bson sort = BasicDBObject.parse(task.getSort());
 
         log.trace("query: {}", query);
         log.trace("projection: {}", projection);
@@ -263,11 +163,13 @@ public class MongodbInputPlugin
         }
 
         pageBuilder.finish();
+        return updateTaskReport(Exec.newTaskReport(), valueCodec, task);
+    }
 
-        TaskReport report = Exec.newTaskReport();
-
-        if (valueCodec.getLastRecord() != null) {
-            DataSource lastRecord = new DataSourceImpl(Exec.getInjector().getInstance(ModelManager.class));
+    private TaskReport updateTaskReport(TaskReport report, ValueCodec valueCodec, PluginTask task)
+    {
+        DataSource lastRecord = new DataSourceImpl(Exec.getInjector().getInstance(ModelManager.class));
+        if (valueCodec.getLastRecord() != null && valueCodec.getProcessedRecordCount() > 0) {
             for (String k : valueCodec.getLastRecord().keySet()) {
                 String value = valueCodec.getLastRecord().get(k).toString();
                 Map<String, String> types = valueCodec.getLastRecordType();
@@ -299,8 +201,15 @@ public class MongodbInputPlugin
                         lastRecord.set(k, value);
                 }
             }
-            report.setNested("last_record", lastRecord);
         }
+        else if (task.getIncrementalField().isPresent() && task.getLastRecord().isPresent()) {
+            for (String field : task.getIncrementalField().get()) {
+                if (task.getLastRecord().get().containsKey(field)) {
+                    lastRecord.set(field, task.getLastRecord().get().get(field));
+                }
+            }
+        }
+        report.setNested("last_record", lastRecord);
         return report;
     }
 
@@ -414,7 +323,7 @@ public class MongodbInputPlugin
 
             return result;
         }
-        catch (JSONParseException | IOException ex) {
+        catch (JsonParseException | IOException ex) {
             throw new ConfigException("Could not generate new query for incremental load.");
         }
     }
@@ -422,9 +331,9 @@ public class MongodbInputPlugin
     private void validateJsonField(String name, String jsonString)
     {
         try {
-            JSON.parse(jsonString);
+            BasicDBObject.parse(jsonString);
         }
-        catch (JSONParseException ex) {
+        catch (JsonParseException ex) {
             throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));
         }
     }

data/src/main/java/org/embulk/input/mongodb/PluginTask.java

@@ -0,0 +1,96 @@
+package org.embulk.input.mongodb;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.Task;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.SchemaConfig;
+
+import javax.validation.constraints.Min;
+
+import java.util.List;
+import java.util.Map;
+
+public interface PluginTask
+        extends Task
+{
+    // MongoDB connection string URI
+    @Config("uri")
+    @ConfigDefault("null")
+    Optional<String> getUri();
+
+    @Config("hosts")
+    @ConfigDefault("null")
+    Optional<List<HostTask>> getHosts();
+
+    @Config("user")
+    @ConfigDefault("null")
+    Optional<String> getUser();
+
+    @Config("password")
+    @ConfigDefault("null")
+    Optional<String> getPassword();
+
+    @Config("database")
+    @ConfigDefault("null")
+    Optional<String> getDatabase();
+
+    @Config("collection")
+    String getCollection();
+
+    @Config("fields")
+    @ConfigDefault("null")
+    Optional<SchemaConfig> getFields();
+
+    @Config("projection")
+    @ConfigDefault("\"{}\"")
+    String getProjection();
+
+    @Config("query")
+    @ConfigDefault("\"{}\"")
+    String getQuery();
+    void setQuery(String query);
+
+    @Config("sort")
+    @ConfigDefault("\"{}\"")
+    String getSort();
+    void setSort(String sort);
+
+    @Config("limit")
+    @ConfigDefault("null")
+    Optional<Integer> getLimit();
+
+    @Config("skip")
+    @ConfigDefault("null")
+    Optional<Integer> getSkip();
+
+    @Config("id_field_name")
+    @ConfigDefault("\"_id\"")
+    String getIdFieldName();
+
+    @Config("batch_size")
+    @ConfigDefault("10000")
+    @Min(1)
+    int getBatchSize();
+
+    @Config("stop_on_invalid_record")
+    @ConfigDefault("false")
+    boolean getStopOnInvalidRecord();
+
+    @Config("json_column_name")
+    @ConfigDefault("\"record\"")
+    String getJsonColumnName();
+
+    @Config("incremental_field")
+    @ConfigDefault("null")
+    Optional<List<String>> getIncrementalField();
+
+    @Config("last_record")
+    @ConfigDefault("null")
+    Optional<Map<String, Object>> getLastRecord();
+
+    @ConfigInject
+    BufferAllocator getBufferAllocator();
+}

data/src/main/java/org/embulk/input/mongodb/ValueCodec.java

@@ -34,12 +34,13 @@ public class ValueCodec implements Codec<Value>
     private final SimpleDateFormat formatter;
     private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
     private final boolean stopOnInvalidRecord;
-    private final MongodbInputPlugin.PluginTask task;
+    private final PluginTask task;
     private final Optional<List<String>> incrementalField;
     private Map<String, Object> lastRecord;
+    private long processedRecordCount = 0;
     private Map<String, String> lastRecordType;
 
-    public ValueCodec(boolean stopOnInvalidRecord, MongodbInputPlugin.PluginTask task)
+    public ValueCodec(boolean stopOnInvalidRecord, PluginTask task)
     {
         this.formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
         formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
@@ -84,6 +85,7 @@ public class ValueCodec implements Codec<Value>
                 log.warn(String.format("Skipped document because field '%s' contains unsupported object type [%s]",
                         fieldName, type));
             }
+            this.processedRecordCount++;
         }
         reader.readEndDocument();
 
@@ -166,6 +168,11 @@ public class ValueCodec implements Codec<Value>
         return this.lastRecord;
     }
 
+    public Long getProcessedRecordCount()
+    {
+        return this.processedRecordCount;
+    }
+
     public Map<String, String> getLastRecordType()
     {
         return this.lastRecordType;

data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java

@@ -23,7 +23,6 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.input.mongodb.MongodbInputPlugin.PluginTask;
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.InputPlugin;
@@ -314,6 +313,35 @@ public class TestMongodbInputPlugin
         assertEquals("true", lastRecord.get(String.class, "boolean_field"));
     }
 
+    @Test
+    public void testRunWithLimitIncrementalLoadWithNoRecord() throws Exception
+    {
+        Map<String, Object> previousLastRecord = new HashMap<>();
+        previousLastRecord.put("int32_field", 1);
+        previousLastRecord.put("datetime_field", "{$date=2015-01-27T10:23:49.000Z}");
+        previousLastRecord.put("boolean_field", true);
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("id_field_name", "int32_field")
+                .set("query", "{\"double_field\":{\"$gte\": 1.23}}")
+                .set("incremental_field", Optional.of(Arrays.asList("int32_field", "datetime_field", "boolean_field")))
+                .set("last_record", previousLastRecord);
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        ConfigDiff diff = plugin.transaction(config, new Control());
+        ConfigDiff lastRecord = diff.getNested("last_record");
+
+        assertEquals("1", lastRecord.get(String.class, "int32_field"));
+        assertEquals("{$date=2015-01-27T10:23:49.000Z}", lastRecord.get(String.class, "datetime_field"));
+        assertEquals("true", lastRecord.get(String.class, "boolean_field"));
+    }
+
     @Test(expected = ConfigException.class)
     public void testRunWithIncrementalLoadUnsupportedType() throws Exception
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-10 00:00:00.000000000 Z
+date: 2018-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,9 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/input/mongodb.rb
+- src/main/java/org/embulk/input/mongodb/HostTask.java
 - src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java
+- src/main/java/org/embulk/input/mongodb/PluginTask.java
 - src/main/java/org/embulk/input/mongodb/ValueCodec.java
 - src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java
 - src/test/resources/basic.yml
@@ -68,7 +70,7 @@ files:
 - src/test/resources/id_field_name.yml
 - src/test/resources/id_field_name_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.6.0.jar
+- classpath/embulk-input-mongodb-0.6.1.jar
 - classpath/mongo-java-driver-3.6.1.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses: