embulk-input-mongodb 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 669b8b7010ff112526a166dd4eef85a50656e77f
-  data.tar.gz: adea0810c4f04ee5cec5ec8ba7414557a8f179a2
+  metadata.gz: 50b7c55abdaeb8b32b0db09e8fdc8c87846ebc1a
+  data.tar.gz: 360dce308e8b5cfa97d8d6075cec6d4b05fc09cc
 SHA512:
-  metadata.gz: b8fad0bab206efc47861e6cb714c1def66cbfcb7440e042584daf454f84ef2673cdd2c8482bb3edfe3920d3bf82827c99c6cccb06d2890ea2a823be8122fffe8
-  data.tar.gz: 277157cc6382dbb98795f6e3e0ef3aaa14a31faec1f663069d9bc133f4b246a6cb73da39a2e5c5b7f80a36ae4ade7145cd01464b9c9feec896eb205dbec2ba7b
+  metadata.gz: 0e0e2d69147d2a7c571c6798d6d7485d28460b958fa4288380070841264bb229d55970f94a506d9d35a069de55017ab8d7e781ddf6ec57073bd8919be4334848
+  data.tar.gz: ae60160723083ca1ba44b2301205bd45b31c3c96fd71f8e66995ac1f10dfcff5307c8899cfed97bf199dc8eae22b81070f13a3518952a6fb4e5b832546b4cbeb

data/.travis.yml

@@ -12,6 +12,7 @@ env:
     - MONGO_URI=mongodb://localhost:27017/my_database
 
 sudo: required
+dist: precise
 
 before_cache:
   - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock

data/README.md

@@ -41,7 +41,7 @@ This plugin only works with embulk >= 0.8.8.
 - **incremental_field** List of field name (list, optional, can't use with sort option)
 - **last_record** Last loaded record for incremental load (hash, optional)
 - **stop_on_invalid_record** Stop bulk load transaction if a document includes invalid record (such as unsupported object type) (boolean, optional, default: false)
-- **json_column_name**: column name used in outputs (string, optional, default: "json")
+- **json_column_name**: column name used in outputs (string, optional, default: "record")
 
 ## Example
 

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.5.0"
+version = "0.5.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Aug 04 15:25:00 JST 2015
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java

@@ -110,6 +110,14 @@ public class MongodbInputPlugin
         String getSort();
         void setSort(String sort);
 
+        @Config("limit")
+        @ConfigDefault("null")
+        Optional<Integer> getLimit();
+
+        @Config("skip")
+        @ConfigDefault("null")
+        Optional<Integer> getSkip();
+
         @Config("id_field_name")
         @ConfigDefault("\"_id\"")
         String getIdFieldName();
@@ -152,6 +160,9 @@ public class MongodbInputPlugin
         if (task.getIncrementalField().isPresent() && !task.getSort().equals("{}")) {
             throw new ConfigException("both of sort and incremental_load can't be used together");
         }
+        if (task.getIncrementalField().isPresent() && task.getSkip().isPresent()) {
+            throw new ConfigException("both of skip and incremental_load can't be used together");
+        }
 
         Map<String, String> newCondition = buildIncrementalCondition(task);
         task.setQuery(newCondition.get("query"));
@@ -228,12 +239,20 @@ public class MongodbInputPlugin
         log.trace("query: {}", query);
         log.trace("projection: {}", projection);
         log.trace("sort: {}", sort);
+        if (task.getLimit().isPresent()) {
+            log.trace("limit: {}", task.getLimit());
+        }
+        if (task.getSkip().isPresent()) {
+            log.trace("skip: {}", task.getSkip());
+        }
 
         try (MongoCursor<Value> cursor = collection
                 .find(query)
                 .projection(projection)
                 .sort(sort)
                 .batchSize(task.getBatchSize())
+                .limit(task.getLimit().or(0))
+                .skip(task.getSkip().or(0))
                 .iterator()) {
             while (cursor.hasNext()) {
                 pageBuilder.setJson(column, cursor.next());

data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java

@@ -97,6 +97,8 @@ public class TestMongodbInputPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
         assertEquals("{}", task.getQuery());
         assertEquals("{}", task.getSort());
+        assertEquals(Optional.<Integer>absent(), task.getLimit());
+        assertEquals(Optional.<Integer>absent(), task.getSkip());
         assertEquals((long) 10000, (long) task.getBatchSize());
         assertEquals("record", task.getJsonColumnName());
         assertEquals(Optional.absent(), task.getIncrementalField());
@@ -134,7 +136,7 @@ public class TestMongodbInputPlugin
     }
 
     @Test(expected = ConfigException.class)
-    public void checkInvalidOptionCombination()
+    public void checkSortCannotUseWithIncremental()
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("uri", MONGO_URI)
@@ -145,6 +147,18 @@ public class TestMongodbInputPlugin
         plugin.transaction(config, new Control());
     }
 
+    @Test(expected = ConfigException.class)
+    public void checkSkipCannotUseWithIncremental()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("skip", 10)
+                .set("incremental_field", Optional.of(Arrays.asList("account")));
+
+        plugin.transaction(config, new Control());
+    }
+
     @Test(expected = ConfigException.class)
     public void checkInvalidQueryOption()
     {
@@ -200,6 +214,41 @@ public class TestMongodbInputPlugin
         assertValidRecords(getFieldSchema(), output);
     }
 
+    @Test
+    public void testRunWithLimit() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("limit", 1);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        plugin.transaction(config, new Control());
+        assertValidRecords(getFieldSchema(), output, 1, 0);
+    }
+
+    @Test
+    public void testRunWithLimitSkip() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("limit", 3)
+                .set("skip", 1);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        plugin.transaction(config, new Control());
+        assertValidRecords(getFieldSchema(), output, 3, 1);
+    }
+
     @Test
     public void testRunWithConnectionParams() throws Exception
     {
@@ -228,7 +277,7 @@ public class TestMongodbInputPlugin
         ConfigSource config = Exec.newConfigSource()
                 .set("uri", MONGO_URI)
                 .set("collection", MONGO_COLLECTION)
-                .set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")));
+                .set("incremental_field", Optional.of(Arrays.asList("int32_field")));
         PluginTask task = config.loadConfig(PluginTask.class);
 
         dropCollection(task, MONGO_COLLECTION);
@@ -238,7 +287,28 @@ public class TestMongodbInputPlugin
         ConfigDiff diff = plugin.transaction(config, new Control());
         ConfigDiff lastRecord = diff.getNested("last_record");
 
-        assertEquals("32864", lastRecord.get(String.class, "int32_field"));
+        assertEquals("5", lastRecord.get(String.class, "int32_field"));
+    }
+
+    @Test
+    public void testRunWithLimitIncrementalLoad() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("uri", MONGO_URI)
+                .set("collection", MONGO_COLLECTION)
+                .set("id_field_name", "int32_field")
+                .set("incremental_field", Optional.of(Arrays.asList("int32_field", "double_field", "datetime_field", "boolean_field")))
+                .set("limit", 1);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        dropCollection(task, MONGO_COLLECTION);
+        createCollection(task, MONGO_COLLECTION);
+        insertDocument(task, createValidDocuments());
+
+        ConfigDiff diff = plugin.transaction(config, new Control());
+        ConfigDiff lastRecord = diff.getNested("last_record");
+
+        assertEquals("1", lastRecord.get(String.class, "int32_field"));
         assertEquals("1.23", lastRecord.get(String.class, "double_field"));
         assertEquals("{$date=2015-01-27T10:23:49.000Z}", lastRecord.get(Map.class, "datetime_field").toString());
         assertEquals("true", lastRecord.get(String.class, "boolean_field"));
@@ -463,7 +533,7 @@ public class TestMongodbInputPlugin
                     .append("null_field", null)
                     .append("regex_field", new BsonRegularExpression(".+?"))
                     .append("javascript_field", new BsonJavaScript("var s = \"javascript\";"))
-                    .append("int32_field", 32864)
+                    .append("int32_field", 1)
                     .append("timestamp_field", new BsonTimestamp(1463991177, 4))
                     .append("int64_field", new BsonInt64(314159265))
                     .append("document_field", new Document("k", true))
@@ -472,14 +542,24 @@ public class TestMongodbInputPlugin
 
         documents.add(
             new Document("boolean_field", false)
+                    .append("int32_field", 2)
                     .append("document_field", new Document("k", 1))
         );
 
-        documents.add(new Document("document_field", new Document("k", 1.23)));
+        documents.add(
+            new Document("int32_field", 3)
+                    .append("document_field", new Document("k", 1.23))
+        );
 
-        documents.add(new Document("document_field", new Document("k", "v")));
+        documents.add(
+            new Document("int32_field", 4)
+                    .append("document_field", new Document("k", "v"))
+        );
 
-        documents.add(new Document("document_field", new Document("k", format.parse("2015-02-02T23:13:45.000Z"))));
+        documents.add(
+            new Document("int32_field", 5)
+                    .append("document_field", new Document("k", format.parse("2015-02-02T23:13:45.000Z")))
+        );
 
         return documents;
     }
@@ -493,49 +573,60 @@ public class TestMongodbInputPlugin
 
     private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception
     {
+        assertValidRecords(schema, output, 5, 0);
+    }
+
+    private void assertValidRecords(Schema schema, MockPageOutput output, int limit, int skip) throws Exception
+    {
+        int maxRecordSize = 5;
+        int actualRecordSize = Math.min(maxRecordSize - skip, limit);
         List<Object[]> records = Pages.toObjects(schema, output.pages);
-        assertEquals(5, records.size());
+        assertEquals(actualRecordSize, records.size());
 
         ObjectMapper mapper = new ObjectMapper();
         mapper.setDateFormat(getUTCDateFormat());
 
-        {
-            JsonNode node = mapper.readTree(records.get(0)[0].toString());
-            assertThat(1.23, is(node.get("double_field").asDouble()));
-            assertEquals("embulk", node.get("string_field").asText());
-            assertEquals("[1,2,3]", node.get("array_field").toString());
-            assertEquals("test", node.get("binary_field").asText());
-            assertEquals(true, node.get("boolean_field").asBoolean());
-            assertEquals("2015-01-27T10:23:49.000Z", node.get("datetime_field").asText());
-            assertEquals("null", node.get("null_field").asText());
-            assertEquals("BsonRegularExpression{pattern='.+?', options=''}", node.get("regex_field").asText());
-            assertEquals("var s = \"javascript\";", node.get("javascript_field").asText());
-            assertEquals(32864L, node.get("int32_field").asLong());
-            assertEquals("1463991177", node.get("timestamp_field").asText());
-            assertEquals(314159265L, node.get("int64_field").asLong());
-            assertEquals("{\"k\":true}", node.get("document_field").toString());
-            assertEquals("symbol", node.get("symbol_field").asText());
-        }
+        int recordIndex = 0;
+        for (int i = skip; i < actualRecordSize; i++) {
+            if (i == 0) {
+                JsonNode node = mapper.readTree(records.get(recordIndex)[0].toString());
+                assertThat(1.23, is(node.get("double_field").asDouble()));
+                assertEquals("embulk", node.get("string_field").asText());
+                assertEquals("[1,2,3]", node.get("array_field").toString());
+                assertEquals("test", node.get("binary_field").asText());
+                assertEquals(true, node.get("boolean_field").asBoolean());
+                assertEquals("2015-01-27T10:23:49.000Z", node.get("datetime_field").asText());
+                assertEquals("null", node.get("null_field").asText());
+                assertEquals("BsonRegularExpression{pattern='.+?', options=''}", node.get("regex_field").asText());
+                assertEquals("var s = \"javascript\";", node.get("javascript_field").asText());
+                assertEquals(1, node.get("int32_field").asLong());
+                assertEquals("1463991177", node.get("timestamp_field").asText());
+                assertEquals(314159265L, node.get("int64_field").asLong());
+                assertEquals("{\"k\":true}", node.get("document_field").toString());
+                assertEquals("symbol", node.get("symbol_field").asText());
+            }
 
-        {
-            JsonNode node = mapper.readTree(records.get(1)[0].toString());
-            assertEquals(false, node.get("boolean_field").asBoolean());
-            assertEquals("{\"k\":1}", node.get("document_field").toString());
-        }
+            if (i == 1) {
+                JsonNode node = mapper.readTree(records.get(recordIndex)[0].toString());
+                assertEquals(false, node.get("boolean_field").asBoolean());
+                assertEquals("{\"k\":1}", node.get("document_field").toString());
+            }
 
-        {
-            JsonNode node = mapper.readTree(records.get(2)[0].toString());
-            assertEquals("{\"k\":1.23}", node.get("document_field").toString());
-        }
+            if (i == 2) {
+                JsonNode node = mapper.readTree(records.get(recordIndex)[0].toString());
+                assertEquals("{\"k\":1.23}", node.get("document_field").toString());
+            }
 
-        {
-            JsonNode node = mapper.readTree(records.get(3)[0].toString());
-            assertEquals("{\"k\":\"v\"}", node.get("document_field").toString());
-        }
+            if (i == 3) {
+                JsonNode node = mapper.readTree(records.get(recordIndex)[0].toString());
+                assertEquals("{\"k\":\"v\"}", node.get("document_field").toString());
+            }
 
-        {
-            JsonNode node = mapper.readTree(records.get(4)[0].toString());
-            assertEquals("{\"k\":\"2015-02-02T23:13:45.000Z\"}", node.get("document_field").toString());
+            if (i == 4) {
+                JsonNode node = mapper.readTree(records.get(recordIndex)[0].toString());
+                assertEquals("{\"k\":\"2015-02-02T23:13:45.000Z\"}", node.get("document_field").toString());
+            }
+            recordIndex++;
         }
     }
 

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-23 00:00:00.000000000 Z
+date: 2017-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Loads records from Mongodb.
 email:
 - hakobera@gmail.com
@@ -68,7 +68,7 @@ files:
 - src/test/resources/id_field_name.yml
 - src/test/resources/id_field_name_expected.csv
 - src/test/resources/my_collection.jsonl
-- classpath/embulk-input-mongodb-0.5.0.jar
+- classpath/embulk-input-mongodb-0.5.1.jar
 - classpath/mongo-java-driver-3.2.2.jar
 homepage: https://github.com/hakobera/embulk-input-mongodb
 licenses: