embulk-output-mongodb_nest 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 805ea709694587e6cc1e272f3f8b73342282f406
-  data.tar.gz: c6197817e97777b57495cdd92c63c2351f54aeaf
+  metadata.gz: 44cd8770b458a56c1302a320b7a835a474b8a066
+  data.tar.gz: 9164e109a20b9f1e21892787f6691bb87789a94c
 SHA512:
-  metadata.gz: a80bf91cb0001014cc8487479743ab01e5a0f3a3ab82ba26ff6c0a7862b3eabccee2ae417981c6af8c3953169b6f5c3325011afb00f975832415832f1e0a14cd
-  data.tar.gz: 072c9a6331ce5c3bab228706d2e9c403d66d62d35899317d5e792529fa868bc59d2f5de21e9db2a7130ecbb4aaa3dc7b5bae3ceb5ed9ad20de0df0f5e4642183
+  metadata.gz: 5fcf602b68ceaa13fdc39a502e4130b68d345d556bad626442cd68b95308a4d4977e43173e99b9087ab26c4e41c3d836b54cff36f7acd22b4d15564ba7c3de7a
+  data.tar.gz: b4a64273bdbe248d6cc284683ae634ae5e56a29b54fb0f8f55c665ee2afeaee95f8375c92d3c371f4a7a221dbb66215326b03aa3b6f1c9a70f17a67d39ca0d8a

data/README.md

@@ -20,7 +20,7 @@ Dumps records to Mongodb with subdocument
 - **user**: mongodb account (string, required)
 - **password**: password (string, required)
 - **collection**: collection name (string, required)
-- **key**: primary key of collection (string, required)
+- **key**: primary keys of collection (string array, required, **since v0.1.2**)
 - **child**: describe subdocument  (object list, default: null)
 - **bulk_size**: bulk upsert size at time (integer, default: `10000`)
 
@@ -36,7 +36,7 @@ out:
   user: your-account-name
   password: your-password
   collection: your-collection-name
-  key: your-key
+  key: [your-key1, key2, ...]
   child:
   - {name: mychild, field: time}
   - {name: yourchild, field: comment}
@@ -75,7 +75,7 @@ out:
   user: your-account-name
   password: your-password
   collection: your-collection-name
-  key: your-key
+  key: [your-key, key2]
   child:
   - {name: mychild, field: time}
   - {name: yourchild, field: comment}
@@ -83,4 +83,9 @@ out:
 ```
 
 
+## Release
+* 0.1.2 2018-06-08 The key field has been changed to a list type. The JSON Parser is supported by the input plugin.
+* 0.1.1 2018-06-05 [bug fix] Fiexed an error when a child field was entered
+* 0.1.0 2018-05-31 first release
+
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/mongodb_nest/MongodbNestOutputPlugin.java

@@ -28,43 +28,43 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 	public interface DefineChildDocument extends Task
 	{
 		@Config("name")
-		public String getName();
+		String getName();
 
 		@Config("field")
-		public String getField();
+		String getField();
 	}
 
 	public interface PluginTask extends Task
 	{
 		@Config("collection")
-		public String getCollection();
+		String getCollection();
 
 		@Config("host")
-		public String getHost();
+		String getHost();
 
 		@Config("port")
 		@ConfigDefault("27017")
-		public int getPort();
+		int getPort();
 
 		@Config("database")
-		public String getDatabase();
+		String getDatabase();
 
 		@Config("user")
-		public String getUser();
+		String getUser();
 
 		@Config("password")
-		public String getPassword();
+		String getPassword();
 
 		@Config("key")
-		public String getKey();
+		List<String> getKey();
 
 		@Config("child")
 		@ConfigDefault("null")
-		public Optional<List<DefineChildDocument>> getChild();
+		Optional<List<DefineChildDocument>> getChild();
 
 		@Config("bulk_size")
 		@ConfigDefault("1000")
-		public int getBulkSize();
+		int getBulkSize();
 
 	}
 
@@ -76,6 +76,9 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 		// return resume(task.dump(), schema, taskCount, control);
 
 		// non-retryable (non-idempotent) output:
+
+		System.out.println(config.toString());
+
 		control.run(task.dump());
 		return Exec.newConfigDiff();
 	}

data/src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java

@@ -8,6 +8,7 @@ import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.model.ReplaceOneModel;
 import com.mongodb.client.model.ReplaceOptions;
 import com.mongodb.client.model.WriteModel;
+import org.bson.BSONObject;
 import org.bson.Document;
 import org.embulk.config.TaskReport;
 import org.embulk.spi.*;
@@ -17,7 +18,6 @@ import org.slf4j.Logger;
 import java.io.UnsupportedEncodingException;
 import java.net.URLEncoder;
 import java.util.ArrayList;
-import java.util.Date;
 import java.util.List;
 import java.util.Map;
 
@@ -43,6 +43,7 @@ public class PluginPageOutput implements TransactionalPageOutput
 	private MongoDatabase             db;
 	private MongoCollection<Document> collection;
 
+
 	PluginPageOutput(MongodbNestOutputPlugin.PluginTask task, Schema schema)
 	{
 		this.pageReader = new PageReader(schema);
@@ -71,7 +72,8 @@ public class PluginPageOutput implements TransactionalPageOutput
 		this.collection = this.db.getCollection(task.getCollection());
 	}
 
-	@Override public void add(Page page)
+	@Override
+	public void add(Page page)
 	{
 		pageReader.setPage(page);
 		List<WriteModel<Document>> replaceModel = new ArrayList<>();
@@ -89,6 +91,10 @@ public class PluginPageOutput implements TransactionalPageOutput
 				{
 					doc.append(t, null);
 				}
+				else if(schema.getColumnType(i).getName().compareTo("json") == 0)
+				{
+					doc.putAll((BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
+				}
 				else if (type.equals(boolean.class))
 				{
 					doc.append(t, pageReader.getBoolean(i));
@@ -109,6 +115,7 @@ public class PluginPageOutput implements TransactionalPageOutput
 				{
 					doc.append(t, new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
 				}
+
 			}
 
 			if (task.getChild().isPresent())
@@ -117,10 +124,9 @@ public class PluginPageOutput implements TransactionalPageOutput
 			}
 
 			replaceModel.add(new ReplaceOneModel<>(
-					new Document(task.getKey(), doc.get(task.getKey())),
+					generateFilter(doc),
 					new Document(doc),
-					new ReplaceOptions().upsert(true))
-			);
+					new ReplaceOptions().upsert(true)));
 
 			if(replaceModel.size() % task.getBulkSize() == 0)
 			{
@@ -133,7 +139,18 @@ public class PluginPageOutput implements TransactionalPageOutput
 		{
 			collection.bulkWrite(replaceModel);
 		}
+	}
+
+	private Document generateFilter(BasicDBObject document)
+	{
+		Document filter = new Document();
+
+		for(String k : task.getKey())
+		{
+			filter.append(k, document.get(k));
+		}
 
+		return filter;
 	}
 
 	private BasicDBObject transformDocument(BasicDBObject doc)
@@ -158,22 +175,26 @@ public class PluginPageOutput implements TransactionalPageOutput
 		return doc;
 	}
 
-	@Override public void finish()
+	@Override
+	public void finish()
 	{
 
 	}
 
-	@Override public void close()
+	@Override
+	public void close()
 	{
 		this.mongo.close();
 	}
 
-	@Override public void abort()
+	@Override
+	public void abort()
 	{
 
 	}
 
-	@Override public TaskReport commit()
+	@Override
+	public TaskReport commit()
 	{
 		return null;
 	}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mongodb_nest
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - focuschange
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-05 00:00:00.000000000 Z
+date: 2018-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -60,7 +60,7 @@ files:
 - src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java
 - src/test/java/org/embulk/output/mongodb_nest/TestMongodbNestOutputPlugin.java
 - classpath/mongo-java-driver-3.7.0.jar
-- classpath/embulk-output-mongodb_nest-0.1.1.jar
+- classpath/embulk-output-mongodb_nest-0.1.2.jar
 homepage: https://github.com/focuschange/embulk-output-mongodb_nest
 licenses:
 - MIT