embulk-output-mongodb_nest 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44cd8770b458a56c1302a320b7a835a474b8a066
-  data.tar.gz: 9164e109a20b9f1e21892787f6691bb87789a94c
+  metadata.gz: bc028440fde1626b707aa7f822053d7ae7d3a790
+  data.tar.gz: 5341b11cea2e731f2eb6ec9623495e728aa9d65c
 SHA512:
-  metadata.gz: 5fcf602b68ceaa13fdc39a502e4130b68d345d556bad626442cd68b95308a4d4977e43173e99b9087ab26c4e41c3d836b54cff36f7acd22b4d15564ba7c3de7a
-  data.tar.gz: b4a64273bdbe248d6cc284683ae634ae5e56a29b54fb0f8f55c665ee2afeaee95f8375c92d3c371f4a7a221dbb66215326b03aa3b6f1c9a70f17a67d39ca0d8a
+  metadata.gz: 6a00fb540bcd9e4eab341d356f75259efdf23baa2e4dbfebe84c94e6537827db53556b3ec35d421d8ad940077e4b63883115a227edda113eceb260cd9af97ae0
+  data.tar.gz: 55130dec190f57f664defe11c1c536b283799e1d87c0e4755178f4ae8d1f44717249427561dab58a53df6f6e580bce22fc22f6d1140e4f2b08f7a3368dc926c0

data/.gitignore

@@ -10,3 +10,5 @@ build/
 /.metadata/
 .classpath
 .project
+/logs/
+/sample/

data/README.md

@@ -23,6 +23,14 @@ Dumps records to Mongodb with subdocument
 - **key**: primary keys of collection (string array, required, **since v0.1.2**)
 - **child**: describe subdocument  (object list, default: null)
 - **bulk_size**: bulk upsert size at time (integer, default: `10000`)
+- **null_value**: set the value to fill if the field value is null (object, optional, **since v0.1.3**)
+-- **string** : default string value (string, default: "")
+-- **long** : default long value (long, default: 0)
+-- **boolean** : default boolean value (boolean, default: false)
+-- **double** : default double value (double, default: 0.0)
+-- **json** : default json value (object, default: "{})
+-- **timestamp** : default timestamp value (timestamp, default : "1970-01-01T00:00:00.000Z")
+
 
 
 ## Example
@@ -41,6 +49,13 @@ out:
   - {name: mychild, field: time}
   - {name: yourchild, field: comment}
   - {name: mychild, field: purchase}
+  null_value:
+    string: ""
+    long: -1
+    json: {key: value}
+    boolean: false
+    double: 0.0
+    timestamp: "1970-01-01T00:00:00.000Z"
 ```
 
 * Embulk input csv sample configuration
@@ -80,10 +95,16 @@ out:
   - {name: mychild, field: time}
   - {name: yourchild, field: comment}
   - {name: mychild, field: purchase}
+  null_value:
+    string: ""
+    long: -1
+    json: {key: value}
+    timestamp: "1990-01-01T00:00:00.000Z"
 ```
 
 
 ## Release
+* 0.1.3 2018-06-25 set the value to fill if the field value is null
 * 0.1.2 2018-06-08 The key field has been changed to a list type. The JSON Parser is supported by the input plugin.
 * 0.1.1 2018-06-05 [bug fix] Fiexed an error when a child field was entered
 * 0.1.0 2018-05-31 first release

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/mongodb_nest/MongodbNestOutputPlugin.java

@@ -1,15 +1,20 @@
 package org.embulk.output.mongodb_nest;
-import java.util.List;
+import java.text.SimpleDateFormat;
+import java.util.*;
+
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.util.ContextInitializer;
+import ch.qos.logback.core.util.StatusPrinter;
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
+import com.google.common.base.Supplier;
+import com.mongodb.BasicDBObject;
+import org.embulk.config.*;
 import org.embulk.spi.*;
 import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
 
 /**
  * <pre>
@@ -34,6 +39,33 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 		String getField();
 	}
 
+	public interface DefineNullValue extends Task
+	{
+		@Config("string")
+		@ConfigDefault("")
+		String getString();
+
+		@Config("json")
+		@ConfigDefault("{}")
+		BasicDBObject getJson();
+
+		@Config("boolean")
+		@ConfigDefault("false")
+		Boolean getBoolean();
+
+		@Config("double")
+		@ConfigDefault("0.0")
+		Double getDouble();
+
+		@Config("long")
+		@ConfigDefault("0")
+		Integer getLong();
+
+		@Config("timestamp")
+		@ConfigDefault("0")
+		java.sql.Timestamp getTimestamp();
+	}
+
 	public interface PluginTask extends Task
 	{
 		@Config("collection")
@@ -63,9 +95,13 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 		Optional<List<DefineChildDocument>> getChild();
 
 		@Config("bulk_size")
-		@ConfigDefault("1000")
+		@ConfigDefault("10000")
 		int getBulkSize();
 
+		@Config("null_value")
+		@ConfigDefault("{\"string\": \"\", \"boolean\":\"false\"}")
+		Optional<DefineNullValue> getNullValue();
+
 	}
 
 	@Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control)
@@ -77,7 +113,21 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 
 		// non-retryable (non-idempotent) output:
 
-		System.out.println(config.toString());
+		logger.debug(task.toString());
+
+		TimeZone timeZone = TimeZone.getTimeZone("UTC");
+		Calendar calendar = Calendar.getInstance(timeZone);
+		SimpleDateFormat simpleDateFormat =
+				new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' zzz", Locale.KOREA);
+		simpleDateFormat.setTimeZone(timeZone);
+
+		logger.info("* setup null value");
+		logger.info("  String value    = \"{}\"", task.getNullValue().get().getString());
+		logger.info("  Long value      = {}", task.getNullValue().get().getLong());
+		logger.info("  Timestamp value = {}", simpleDateFormat.format(task.getNullValue().get().getTimestamp()));
+		logger.info("  Boolean value   = {}", task.getNullValue().get().getBoolean());
+		logger.info("  Double value    = {}", task.getNullValue().get().getDouble());
+		logger.info("  Json value      = {}", task.getNullValue().get().getJson());
 
 		control.run(task.dump());
 		return Exec.newConfigDiff();
@@ -96,6 +146,7 @@ public class MongodbNestOutputPlugin implements OutputPlugin
 	{
 		PluginTask task = taskSource.loadTask(PluginTask.class);
 
+
 		return new PluginPageOutput(task, schema);
 	}
 }

data/src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java

@@ -8,6 +8,7 @@ import com.mongodb.client.MongoDatabase;
 import com.mongodb.client.model.ReplaceOneModel;
 import com.mongodb.client.model.ReplaceOptions;
 import com.mongodb.client.model.WriteModel;
+import org.bson.BSON;
 import org.bson.BSONObject;
 import org.bson.Document;
 import org.embulk.config.TaskReport;
@@ -87,33 +88,31 @@ public class PluginPageOutput implements TransactionalPageOutput
 				String t = schema.getColumnName(i);
 				Class<?> type = schema.getColumnType(i).getJavaType();
 
-				if (pageReader.isNull(i))
-				{
-					doc.append(t, null);
-				}
-				else if(schema.getColumnType(i).getName().compareTo("json") == 0)
+				boolean isnull = pageReader.isNull(i);
+
+				if(schema.getColumnType(i).getName().compareTo("json") == 0)
 				{
-					doc.putAll((BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
+					doc.putAll(isnull ? task.getNullValue().get().getJson() : (BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
 				}
 				else if (type.equals(boolean.class))
 				{
-					doc.append(t, pageReader.getBoolean(i));
+					doc.append(t, isnull ? task.getNullValue().get().getBoolean() : pageReader.getBoolean(i));
 				}
 				else if (type.equals(double.class))
 				{
-					doc.append(t, pageReader.getDouble(i));
+					doc.append(t, isnull ? task.getNullValue().get().getDouble() : pageReader.getDouble(i));
 				}
 				else if (type.equals(long.class))
 				{
-					doc.append(t, pageReader.getLong(i));
+					doc.append(t, isnull ? task.getNullValue().get().getLong() : pageReader.getLong(i));
 				}
 				else if (type.equals(String.class))
 				{
-					doc.append(t, pageReader.getString(i));
+					doc.append(t, isnull ? task.getNullValue().get().getString() : pageReader.getString(i));
 				}
 				else if (type.equals(Timestamp.class))
 				{
-					doc.append(t, new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
+					doc.append(t, isnull ? task.getNullValue().get().getTimestamp() : new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
 				}
 
 			}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mongodb_nest
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - focuschange
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-08 00:00:00.000000000 Z
+date: 2018-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -60,7 +60,7 @@ files:
 - src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java
 - src/test/java/org/embulk/output/mongodb_nest/TestMongodbNestOutputPlugin.java
 - classpath/mongo-java-driver-3.7.0.jar
-- classpath/embulk-output-mongodb_nest-0.1.2.jar
+- classpath/embulk-output-mongodb_nest-0.1.3.jar
 homepage: https://github.com/focuschange/embulk-output-mongodb_nest
 licenses:
 - MIT