embulk-output-mongodb_nest 0.1.2 → 0.1.3

This diff has not been reviewed by any users.
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 44cd8770b458a56c1302a320b7a835a474b8a066
4
- data.tar.gz: 9164e109a20b9f1e21892787f6691bb87789a94c
3
+ metadata.gz: bc028440fde1626b707aa7f822053d7ae7d3a790
4
+ data.tar.gz: 5341b11cea2e731f2eb6ec9623495e728aa9d65c
5
5
  SHA512:
6
- metadata.gz: 5fcf602b68ceaa13fdc39a502e4130b68d345d556bad626442cd68b95308a4d4977e43173e99b9087ab26c4e41c3d836b54cff36f7acd22b4d15564ba7c3de7a
7
- data.tar.gz: b4a64273bdbe248d6cc284683ae634ae5e56a29b54fb0f8f55c665ee2afeaee95f8375c92d3c371f4a7a221dbb66215326b03aa3b6f1c9a70f17a67d39ca0d8a
6
+ metadata.gz: 6a00fb540bcd9e4eab341d356f75259efdf23baa2e4dbfebe84c94e6537827db53556b3ec35d421d8ad940077e4b63883115a227edda113eceb260cd9af97ae0
7
+ data.tar.gz: 55130dec190f57f664defe11c1c536b283799e1d87c0e4755178f4ae8d1f44717249427561dab58a53df6f6e580bce22fc22f6d1140e4f2b08f7a3368dc926c0
data/.gitignore CHANGED
@@ -10,3 +10,5 @@ build/
10
10
  /.metadata/
11
11
  .classpath
12
12
  .project
13
+ /logs/
14
+ /sample/
data/README.md CHANGED
@@ -23,6 +23,14 @@ Dumps records to Mongodb with subdocument
23
23
  - **key**: primary keys of collection (string array, required, **since v0.1.2**)
24
24
  - **child**: describe subdocument (object list, default: null)
25
25
  - **bulk_size**: bulk upsert size at time (integer, default: `10000`)
26
+ - **null_value**: set the value to fill if the field value is null (object, optional, **since v0.1.3**)
27
+ -- **string** : default string value (string, default: "")
28
+ -- **long** : default long value (long, default: 0)
29
+ -- **boolean** : default boolean value (boolean, default: false)
30
+ -- **double** : default double value (double, default: 0.0)
31
+ -- **json** : default json value (object, default: "{})
32
+ -- **timestamp** : default timestamp value (timestamp, default : "1970-01-01T00:00:00.000Z")
33
+
26
34
 
27
35
 
28
36
  ## Example
@@ -41,6 +49,13 @@ out:
41
49
  - {name: mychild, field: time}
42
50
  - {name: yourchild, field: comment}
43
51
  - {name: mychild, field: purchase}
52
+ null_value:
53
+ string: ""
54
+ long: -1
55
+ json: {key: value}
56
+ boolean: false
57
+ double: 0.0
58
+ timestamp: "1970-01-01T00:00:00.000Z"
44
59
  ```
45
60
 
46
61
  * Embulk input csv sample configuration
@@ -80,10 +95,16 @@ out:
80
95
  - {name: mychild, field: time}
81
96
  - {name: yourchild, field: comment}
82
97
  - {name: mychild, field: purchase}
98
+ null_value:
99
+ string: ""
100
+ long: -1
101
+ json: {key: value}
102
+ timestamp: "1990-01-01T00:00:00.000Z"
83
103
  ```
84
104
 
85
105
 
86
106
  ## Release
107
+ * 0.1.3 2018-06-25 set the value to fill if the field value is null
87
108
  * 0.1.2 2018-06-08 The key field has been changed to a list type. The JSON Parser is supported by the input plugin.
88
109
  * 0.1.1 2018-06-05 [bug fix] Fiexed an error when a child field was entered
89
110
  * 0.1.0 2018-05-31 first release
@@ -13,7 +13,7 @@ configurations {
13
13
  provided
14
14
  }
15
15
 
16
- version = "0.1.2"
16
+ version = "0.1.3"
17
17
 
18
18
  sourceCompatibility = 1.8
19
19
  targetCompatibility = 1.8
@@ -1,15 +1,20 @@
1
1
  package org.embulk.output.mongodb_nest;
2
- import java.util.List;
2
+ import java.text.SimpleDateFormat;
3
+ import java.util.*;
4
+
5
+ import ch.qos.logback.classic.LoggerContext;
6
+ import ch.qos.logback.classic.util.ContextInitializer;
7
+ import ch.qos.logback.core.util.StatusPrinter;
8
+ import com.google.common.base.Function;
3
9
  import com.google.common.base.Optional;
4
- import org.embulk.config.Config;
5
- import org.embulk.config.ConfigDefault;
6
- import org.embulk.config.ConfigDiff;
7
- import org.embulk.config.ConfigSource;
8
- import org.embulk.config.Task;
9
- import org.embulk.config.TaskReport;
10
- import org.embulk.config.TaskSource;
10
+ import com.google.common.base.Supplier;
11
+ import com.mongodb.BasicDBObject;
12
+ import org.embulk.config.*;
11
13
  import org.embulk.spi.*;
12
14
  import org.slf4j.Logger;
15
+ import org.slf4j.LoggerFactory;
16
+
17
+ import javax.annotation.Nullable;
13
18
 
14
19
  /**
15
20
  * <pre>
@@ -34,6 +39,33 @@ public class MongodbNestOutputPlugin implements OutputPlugin
34
39
  String getField();
35
40
  }
36
41
 
42
+ public interface DefineNullValue extends Task
43
+ {
44
+ @Config("string")
45
+ @ConfigDefault("")
46
+ String getString();
47
+
48
+ @Config("json")
49
+ @ConfigDefault("{}")
50
+ BasicDBObject getJson();
51
+
52
+ @Config("boolean")
53
+ @ConfigDefault("false")
54
+ Boolean getBoolean();
55
+
56
+ @Config("double")
57
+ @ConfigDefault("0.0")
58
+ Double getDouble();
59
+
60
+ @Config("long")
61
+ @ConfigDefault("0")
62
+ Integer getLong();
63
+
64
+ @Config("timestamp")
65
+ @ConfigDefault("0")
66
+ java.sql.Timestamp getTimestamp();
67
+ }
68
+
37
69
  public interface PluginTask extends Task
38
70
  {
39
71
  @Config("collection")
@@ -63,9 +95,13 @@ public class MongodbNestOutputPlugin implements OutputPlugin
63
95
  Optional<List<DefineChildDocument>> getChild();
64
96
 
65
97
  @Config("bulk_size")
66
- @ConfigDefault("1000")
98
+ @ConfigDefault("10000")
67
99
  int getBulkSize();
68
100
 
101
+ @Config("null_value")
102
+ @ConfigDefault("{\"string\": \"\", \"boolean\":\"false\"}")
103
+ Optional<DefineNullValue> getNullValue();
104
+
69
105
  }
70
106
 
71
107
  @Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control)
@@ -77,7 +113,21 @@ public class MongodbNestOutputPlugin implements OutputPlugin
77
113
 
78
114
  // non-retryable (non-idempotent) output:
79
115
 
80
- System.out.println(config.toString());
116
+ logger.debug(task.toString());
117
+
118
+ TimeZone timeZone = TimeZone.getTimeZone("UTC");
119
+ Calendar calendar = Calendar.getInstance(timeZone);
120
+ SimpleDateFormat simpleDateFormat =
121
+ new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' zzz", Locale.KOREA);
122
+ simpleDateFormat.setTimeZone(timeZone);
123
+
124
+ logger.info("* setup null value");
125
+ logger.info(" String value = \"{}\"", task.getNullValue().get().getString());
126
+ logger.info(" Long value = {}", task.getNullValue().get().getLong());
127
+ logger.info(" Timestamp value = {}", simpleDateFormat.format(task.getNullValue().get().getTimestamp()));
128
+ logger.info(" Boolean value = {}", task.getNullValue().get().getBoolean());
129
+ logger.info(" Double value = {}", task.getNullValue().get().getDouble());
130
+ logger.info(" Json value = {}", task.getNullValue().get().getJson());
81
131
 
82
132
  control.run(task.dump());
83
133
  return Exec.newConfigDiff();
@@ -96,6 +146,7 @@ public class MongodbNestOutputPlugin implements OutputPlugin
96
146
  {
97
147
  PluginTask task = taskSource.loadTask(PluginTask.class);
98
148
 
149
+
99
150
  return new PluginPageOutput(task, schema);
100
151
  }
101
152
  }
@@ -8,6 +8,7 @@ import com.mongodb.client.MongoDatabase;
8
8
  import com.mongodb.client.model.ReplaceOneModel;
9
9
  import com.mongodb.client.model.ReplaceOptions;
10
10
  import com.mongodb.client.model.WriteModel;
11
+ import org.bson.BSON;
11
12
  import org.bson.BSONObject;
12
13
  import org.bson.Document;
13
14
  import org.embulk.config.TaskReport;
@@ -87,33 +88,31 @@ public class PluginPageOutput implements TransactionalPageOutput
87
88
  String t = schema.getColumnName(i);
88
89
  Class<?> type = schema.getColumnType(i).getJavaType();
89
90
 
90
- if (pageReader.isNull(i))
91
- {
92
- doc.append(t, null);
93
- }
94
- else if(schema.getColumnType(i).getName().compareTo("json") == 0)
91
+ boolean isnull = pageReader.isNull(i);
92
+
93
+ if(schema.getColumnType(i).getName().compareTo("json") == 0)
95
94
  {
96
- doc.putAll((BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
95
+ doc.putAll(isnull ? task.getNullValue().get().getJson() : (BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
97
96
  }
98
97
  else if (type.equals(boolean.class))
99
98
  {
100
- doc.append(t, pageReader.getBoolean(i));
99
+ doc.append(t, isnull ? task.getNullValue().get().getBoolean() : pageReader.getBoolean(i));
101
100
  }
102
101
  else if (type.equals(double.class))
103
102
  {
104
- doc.append(t, pageReader.getDouble(i));
103
+ doc.append(t, isnull ? task.getNullValue().get().getDouble() : pageReader.getDouble(i));
105
104
  }
106
105
  else if (type.equals(long.class))
107
106
  {
108
- doc.append(t, pageReader.getLong(i));
107
+ doc.append(t, isnull ? task.getNullValue().get().getLong() : pageReader.getLong(i));
109
108
  }
110
109
  else if (type.equals(String.class))
111
110
  {
112
- doc.append(t, pageReader.getString(i));
111
+ doc.append(t, isnull ? task.getNullValue().get().getString() : pageReader.getString(i));
113
112
  }
114
113
  else if (type.equals(Timestamp.class))
115
114
  {
116
- doc.append(t, new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
115
+ doc.append(t, isnull ? task.getNullValue().get().getTimestamp() : new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
117
116
  }
118
117
 
119
118
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-mongodb_nest
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - focuschange
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-06-08 00:00:00.000000000 Z
11
+ date: 2018-06-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -60,7 +60,7 @@ files:
60
60
  - src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java
61
61
  - src/test/java/org/embulk/output/mongodb_nest/TestMongodbNestOutputPlugin.java
62
62
  - classpath/mongo-java-driver-3.7.0.jar
63
- - classpath/embulk-output-mongodb_nest-0.1.2.jar
63
+ - classpath/embulk-output-mongodb_nest-0.1.3.jar
64
64
  homepage: https://github.com/focuschange/embulk-output-mongodb_nest
65
65
  licenses:
66
66
  - MIT