embulk-output-mongodb_nest 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bc028440fde1626b707aa7f822053d7ae7d3a790
|
4
|
+
data.tar.gz: 5341b11cea2e731f2eb6ec9623495e728aa9d65c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6a00fb540bcd9e4eab341d356f75259efdf23baa2e4dbfebe84c94e6537827db53556b3ec35d421d8ad940077e4b63883115a227edda113eceb260cd9af97ae0
|
7
|
+
data.tar.gz: 55130dec190f57f664defe11c1c536b283799e1d87c0e4755178f4ae8d1f44717249427561dab58a53df6f6e580bce22fc22f6d1140e4f2b08f7a3368dc926c0
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -23,6 +23,14 @@ Dumps records to Mongodb with subdocument
|
|
23
23
|
- **key**: primary keys of collection (string array, required, **since v0.1.2**)
|
24
24
|
- **child**: describe subdocument (object list, default: null)
|
25
25
|
- **bulk_size**: bulk upsert size at time (integer, default: `10000`)
|
26
|
+
- **null_value**: set the value to fill if the field value is null (object, optional, **since v0.1.3**)
|
27
|
+
-- **string** : default string value (string, default: "")
|
28
|
+
-- **long** : default long value (long, default: 0)
|
29
|
+
-- **boolean** : default boolean value (boolean, default: false)
|
30
|
+
-- **double** : default double value (double, default: 0.0)
|
31
|
+
-- **json** : default json value (object, default: "{})
|
32
|
+
-- **timestamp** : default timestamp value (timestamp, default : "1970-01-01T00:00:00.000Z")
|
33
|
+
|
26
34
|
|
27
35
|
|
28
36
|
## Example
|
@@ -41,6 +49,13 @@ out:
|
|
41
49
|
- {name: mychild, field: time}
|
42
50
|
- {name: yourchild, field: comment}
|
43
51
|
- {name: mychild, field: purchase}
|
52
|
+
null_value:
|
53
|
+
string: ""
|
54
|
+
long: -1
|
55
|
+
json: {key: value}
|
56
|
+
boolean: false
|
57
|
+
double: 0.0
|
58
|
+
timestamp: "1970-01-01T00:00:00.000Z"
|
44
59
|
```
|
45
60
|
|
46
61
|
* Embulk input csv sample configuration
|
@@ -80,10 +95,16 @@ out:
|
|
80
95
|
- {name: mychild, field: time}
|
81
96
|
- {name: yourchild, field: comment}
|
82
97
|
- {name: mychild, field: purchase}
|
98
|
+
null_value:
|
99
|
+
string: ""
|
100
|
+
long: -1
|
101
|
+
json: {key: value}
|
102
|
+
timestamp: "1990-01-01T00:00:00.000Z"
|
83
103
|
```
|
84
104
|
|
85
105
|
|
86
106
|
## Release
|
107
|
+
* 0.1.3 2018-06-25 set the value to fill if the field value is null
|
87
108
|
* 0.1.2 2018-06-08 The key field has been changed to a list type. The JSON Parser is supported by the input plugin.
|
88
109
|
* 0.1.1 2018-06-05 [bug fix] Fiexed an error when a child field was entered
|
89
110
|
* 0.1.0 2018-05-31 first release
|
data/build.gradle
CHANGED
@@ -1,15 +1,20 @@
|
|
1
1
|
package org.embulk.output.mongodb_nest;
|
2
|
-
import java.
|
2
|
+
import java.text.SimpleDateFormat;
|
3
|
+
import java.util.*;
|
4
|
+
|
5
|
+
import ch.qos.logback.classic.LoggerContext;
|
6
|
+
import ch.qos.logback.classic.util.ContextInitializer;
|
7
|
+
import ch.qos.logback.core.util.StatusPrinter;
|
8
|
+
import com.google.common.base.Function;
|
3
9
|
import com.google.common.base.Optional;
|
4
|
-
import
|
5
|
-
import
|
6
|
-
import org.embulk.config
|
7
|
-
import org.embulk.config.ConfigSource;
|
8
|
-
import org.embulk.config.Task;
|
9
|
-
import org.embulk.config.TaskReport;
|
10
|
-
import org.embulk.config.TaskSource;
|
10
|
+
import com.google.common.base.Supplier;
|
11
|
+
import com.mongodb.BasicDBObject;
|
12
|
+
import org.embulk.config.*;
|
11
13
|
import org.embulk.spi.*;
|
12
14
|
import org.slf4j.Logger;
|
15
|
+
import org.slf4j.LoggerFactory;
|
16
|
+
|
17
|
+
import javax.annotation.Nullable;
|
13
18
|
|
14
19
|
/**
|
15
20
|
* <pre>
|
@@ -34,6 +39,33 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
34
39
|
String getField();
|
35
40
|
}
|
36
41
|
|
42
|
+
public interface DefineNullValue extends Task
|
43
|
+
{
|
44
|
+
@Config("string")
|
45
|
+
@ConfigDefault("")
|
46
|
+
String getString();
|
47
|
+
|
48
|
+
@Config("json")
|
49
|
+
@ConfigDefault("{}")
|
50
|
+
BasicDBObject getJson();
|
51
|
+
|
52
|
+
@Config("boolean")
|
53
|
+
@ConfigDefault("false")
|
54
|
+
Boolean getBoolean();
|
55
|
+
|
56
|
+
@Config("double")
|
57
|
+
@ConfigDefault("0.0")
|
58
|
+
Double getDouble();
|
59
|
+
|
60
|
+
@Config("long")
|
61
|
+
@ConfigDefault("0")
|
62
|
+
Integer getLong();
|
63
|
+
|
64
|
+
@Config("timestamp")
|
65
|
+
@ConfigDefault("0")
|
66
|
+
java.sql.Timestamp getTimestamp();
|
67
|
+
}
|
68
|
+
|
37
69
|
public interface PluginTask extends Task
|
38
70
|
{
|
39
71
|
@Config("collection")
|
@@ -63,9 +95,13 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
63
95
|
Optional<List<DefineChildDocument>> getChild();
|
64
96
|
|
65
97
|
@Config("bulk_size")
|
66
|
-
@ConfigDefault("
|
98
|
+
@ConfigDefault("10000")
|
67
99
|
int getBulkSize();
|
68
100
|
|
101
|
+
@Config("null_value")
|
102
|
+
@ConfigDefault("{\"string\": \"\", \"boolean\":\"false\"}")
|
103
|
+
Optional<DefineNullValue> getNullValue();
|
104
|
+
|
69
105
|
}
|
70
106
|
|
71
107
|
@Override public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control)
|
@@ -77,7 +113,21 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
77
113
|
|
78
114
|
// non-retryable (non-idempotent) output:
|
79
115
|
|
80
|
-
|
116
|
+
logger.debug(task.toString());
|
117
|
+
|
118
|
+
TimeZone timeZone = TimeZone.getTimeZone("UTC");
|
119
|
+
Calendar calendar = Calendar.getInstance(timeZone);
|
120
|
+
SimpleDateFormat simpleDateFormat =
|
121
|
+
new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z' zzz", Locale.KOREA);
|
122
|
+
simpleDateFormat.setTimeZone(timeZone);
|
123
|
+
|
124
|
+
logger.info("* setup null value");
|
125
|
+
logger.info(" String value = \"{}\"", task.getNullValue().get().getString());
|
126
|
+
logger.info(" Long value = {}", task.getNullValue().get().getLong());
|
127
|
+
logger.info(" Timestamp value = {}", simpleDateFormat.format(task.getNullValue().get().getTimestamp()));
|
128
|
+
logger.info(" Boolean value = {}", task.getNullValue().get().getBoolean());
|
129
|
+
logger.info(" Double value = {}", task.getNullValue().get().getDouble());
|
130
|
+
logger.info(" Json value = {}", task.getNullValue().get().getJson());
|
81
131
|
|
82
132
|
control.run(task.dump());
|
83
133
|
return Exec.newConfigDiff();
|
@@ -96,6 +146,7 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
96
146
|
{
|
97
147
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
98
148
|
|
149
|
+
|
99
150
|
return new PluginPageOutput(task, schema);
|
100
151
|
}
|
101
152
|
}
|
@@ -8,6 +8,7 @@ import com.mongodb.client.MongoDatabase;
|
|
8
8
|
import com.mongodb.client.model.ReplaceOneModel;
|
9
9
|
import com.mongodb.client.model.ReplaceOptions;
|
10
10
|
import com.mongodb.client.model.WriteModel;
|
11
|
+
import org.bson.BSON;
|
11
12
|
import org.bson.BSONObject;
|
12
13
|
import org.bson.Document;
|
13
14
|
import org.embulk.config.TaskReport;
|
@@ -87,33 +88,31 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
87
88
|
String t = schema.getColumnName(i);
|
88
89
|
Class<?> type = schema.getColumnType(i).getJavaType();
|
89
90
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
}
|
94
|
-
else if(schema.getColumnType(i).getName().compareTo("json") == 0)
|
91
|
+
boolean isnull = pageReader.isNull(i);
|
92
|
+
|
93
|
+
if(schema.getColumnType(i).getName().compareTo("json") == 0)
|
95
94
|
{
|
96
|
-
doc.putAll((BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
|
95
|
+
doc.putAll(isnull ? task.getNullValue().get().getJson() : (BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
|
97
96
|
}
|
98
97
|
else if (type.equals(boolean.class))
|
99
98
|
{
|
100
|
-
doc.append(t, pageReader.getBoolean(i));
|
99
|
+
doc.append(t, isnull ? task.getNullValue().get().getBoolean() : pageReader.getBoolean(i));
|
101
100
|
}
|
102
101
|
else if (type.equals(double.class))
|
103
102
|
{
|
104
|
-
doc.append(t, pageReader.getDouble(i));
|
103
|
+
doc.append(t, isnull ? task.getNullValue().get().getDouble() : pageReader.getDouble(i));
|
105
104
|
}
|
106
105
|
else if (type.equals(long.class))
|
107
106
|
{
|
108
|
-
doc.append(t, pageReader.getLong(i));
|
107
|
+
doc.append(t, isnull ? task.getNullValue().get().getLong() : pageReader.getLong(i));
|
109
108
|
}
|
110
109
|
else if (type.equals(String.class))
|
111
110
|
{
|
112
|
-
doc.append(t, pageReader.getString(i));
|
111
|
+
doc.append(t, isnull ? task.getNullValue().get().getString() : pageReader.getString(i));
|
113
112
|
}
|
114
113
|
else if (type.equals(Timestamp.class))
|
115
114
|
{
|
116
|
-
doc.append(t, new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
|
115
|
+
doc.append(t, isnull ? task.getNullValue().get().getTimestamp() : new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
|
117
116
|
}
|
118
117
|
|
119
118
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-mongodb_nest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- focuschange
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -60,7 +60,7 @@ files:
|
|
60
60
|
- src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java
|
61
61
|
- src/test/java/org/embulk/output/mongodb_nest/TestMongodbNestOutputPlugin.java
|
62
62
|
- classpath/mongo-java-driver-3.7.0.jar
|
63
|
-
- classpath/embulk-output-mongodb_nest-0.1.
|
63
|
+
- classpath/embulk-output-mongodb_nest-0.1.3.jar
|
64
64
|
homepage: https://github.com/focuschange/embulk-output-mongodb_nest
|
65
65
|
licenses:
|
66
66
|
- MIT
|