embulk-output-mongodb_nest 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 44cd8770b458a56c1302a320b7a835a474b8a066
|
4
|
+
data.tar.gz: 9164e109a20b9f1e21892787f6691bb87789a94c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5fcf602b68ceaa13fdc39a502e4130b68d345d556bad626442cd68b95308a4d4977e43173e99b9087ab26c4e41c3d836b54cff36f7acd22b4d15564ba7c3de7a
|
7
|
+
data.tar.gz: b4a64273bdbe248d6cc284683ae634ae5e56a29b54fb0f8f55c665ee2afeaee95f8375c92d3c371f4a7a221dbb66215326b03aa3b6f1c9a70f17a67d39ca0d8a
|
data/README.md
CHANGED
@@ -20,7 +20,7 @@ Dumps records to Mongodb with subdocument
|
|
20
20
|
- **user**: mongodb account (string, required)
|
21
21
|
- **password**: password (string, required)
|
22
22
|
- **collection**: collection name (string, required)
|
23
|
-
- **key**: primary
|
23
|
+
- **key**: primary keys of collection (string array, required, **since v0.1.2**)
|
24
24
|
- **child**: describe subdocument (object list, default: null)
|
25
25
|
- **bulk_size**: bulk upsert size at time (integer, default: `10000`)
|
26
26
|
|
@@ -36,7 +36,7 @@ out:
|
|
36
36
|
user: your-account-name
|
37
37
|
password: your-password
|
38
38
|
collection: your-collection-name
|
39
|
-
key: your-
|
39
|
+
key: [your-key1, key2, ...]
|
40
40
|
child:
|
41
41
|
- {name: mychild, field: time}
|
42
42
|
- {name: yourchild, field: comment}
|
@@ -75,7 +75,7 @@ out:
|
|
75
75
|
user: your-account-name
|
76
76
|
password: your-password
|
77
77
|
collection: your-collection-name
|
78
|
-
key: your-key
|
78
|
+
key: [your-key, key2]
|
79
79
|
child:
|
80
80
|
- {name: mychild, field: time}
|
81
81
|
- {name: yourchild, field: comment}
|
@@ -83,4 +83,9 @@ out:
|
|
83
83
|
```
|
84
84
|
|
85
85
|
|
86
|
+
## Release
|
87
|
+
* 0.1.2 2018-06-08 The key field has been changed to a list type. The JSON Parser is supported by the input plugin.
|
88
|
+
* 0.1.1 2018-06-05 [bug fix] Fiexed an error when a child field was entered
|
89
|
+
* 0.1.0 2018-05-31 first release
|
90
|
+
|
86
91
|
|
data/build.gradle
CHANGED
@@ -28,43 +28,43 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
28
28
|
public interface DefineChildDocument extends Task
|
29
29
|
{
|
30
30
|
@Config("name")
|
31
|
-
|
31
|
+
String getName();
|
32
32
|
|
33
33
|
@Config("field")
|
34
|
-
|
34
|
+
String getField();
|
35
35
|
}
|
36
36
|
|
37
37
|
public interface PluginTask extends Task
|
38
38
|
{
|
39
39
|
@Config("collection")
|
40
|
-
|
40
|
+
String getCollection();
|
41
41
|
|
42
42
|
@Config("host")
|
43
|
-
|
43
|
+
String getHost();
|
44
44
|
|
45
45
|
@Config("port")
|
46
46
|
@ConfigDefault("27017")
|
47
|
-
|
47
|
+
int getPort();
|
48
48
|
|
49
49
|
@Config("database")
|
50
|
-
|
50
|
+
String getDatabase();
|
51
51
|
|
52
52
|
@Config("user")
|
53
|
-
|
53
|
+
String getUser();
|
54
54
|
|
55
55
|
@Config("password")
|
56
|
-
|
56
|
+
String getPassword();
|
57
57
|
|
58
58
|
@Config("key")
|
59
|
-
|
59
|
+
List<String> getKey();
|
60
60
|
|
61
61
|
@Config("child")
|
62
62
|
@ConfigDefault("null")
|
63
|
-
|
63
|
+
Optional<List<DefineChildDocument>> getChild();
|
64
64
|
|
65
65
|
@Config("bulk_size")
|
66
66
|
@ConfigDefault("1000")
|
67
|
-
|
67
|
+
int getBulkSize();
|
68
68
|
|
69
69
|
}
|
70
70
|
|
@@ -76,6 +76,9 @@ public class MongodbNestOutputPlugin implements OutputPlugin
|
|
76
76
|
// return resume(task.dump(), schema, taskCount, control);
|
77
77
|
|
78
78
|
// non-retryable (non-idempotent) output:
|
79
|
+
|
80
|
+
System.out.println(config.toString());
|
81
|
+
|
79
82
|
control.run(task.dump());
|
80
83
|
return Exec.newConfigDiff();
|
81
84
|
}
|
@@ -8,6 +8,7 @@ import com.mongodb.client.MongoDatabase;
|
|
8
8
|
import com.mongodb.client.model.ReplaceOneModel;
|
9
9
|
import com.mongodb.client.model.ReplaceOptions;
|
10
10
|
import com.mongodb.client.model.WriteModel;
|
11
|
+
import org.bson.BSONObject;
|
11
12
|
import org.bson.Document;
|
12
13
|
import org.embulk.config.TaskReport;
|
13
14
|
import org.embulk.spi.*;
|
@@ -17,7 +18,6 @@ import org.slf4j.Logger;
|
|
17
18
|
import java.io.UnsupportedEncodingException;
|
18
19
|
import java.net.URLEncoder;
|
19
20
|
import java.util.ArrayList;
|
20
|
-
import java.util.Date;
|
21
21
|
import java.util.List;
|
22
22
|
import java.util.Map;
|
23
23
|
|
@@ -43,6 +43,7 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
43
43
|
private MongoDatabase db;
|
44
44
|
private MongoCollection<Document> collection;
|
45
45
|
|
46
|
+
|
46
47
|
PluginPageOutput(MongodbNestOutputPlugin.PluginTask task, Schema schema)
|
47
48
|
{
|
48
49
|
this.pageReader = new PageReader(schema);
|
@@ -71,7 +72,8 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
71
72
|
this.collection = this.db.getCollection(task.getCollection());
|
72
73
|
}
|
73
74
|
|
74
|
-
@Override
|
75
|
+
@Override
|
76
|
+
public void add(Page page)
|
75
77
|
{
|
76
78
|
pageReader.setPage(page);
|
77
79
|
List<WriteModel<Document>> replaceModel = new ArrayList<>();
|
@@ -89,6 +91,10 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
89
91
|
{
|
90
92
|
doc.append(t, null);
|
91
93
|
}
|
94
|
+
else if(schema.getColumnType(i).getName().compareTo("json") == 0)
|
95
|
+
{
|
96
|
+
doc.putAll((BSONObject) BasicDBObject.parse(pageReader.getJson(i).toJson()));
|
97
|
+
}
|
92
98
|
else if (type.equals(boolean.class))
|
93
99
|
{
|
94
100
|
doc.append(t, pageReader.getBoolean(i));
|
@@ -109,6 +115,7 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
109
115
|
{
|
110
116
|
doc.append(t, new java.sql.Timestamp(pageReader.getTimestamp(i).toEpochMilli()));
|
111
117
|
}
|
118
|
+
|
112
119
|
}
|
113
120
|
|
114
121
|
if (task.getChild().isPresent())
|
@@ -117,10 +124,9 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
117
124
|
}
|
118
125
|
|
119
126
|
replaceModel.add(new ReplaceOneModel<>(
|
120
|
-
|
127
|
+
generateFilter(doc),
|
121
128
|
new Document(doc),
|
122
|
-
new ReplaceOptions().upsert(true))
|
123
|
-
);
|
129
|
+
new ReplaceOptions().upsert(true)));
|
124
130
|
|
125
131
|
if(replaceModel.size() % task.getBulkSize() == 0)
|
126
132
|
{
|
@@ -133,7 +139,18 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
133
139
|
{
|
134
140
|
collection.bulkWrite(replaceModel);
|
135
141
|
}
|
142
|
+
}
|
143
|
+
|
144
|
+
private Document generateFilter(BasicDBObject document)
|
145
|
+
{
|
146
|
+
Document filter = new Document();
|
147
|
+
|
148
|
+
for(String k : task.getKey())
|
149
|
+
{
|
150
|
+
filter.append(k, document.get(k));
|
151
|
+
}
|
136
152
|
|
153
|
+
return filter;
|
137
154
|
}
|
138
155
|
|
139
156
|
private BasicDBObject transformDocument(BasicDBObject doc)
|
@@ -158,22 +175,26 @@ public class PluginPageOutput implements TransactionalPageOutput
|
|
158
175
|
return doc;
|
159
176
|
}
|
160
177
|
|
161
|
-
@Override
|
178
|
+
@Override
|
179
|
+
public void finish()
|
162
180
|
{
|
163
181
|
|
164
182
|
}
|
165
183
|
|
166
|
-
@Override
|
184
|
+
@Override
|
185
|
+
public void close()
|
167
186
|
{
|
168
187
|
this.mongo.close();
|
169
188
|
}
|
170
189
|
|
171
|
-
@Override
|
190
|
+
@Override
|
191
|
+
public void abort()
|
172
192
|
{
|
173
193
|
|
174
194
|
}
|
175
195
|
|
176
|
-
@Override
|
196
|
+
@Override
|
197
|
+
public TaskReport commit()
|
177
198
|
{
|
178
199
|
return null;
|
179
200
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-mongodb_nest
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- focuschange
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -60,7 +60,7 @@ files:
|
|
60
60
|
- src/main/java/org/embulk/output/mongodb_nest/PluginPageOutput.java
|
61
61
|
- src/test/java/org/embulk/output/mongodb_nest/TestMongodbNestOutputPlugin.java
|
62
62
|
- classpath/mongo-java-driver-3.7.0.jar
|
63
|
-
- classpath/embulk-output-mongodb_nest-0.1.
|
63
|
+
- classpath/embulk-output-mongodb_nest-0.1.2.jar
|
64
64
|
homepage: https://github.com/focuschange/embulk-output-mongodb_nest
|
65
65
|
licenses:
|
66
66
|
- MIT
|