embulk-input-mongodb 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +25 -8
- data/README.md +1 -0
- data/build.gradle +19 -1
- data/classpath/embulk-input-mongodb-0.3.1.jar +0 -0
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java +17 -6
- data/src/main/java/org/embulk/input/mongodb/ValueCodec.java +25 -27
- data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java +82 -39
- data/src/test/resources/id_field_name.yml +18 -0
- data/src/test/resources/id_field_name_expected.csv +8 -0
- metadata +7 -3
- data/classpath/embulk-input-mongodb-0.3.0.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b2a60bb559e33bfa9fb7e8f9f0b6453e070d995e
|
|
4
|
+
data.tar.gz: b6da249c33d89b3a84d070f48e18a97243061bf5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1c3a1f2a3ae383587c548d5e7fcae302eeb17275e784498958e04436fc3e86fc49f2aa4197d2fae3436c009425c9414d4ee1bb358fd9bd2811241b4484c5f644
|
|
7
|
+
data.tar.gz: 89d72c6f271dfbfed98fbf5ffbbb72851cc4d48aaa4af202a1fb09fe07caaefd2f835b2a94ee7c3ac733992b5a3ec1233ace1ac45221fb50284263f20452387c
|
data/.travis.yml
CHANGED
|
@@ -5,8 +5,22 @@ jdk:
|
|
|
5
5
|
- oraclejdk7
|
|
6
6
|
- openjdk7
|
|
7
7
|
|
|
8
|
+
env:
|
|
9
|
+
global:
|
|
10
|
+
- MONGO_DATABASE=my_database
|
|
11
|
+
- MONGO_COLLECTION=my_collection
|
|
12
|
+
- MONGO_URI=mongodb://localhost:27017/my_database
|
|
13
|
+
|
|
8
14
|
sudo: required
|
|
9
15
|
|
|
16
|
+
before_cache:
|
|
17
|
+
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
|
|
18
|
+
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
|
|
19
|
+
cache:
|
|
20
|
+
directories:
|
|
21
|
+
- $HOME/.gradle/caches/
|
|
22
|
+
- $HOME/.gradle/wrapper/
|
|
23
|
+
|
|
10
24
|
# Work around fix for buffer overflow error on OpenJDK7
|
|
11
25
|
# ref: https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165131913
|
|
12
26
|
before_install:
|
|
@@ -29,14 +43,17 @@ install:
|
|
|
29
43
|
before_script:
|
|
30
44
|
- echo "Wait mongodb wakeup"
|
|
31
45
|
- sleep 10
|
|
32
|
-
- mongoimport --db my_database --collection my_collection --type json --drop src/test/resources/my_collection.jsonl
|
|
33
|
-
- ./gradlew package
|
|
34
46
|
- mkdir -p ./tmp
|
|
47
|
+
- date
|
|
35
48
|
|
|
36
49
|
script:
|
|
37
|
-
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
50
|
+
- ./gradlew check
|
|
51
|
+
- ./gradlew package
|
|
52
|
+
- mongoimport --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
|
|
53
|
+
- |
|
|
54
|
+
for target in basic full id_field_name
|
|
55
|
+
do
|
|
56
|
+
embulk run -L . src/test/resources/${target}.yml
|
|
57
|
+
cat tmp/${target}000.00.csv
|
|
58
|
+
cmp tmp/${target}000.00.csv src/test/resources/${target}_expected.csv
|
|
59
|
+
done
|
data/README.md
CHANGED
|
@@ -24,6 +24,7 @@ This plugin only works with embulk >= 0.8.8.
|
|
|
24
24
|
~~- double~~
|
|
25
25
|
~~- string~~
|
|
26
26
|
~~- timestamp~~
|
|
27
|
+
- **id_field_name** (string, optional, default: "_id") Name of Object ID field name. Set if you want to change the default name `_id`
|
|
27
28
|
- **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
|
|
28
29
|
- **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
|
|
29
30
|
- **sort**: ordering of results (string, optional)
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
id "jacoco"
|
|
6
7
|
}
|
|
7
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -16,7 +17,7 @@ configurations {
|
|
|
16
17
|
provided
|
|
17
18
|
}
|
|
18
19
|
|
|
19
|
-
version = "0.3.
|
|
20
|
+
version = "0.3.1"
|
|
20
21
|
|
|
21
22
|
sourceCompatibility = 1.7
|
|
22
23
|
targetCompatibility = 1.7
|
|
@@ -38,6 +39,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
38
39
|
}
|
|
39
40
|
clean { delete "classpath" }
|
|
40
41
|
|
|
42
|
+
checkstyle {
|
|
43
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
44
|
+
toolVersion = '6.14.1'
|
|
45
|
+
}
|
|
46
|
+
checkstyleMain {
|
|
47
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
48
|
+
ignoreFailures = true
|
|
49
|
+
}
|
|
50
|
+
checkstyleTest {
|
|
51
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
52
|
+
ignoreFailures = true
|
|
53
|
+
}
|
|
54
|
+
task checkstyle(type: Checkstyle) {
|
|
55
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
56
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
57
|
+
}
|
|
58
|
+
|
|
41
59
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
42
60
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
43
61
|
script "${project.name}.gemspec"
|
|
Binary file
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="ClassTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
<module name="MethodTypeParameterName">
|
|
92
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
93
|
+
</module>
|
|
94
|
+
|
|
95
|
+
<module name="WhitespaceAround">
|
|
96
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
97
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
98
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
99
|
+
<property name="tokens" value="
|
|
100
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
101
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
102
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
103
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
104
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
105
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
106
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
107
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
108
|
+
</module>
|
|
109
|
+
</module>
|
|
110
|
+
</module>
|
|
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
|
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -33,7 +33,9 @@ import org.embulk.spi.SchemaConfig;
|
|
|
33
33
|
import org.embulk.spi.type.Types;
|
|
34
34
|
import org.msgpack.value.Value;
|
|
35
35
|
import org.slf4j.Logger;
|
|
36
|
+
|
|
36
37
|
import javax.validation.constraints.Min;
|
|
38
|
+
|
|
37
39
|
import java.net.UnknownHostException;
|
|
38
40
|
import java.util.List;
|
|
39
41
|
|
|
@@ -66,6 +68,10 @@ public class MongodbInputPlugin
|
|
|
66
68
|
@ConfigDefault("\"{}\"")
|
|
67
69
|
String getSort();
|
|
68
70
|
|
|
71
|
+
@Config("id_field_name")
|
|
72
|
+
@ConfigDefault("\"_id\"")
|
|
73
|
+
String getIdFieldName();
|
|
74
|
+
|
|
69
75
|
@Config("batch_size")
|
|
70
76
|
@ConfigDefault("10000")
|
|
71
77
|
@Min(1)
|
|
@@ -101,7 +107,8 @@ public class MongodbInputPlugin
|
|
|
101
107
|
// Connect once to throw ConfigException in earlier stage of excecution
|
|
102
108
|
try {
|
|
103
109
|
connect(task);
|
|
104
|
-
}
|
|
110
|
+
}
|
|
111
|
+
catch (UnknownHostException | MongoException ex) {
|
|
105
112
|
throw new ConfigException(ex);
|
|
106
113
|
}
|
|
107
114
|
Schema schema = Schema.builder().add(task.getJsonColumnName(), Types.JSON).build();
|
|
@@ -141,11 +148,12 @@ public class MongodbInputPlugin
|
|
|
141
148
|
|
|
142
149
|
CodecRegistry registry = CodecRegistries.fromRegistries(
|
|
143
150
|
MongoClient.getDefaultCodecRegistry(),
|
|
144
|
-
CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord()))
|
|
151
|
+
CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord(), task))
|
|
145
152
|
);
|
|
146
153
|
collection = db.getCollection(task.getCollection(), Value.class)
|
|
147
154
|
.withCodecRegistry(registry);
|
|
148
|
-
}
|
|
155
|
+
}
|
|
156
|
+
catch (UnknownHostException | MongoException ex) {
|
|
149
157
|
throw new ConfigException(ex);
|
|
150
158
|
}
|
|
151
159
|
|
|
@@ -182,7 +190,8 @@ public class MongodbInputPlugin
|
|
|
182
190
|
return Exec.newConfigDiff();
|
|
183
191
|
}
|
|
184
192
|
|
|
185
|
-
private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException
|
|
193
|
+
private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException
|
|
194
|
+
{
|
|
186
195
|
MongoClientURI uri = new MongoClientURI(task.getUri());
|
|
187
196
|
MongoClient mongoClient = new MongoClient(uri);
|
|
188
197
|
|
|
@@ -192,10 +201,12 @@ public class MongodbInputPlugin
|
|
|
192
201
|
return db;
|
|
193
202
|
}
|
|
194
203
|
|
|
195
|
-
private void validateJsonField(String name, String jsonString)
|
|
204
|
+
private void validateJsonField(String name, String jsonString)
|
|
205
|
+
{
|
|
196
206
|
try {
|
|
197
207
|
JSON.parse(jsonString);
|
|
198
|
-
}
|
|
208
|
+
}
|
|
209
|
+
catch (JSONParseException ex) {
|
|
199
210
|
throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));
|
|
200
211
|
}
|
|
201
212
|
}
|
|
@@ -11,15 +11,6 @@ import org.embulk.spi.Exec;
|
|
|
11
11
|
import org.msgpack.value.Value;
|
|
12
12
|
import org.slf4j.Logger;
|
|
13
13
|
|
|
14
|
-
import static org.msgpack.value.ValueFactory.newArray;
|
|
15
|
-
import static org.msgpack.value.ValueFactory.newBinary;
|
|
16
|
-
import static org.msgpack.value.ValueFactory.newBoolean;
|
|
17
|
-
import static org.msgpack.value.ValueFactory.newFloat;
|
|
18
|
-
import static org.msgpack.value.ValueFactory.newInteger;
|
|
19
|
-
import static org.msgpack.value.ValueFactory.newMap;
|
|
20
|
-
import static org.msgpack.value.ValueFactory.newNil;
|
|
21
|
-
import static org.msgpack.value.ValueFactory.newString;
|
|
22
|
-
|
|
23
14
|
import java.text.SimpleDateFormat;
|
|
24
15
|
import java.util.ArrayList;
|
|
25
16
|
import java.util.Date;
|
|
@@ -28,39 +19,44 @@ import java.util.List;
|
|
|
28
19
|
import java.util.Map;
|
|
29
20
|
import java.util.TimeZone;
|
|
30
21
|
|
|
31
|
-
|
|
22
|
+
import static org.msgpack.value.ValueFactory.*;
|
|
23
|
+
|
|
24
|
+
public class ValueCodec implements Codec<Value>
|
|
25
|
+
{
|
|
32
26
|
private final SimpleDateFormat formatter;
|
|
33
27
|
private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
|
|
34
28
|
private final boolean stopOnInvalidRecord;
|
|
29
|
+
private final MongodbInputPlugin.PluginTask task;
|
|
35
30
|
|
|
36
|
-
public ValueCodec(boolean stopOnInvalidRecord)
|
|
31
|
+
public ValueCodec(boolean stopOnInvalidRecord, MongodbInputPlugin.PluginTask task)
|
|
32
|
+
{
|
|
37
33
|
this.formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
|
|
38
34
|
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
|
|
39
35
|
this.stopOnInvalidRecord = stopOnInvalidRecord;
|
|
36
|
+
this.task = task;
|
|
40
37
|
}
|
|
41
38
|
|
|
42
39
|
@Override
|
|
43
|
-
public void encode(final BsonWriter writer, final Value value, final EncoderContext encoderContext)
|
|
40
|
+
public void encode(final BsonWriter writer, final Value value, final EncoderContext encoderContext)
|
|
41
|
+
{
|
|
44
42
|
throw new UnsupportedOperationException();
|
|
45
43
|
}
|
|
46
44
|
|
|
47
45
|
@Override
|
|
48
|
-
public Value decode(final BsonReader reader, final DecoderContext decoderContext)
|
|
46
|
+
public Value decode(final BsonReader reader, final DecoderContext decoderContext)
|
|
47
|
+
{
|
|
49
48
|
Map<Value, Value> kvs = new LinkedHashMap<>();
|
|
50
49
|
|
|
51
50
|
reader.readStartDocument();
|
|
52
|
-
boolean isTopLevelNode = false;
|
|
53
51
|
while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
|
|
54
52
|
String fieldName = reader.readName();
|
|
55
53
|
BsonType type = reader.getCurrentBsonType();
|
|
56
|
-
|
|
57
|
-
isTopLevelNode = true;
|
|
58
|
-
}
|
|
59
|
-
fieldName = normalize(fieldName, isTopLevelNode);
|
|
54
|
+
fieldName = normalize(fieldName);
|
|
60
55
|
|
|
61
56
|
try {
|
|
62
57
|
kvs.put(newString(fieldName), readValue(reader, decoderContext));
|
|
63
|
-
}
|
|
58
|
+
}
|
|
59
|
+
catch (UnknownTypeFoundException ex) {
|
|
64
60
|
reader.skipValue();
|
|
65
61
|
if (stopOnInvalidRecord) {
|
|
66
62
|
throw ex;
|
|
@@ -74,7 +70,8 @@ public class ValueCodec implements Codec<Value> {
|
|
|
74
70
|
return newMap(kvs);
|
|
75
71
|
}
|
|
76
72
|
|
|
77
|
-
public Value decodeArray(final BsonReader reader, final DecoderContext decoderContext)
|
|
73
|
+
public Value decodeArray(final BsonReader reader, final DecoderContext decoderContext)
|
|
74
|
+
{
|
|
78
75
|
List<Value> list = new ArrayList<>();
|
|
79
76
|
|
|
80
77
|
reader.readStartArray();
|
|
@@ -86,7 +83,8 @@ public class ValueCodec implements Codec<Value> {
|
|
|
86
83
|
return newArray(list);
|
|
87
84
|
}
|
|
88
85
|
|
|
89
|
-
private Value readValue(BsonReader reader, DecoderContext decoderContext)
|
|
86
|
+
private Value readValue(BsonReader reader, DecoderContext decoderContext)
|
|
87
|
+
{
|
|
90
88
|
switch (reader.getCurrentBsonType()) {
|
|
91
89
|
// https://docs.mongodb.com/manual/reference/bson-types/
|
|
92
90
|
// https://github.com/mongodb/mongo-java-driver/tree/master/bson/src/main/org/bson/codecs
|
|
@@ -128,15 +126,15 @@ public class ValueCodec implements Codec<Value> {
|
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
@Override
|
|
131
|
-
public Class<Value> getEncoderClass()
|
|
129
|
+
public Class<Value> getEncoderClass()
|
|
130
|
+
{
|
|
132
131
|
return Value.class;
|
|
133
132
|
}
|
|
134
133
|
|
|
135
|
-
private String normalize(String key
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
return "_id";
|
|
134
|
+
private String normalize(String key)
|
|
135
|
+
{
|
|
136
|
+
if (key.equals("_id")) {
|
|
137
|
+
return task.getIdFieldName();
|
|
140
138
|
}
|
|
141
139
|
return key;
|
|
142
140
|
}
|
|
@@ -34,17 +34,20 @@ import org.junit.rules.ExpectedException;
|
|
|
34
34
|
|
|
35
35
|
import java.lang.reflect.InvocationTargetException;
|
|
36
36
|
import java.lang.reflect.Method;
|
|
37
|
+
import java.text.DateFormat;
|
|
37
38
|
import java.text.SimpleDateFormat;
|
|
38
39
|
import java.util.ArrayList;
|
|
39
40
|
import java.util.Arrays;
|
|
40
41
|
import java.util.List;
|
|
41
42
|
import java.util.Locale;
|
|
43
|
+
import java.util.TimeZone;
|
|
42
44
|
|
|
43
45
|
import static org.hamcrest.CoreMatchers.is;
|
|
44
46
|
import static org.junit.Assert.assertEquals;
|
|
45
47
|
import static org.junit.Assert.assertThat;
|
|
46
48
|
|
|
47
|
-
public class TestMongodbInputPlugin
|
|
49
|
+
public class TestMongodbInputPlugin
|
|
50
|
+
{
|
|
48
51
|
private static String MONGO_URI;
|
|
49
52
|
private static String MONGO_COLLECTION;
|
|
50
53
|
|
|
@@ -64,20 +67,23 @@ public class TestMongodbInputPlugin {
|
|
|
64
67
|
* MONGO_COLLECTION
|
|
65
68
|
*/
|
|
66
69
|
@BeforeClass
|
|
67
|
-
public static void initializeConstant()
|
|
70
|
+
public static void initializeConstant()
|
|
71
|
+
{
|
|
68
72
|
MONGO_URI = System.getenv("MONGO_URI");
|
|
69
73
|
MONGO_COLLECTION = System.getenv("MONGO_COLLECTION");
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
@Before
|
|
73
|
-
public void createResources() throws Exception
|
|
77
|
+
public void createResources() throws Exception
|
|
78
|
+
{
|
|
74
79
|
config = config();
|
|
75
80
|
plugin = new MongodbInputPlugin();
|
|
76
81
|
output = new MockPageOutput();
|
|
77
82
|
}
|
|
78
83
|
|
|
79
84
|
@Test
|
|
80
|
-
public void checkDefaultValues()
|
|
85
|
+
public void checkDefaultValues()
|
|
86
|
+
{
|
|
81
87
|
ConfigSource config = Exec.newConfigSource()
|
|
82
88
|
.set("uri", MONGO_URI)
|
|
83
89
|
.set("collection", MONGO_COLLECTION);
|
|
@@ -90,7 +96,8 @@ public class TestMongodbInputPlugin {
|
|
|
90
96
|
}
|
|
91
97
|
|
|
92
98
|
@Test(expected = ConfigException.class)
|
|
93
|
-
public void checkDefaultValuesUriIsNull()
|
|
99
|
+
public void checkDefaultValuesUriIsNull()
|
|
100
|
+
{
|
|
94
101
|
ConfigSource config = Exec.newConfigSource()
|
|
95
102
|
.set("uri", null)
|
|
96
103
|
.set("collection", MONGO_COLLECTION);
|
|
@@ -109,7 +116,8 @@ public class TestMongodbInputPlugin {
|
|
|
109
116
|
}
|
|
110
117
|
|
|
111
118
|
@Test(expected = ConfigException.class)
|
|
112
|
-
public void checkDefaultValuesCollectionIsNull()
|
|
119
|
+
public void checkDefaultValuesCollectionIsNull()
|
|
120
|
+
{
|
|
113
121
|
ConfigSource config = Exec.newConfigSource()
|
|
114
122
|
.set("uri", MONGO_URI)
|
|
115
123
|
.set("collection", null);
|
|
@@ -118,12 +126,14 @@ public class TestMongodbInputPlugin {
|
|
|
118
126
|
}
|
|
119
127
|
|
|
120
128
|
@Test
|
|
121
|
-
public void testResume()
|
|
129
|
+
public void testResume()
|
|
130
|
+
{
|
|
122
131
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
123
132
|
final Schema schema = getFieldSchema();
|
|
124
133
|
plugin.resume(task.dump(), schema, 0, new InputPlugin.Control() {
|
|
125
134
|
@Override
|
|
126
|
-
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
135
|
+
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
136
|
+
{
|
|
127
137
|
return emptyTaskReports(taskCount);
|
|
128
138
|
}
|
|
129
139
|
});
|
|
@@ -131,19 +141,22 @@ public class TestMongodbInputPlugin {
|
|
|
131
141
|
}
|
|
132
142
|
|
|
133
143
|
@Test
|
|
134
|
-
public void testCleanup()
|
|
144
|
+
public void testCleanup()
|
|
145
|
+
{
|
|
135
146
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
136
147
|
Schema schema = getFieldSchema();
|
|
137
148
|
plugin.cleanup(task.dump(), schema, 0, Lists.<TaskReport>newArrayList()); // no errors happens
|
|
138
149
|
}
|
|
139
150
|
|
|
140
151
|
@Test
|
|
141
|
-
public void testGuess()
|
|
152
|
+
public void testGuess()
|
|
153
|
+
{
|
|
142
154
|
plugin.guess(config); // no errors happens
|
|
143
155
|
}
|
|
144
156
|
|
|
145
157
|
@Test
|
|
146
|
-
public void testRun() throws Exception
|
|
158
|
+
public void testRun() throws Exception
|
|
159
|
+
{
|
|
147
160
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
148
161
|
|
|
149
162
|
dropCollection(task, MONGO_COLLECTION);
|
|
@@ -155,7 +168,8 @@ public class TestMongodbInputPlugin {
|
|
|
155
168
|
}
|
|
156
169
|
|
|
157
170
|
@Test(expected = ValueCodec.UnknownTypeFoundException.class)
|
|
158
|
-
public void testRunWithUnsupportedType() throws Exception
|
|
171
|
+
public void testRunWithUnsupportedType() throws Exception
|
|
172
|
+
{
|
|
159
173
|
ConfigSource config = Exec.newConfigSource()
|
|
160
174
|
.set("uri", MONGO_URI)
|
|
161
175
|
.set("collection", MONGO_COLLECTION)
|
|
@@ -176,33 +190,47 @@ public class TestMongodbInputPlugin {
|
|
|
176
190
|
}
|
|
177
191
|
|
|
178
192
|
@Test
|
|
179
|
-
public void testNormalize() throws Exception
|
|
180
|
-
|
|
193
|
+
public void testNormalize() throws Exception
|
|
194
|
+
{
|
|
195
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
196
|
+
ValueCodec codec = new ValueCodec(true, task);
|
|
181
197
|
|
|
182
|
-
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class
|
|
198
|
+
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class);
|
|
183
199
|
normalize.setAccessible(true);
|
|
184
|
-
assertEquals("_id", normalize.invoke(codec, "
|
|
185
|
-
assertEquals("
|
|
186
|
-
|
|
200
|
+
assertEquals("_id", normalize.invoke(codec, "_id").toString());
|
|
201
|
+
assertEquals("f1", normalize.invoke(codec, "f1").toString());
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@Test
|
|
205
|
+
public void testNormlizeWithIdFieldName() throws Exception
|
|
206
|
+
{
|
|
207
|
+
ConfigSource config = config().set("id_field_name", "object_id");
|
|
208
|
+
|
|
209
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
210
|
+
ValueCodec codec = new ValueCodec(true, task);
|
|
187
211
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
assertEquals("
|
|
212
|
+
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class);
|
|
213
|
+
normalize.setAccessible(true);
|
|
214
|
+
assertEquals("object_id", normalize.invoke(codec, "_id").toString());
|
|
215
|
+
assertEquals("f1", normalize.invoke(codec, "f1").toString());
|
|
191
216
|
}
|
|
192
217
|
|
|
193
218
|
@Test
|
|
194
|
-
public void testValidateJsonField() throws Exception
|
|
219
|
+
public void testValidateJsonField() throws Exception
|
|
220
|
+
{
|
|
195
221
|
Method validate = MongodbInputPlugin.class.getDeclaredMethod("validateJsonField", String.class, String.class);
|
|
196
222
|
validate.setAccessible(true);
|
|
197
223
|
String invalidJsonString = "{\"name\": invalid}";
|
|
198
224
|
try {
|
|
199
225
|
validate.invoke(plugin, "name", invalidJsonString);
|
|
200
|
-
}
|
|
226
|
+
}
|
|
227
|
+
catch (InvocationTargetException ex) {
|
|
201
228
|
assertEquals(ConfigException.class, ex.getCause().getClass());
|
|
202
229
|
}
|
|
203
230
|
}
|
|
204
231
|
|
|
205
|
-
static List<TaskReport> emptyTaskReports(int taskCount)
|
|
232
|
+
static List<TaskReport> emptyTaskReports(int taskCount)
|
|
233
|
+
{
|
|
206
234
|
ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
|
207
235
|
for (int i = 0; i < taskCount; i++) {
|
|
208
236
|
reports.add(Exec.newTaskReport());
|
|
@@ -211,9 +239,11 @@ public class TestMongodbInputPlugin {
|
|
|
211
239
|
}
|
|
212
240
|
|
|
213
241
|
private class Control
|
|
214
|
-
implements InputPlugin.Control
|
|
242
|
+
implements InputPlugin.Control
|
|
243
|
+
{
|
|
215
244
|
@Override
|
|
216
|
-
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
245
|
+
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
246
|
+
{
|
|
217
247
|
List<TaskReport> reports = new ArrayList<>();
|
|
218
248
|
for (int i = 0; i < taskCount; i++) {
|
|
219
249
|
reports.add(plugin.run(taskSource, schema, i, output));
|
|
@@ -222,24 +252,25 @@ public class TestMongodbInputPlugin {
|
|
|
222
252
|
}
|
|
223
253
|
}
|
|
224
254
|
|
|
225
|
-
private ConfigSource config()
|
|
255
|
+
private ConfigSource config()
|
|
256
|
+
{
|
|
226
257
|
return Exec.newConfigSource()
|
|
227
258
|
.set("uri", MONGO_URI)
|
|
228
|
-
.set("collection", MONGO_COLLECTION)
|
|
229
|
-
.set("last_path", "");
|
|
259
|
+
.set("collection", MONGO_COLLECTION);
|
|
230
260
|
}
|
|
231
261
|
|
|
232
|
-
private List<Document> createValidDocuments() throws Exception
|
|
233
|
-
|
|
262
|
+
private List<Document> createValidDocuments() throws Exception
|
|
263
|
+
{
|
|
264
|
+
DateFormat format = getUTCDateFormat();
|
|
234
265
|
|
|
235
266
|
List<Document> documents = new ArrayList<>();
|
|
236
267
|
documents.add(
|
|
237
268
|
new Document("double_field", 1.23)
|
|
238
269
|
.append("string_field", "embulk")
|
|
239
|
-
.append("array_field", Arrays.asList(1,2,3))
|
|
270
|
+
.append("array_field", Arrays.asList(1, 2, 3))
|
|
240
271
|
.append("binary_field", new BsonBinary(("test").getBytes("UTF-8")))
|
|
241
272
|
.append("boolean_field", true)
|
|
242
|
-
.append("datetime_field", format.parse("2015-01-
|
|
273
|
+
.append("datetime_field", format.parse("2015-01-27T10:23:49.000Z"))
|
|
243
274
|
.append("null_field", null)
|
|
244
275
|
.append("regex_field", new BsonRegularExpression(".+?"))
|
|
245
276
|
.append("javascript_field", new BsonJavaScript("var s = \"javascript\";"))
|
|
@@ -258,22 +289,25 @@ public class TestMongodbInputPlugin {
|
|
|
258
289
|
|
|
259
290
|
documents.add(new Document("document_field", new Document("k", "v")));
|
|
260
291
|
|
|
261
|
-
documents.add(new Document("document_field", new Document("k", format.parse("2015-02-
|
|
292
|
+
documents.add(new Document("document_field", new Document("k", format.parse("2015-02-02T23:13:45.000Z"))));
|
|
262
293
|
|
|
263
294
|
return documents;
|
|
264
295
|
}
|
|
265
296
|
|
|
266
|
-
private Schema getFieldSchema()
|
|
297
|
+
private Schema getFieldSchema()
|
|
298
|
+
{
|
|
267
299
|
ImmutableList.Builder<Column> columns = ImmutableList.builder();
|
|
268
300
|
columns.add(new Column(0, "record", Types.JSON));
|
|
269
301
|
return new Schema(columns.build());
|
|
270
302
|
}
|
|
271
303
|
|
|
272
|
-
private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception
|
|
304
|
+
private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception
|
|
305
|
+
{
|
|
273
306
|
List<Object[]> records = Pages.toObjects(schema, output.pages);
|
|
274
307
|
assertEquals(5, records.size());
|
|
275
308
|
|
|
276
309
|
ObjectMapper mapper = new ObjectMapper();
|
|
310
|
+
mapper.setDateFormat(getUTCDateFormat());
|
|
277
311
|
|
|
278
312
|
{
|
|
279
313
|
JsonNode node = mapper.readTree(records.get(0)[0].toString());
|
|
@@ -314,14 +348,16 @@ public class TestMongodbInputPlugin {
|
|
|
314
348
|
}
|
|
315
349
|
}
|
|
316
350
|
|
|
317
|
-
private void createCollection(PluginTask task, String collectionName) throws Exception
|
|
351
|
+
private void createCollection(PluginTask task, String collectionName) throws Exception
|
|
352
|
+
{
|
|
318
353
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
319
354
|
method.setAccessible(true);
|
|
320
355
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
321
356
|
db.createCollection(collectionName);
|
|
322
357
|
}
|
|
323
358
|
|
|
324
|
-
private void dropCollection(PluginTask task, String collectionName) throws Exception
|
|
359
|
+
private void dropCollection(PluginTask task, String collectionName) throws Exception
|
|
360
|
+
{
|
|
325
361
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
326
362
|
method.setAccessible(true);
|
|
327
363
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
@@ -329,11 +365,18 @@ public class TestMongodbInputPlugin {
|
|
|
329
365
|
collection.drop();
|
|
330
366
|
}
|
|
331
367
|
|
|
332
|
-
private void insertDocument(PluginTask task, List<Document> documents) throws Exception
|
|
368
|
+
private void insertDocument(PluginTask task, List<Document> documents) throws Exception
|
|
369
|
+
{
|
|
333
370
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
334
371
|
method.setAccessible(true);
|
|
335
372
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
336
373
|
MongoCollection collection = db.getCollection(task.getCollection());
|
|
337
374
|
collection.insertMany(documents);
|
|
338
375
|
}
|
|
376
|
+
|
|
377
|
+
private DateFormat getUTCDateFormat() {
|
|
378
|
+
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
|
|
379
|
+
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
|
|
380
|
+
return dateFormat;
|
|
381
|
+
}
|
|
339
382
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: mongodb
|
|
3
|
+
uri: mongodb://localhost:27017/my_database
|
|
4
|
+
collection: "my_collection"
|
|
5
|
+
json_column_name: "json"
|
|
6
|
+
query: '{ rank: { $gte: 3 } }'
|
|
7
|
+
sort: '{ rank: -1 }'
|
|
8
|
+
id_field_name: "object_id"
|
|
9
|
+
batch_size: 100
|
|
10
|
+
out:
|
|
11
|
+
type: file
|
|
12
|
+
path_prefix: ./tmp/id_field_name
|
|
13
|
+
file_ext: csv
|
|
14
|
+
formatter:
|
|
15
|
+
type: csv
|
|
16
|
+
header_line: true
|
|
17
|
+
charset: UTF-8
|
|
18
|
+
newline: CRLF
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
json
|
|
2
|
+
"{""object_id"":""55eae883689a08361045d652"",""name"":""obj9"",""rank"":9,""value"":9.9,""created_at"":""2015-09-06T10:05:18.786Z"",""embeded"":{""key"":""value9""}}"
|
|
3
|
+
"{""object_id"":""55eae883689a08361045d651"",""name"":""obj8"",""rank"":8,""value"":8.8,""created_at"":""2015-09-06T10:05:28.786Z"",""embeded"":{""key"":""value8""}}"
|
|
4
|
+
"{""object_id"":""55eae883689a08361045d650"",""name"":""obj7"",""rank"":7,""value"":7.7,""created_at"":""2015-09-06T10:05:38.786Z"",""embeded"":{""key"":""value7""}}"
|
|
5
|
+
"{""object_id"":""55eae883689a08361045d64f"",""name"":""obj6"",""rank"":6,""value"":6.6,""created_at"":""2015-09-06T10:05:48.786Z"",""embeded"":{""key"":""value6""}}"
|
|
6
|
+
"{""object_id"":""55eae883689a08361045d64e"",""name"":""obj5"",""rank"":5,""value"":5.5,""created_at"":""2015-09-06T10:05:58.786Z"",""embeded"":{""key"":""value5""}}"
|
|
7
|
+
"{""object_id"":""55eae883689a08361045d64d"",""name"":""obj4"",""rank"":4,""value"":4.4,""created_at"":""2015-09-06T10:06:08.786Z"",""embeded"":{""key"":{""inner_key"":""value4""}}}"
|
|
8
|
+
"{""object_id"":""55eae883689a08361045d64c"",""name"":""obj3"",""rank"":3,""value"":3.3,""created_at"":""2015-09-06T10:06:18.786Z"",""embeded"":{""key"":[""v3-1"",""v3-2""]}}"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-mongodb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kazuyuki Honda
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-06-
|
|
11
|
+
date: 2016-06-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -50,6 +50,8 @@ files:
|
|
|
50
50
|
- LICENSE.txt
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
+
- config/checkstyle/checkstyle.xml
|
|
54
|
+
- config/checkstyle/default.xml
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
56
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
57
|
- gradlew
|
|
@@ -62,8 +64,10 @@ files:
|
|
|
62
64
|
- src/test/resources/basic_expected.csv
|
|
63
65
|
- src/test/resources/full.yml
|
|
64
66
|
- src/test/resources/full_expected.csv
|
|
67
|
+
- src/test/resources/id_field_name.yml
|
|
68
|
+
- src/test/resources/id_field_name_expected.csv
|
|
65
69
|
- src/test/resources/my_collection.jsonl
|
|
66
|
-
- classpath/embulk-input-mongodb-0.3.
|
|
70
|
+
- classpath/embulk-input-mongodb-0.3.1.jar
|
|
67
71
|
- classpath/mongo-java-driver-3.2.2.jar
|
|
68
72
|
homepage: https://github.com/hakobera/embulk-input-mongodb
|
|
69
73
|
licenses:
|
|
Binary file
|