embulk-input-mongodb 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +25 -8
- data/README.md +1 -0
- data/build.gradle +19 -1
- data/classpath/embulk-input-mongodb-0.3.1.jar +0 -0
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -1
- data/src/main/java/org/embulk/input/mongodb/MongodbInputPlugin.java +17 -6
- data/src/main/java/org/embulk/input/mongodb/ValueCodec.java +25 -27
- data/src/test/java/org/embulk/input/mongodb/TestMongodbInputPlugin.java +82 -39
- data/src/test/resources/id_field_name.yml +18 -0
- data/src/test/resources/id_field_name_expected.csv +8 -0
- metadata +7 -3
- data/classpath/embulk-input-mongodb-0.3.0.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b2a60bb559e33bfa9fb7e8f9f0b6453e070d995e
|
|
4
|
+
data.tar.gz: b6da249c33d89b3a84d070f48e18a97243061bf5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1c3a1f2a3ae383587c548d5e7fcae302eeb17275e784498958e04436fc3e86fc49f2aa4197d2fae3436c009425c9414d4ee1bb358fd9bd2811241b4484c5f644
|
|
7
|
+
data.tar.gz: 89d72c6f271dfbfed98fbf5ffbbb72851cc4d48aaa4af202a1fb09fe07caaefd2f835b2a94ee7c3ac733992b5a3ec1233ace1ac45221fb50284263f20452387c
|
data/.travis.yml
CHANGED
|
@@ -5,8 +5,22 @@ jdk:
|
|
|
5
5
|
- oraclejdk7
|
|
6
6
|
- openjdk7
|
|
7
7
|
|
|
8
|
+
env:
|
|
9
|
+
global:
|
|
10
|
+
- MONGO_DATABASE=my_database
|
|
11
|
+
- MONGO_COLLECTION=my_collection
|
|
12
|
+
- MONGO_URI=mongodb://localhost:27017/my_database
|
|
13
|
+
|
|
8
14
|
sudo: required
|
|
9
15
|
|
|
16
|
+
before_cache:
|
|
17
|
+
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
|
|
18
|
+
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
|
|
19
|
+
cache:
|
|
20
|
+
directories:
|
|
21
|
+
- $HOME/.gradle/caches/
|
|
22
|
+
- $HOME/.gradle/wrapper/
|
|
23
|
+
|
|
10
24
|
# Work around fix for buffer overflow error on OpenJDK7
|
|
11
25
|
# ref: https://github.com/travis-ci/travis-ci/issues/5227#issuecomment-165131913
|
|
12
26
|
before_install:
|
|
@@ -29,14 +43,17 @@ install:
|
|
|
29
43
|
before_script:
|
|
30
44
|
- echo "Wait mongodb wakeup"
|
|
31
45
|
- sleep 10
|
|
32
|
-
- mongoimport --db my_database --collection my_collection --type json --drop src/test/resources/my_collection.jsonl
|
|
33
|
-
- ./gradlew package
|
|
34
46
|
- mkdir -p ./tmp
|
|
47
|
+
- date
|
|
35
48
|
|
|
36
49
|
script:
|
|
37
|
-
-
|
|
38
|
-
-
|
|
39
|
-
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
50
|
+
- ./gradlew check
|
|
51
|
+
- ./gradlew package
|
|
52
|
+
- mongoimport --db $MONGO_DATABASE --collection $MONGO_COLLECTION --type json --drop src/test/resources/my_collection.jsonl
|
|
53
|
+
- |
|
|
54
|
+
for target in basic full id_field_name
|
|
55
|
+
do
|
|
56
|
+
embulk run -L . src/test/resources/${target}.yml
|
|
57
|
+
cat tmp/${target}000.00.csv
|
|
58
|
+
cmp tmp/${target}000.00.csv src/test/resources/${target}_expected.csv
|
|
59
|
+
done
|
data/README.md
CHANGED
|
@@ -24,6 +24,7 @@ This plugin only works with embulk >= 0.8.8.
|
|
|
24
24
|
~~- double~~
|
|
25
25
|
~~- string~~
|
|
26
26
|
~~- timestamp~~
|
|
27
|
+
- **id_field_name** (string, optional, default: "_id") Name of Object ID field name. Set if you want to change the default name `_id`
|
|
27
28
|
- **query**: a JSON document used for [querying](https://docs.mongodb.com/manual/tutorial/query-documents/) on the source collection. Documents are loaded from the colleciton if they match with this condition. (string, optional)
|
|
28
29
|
- **projection**: A JSON document used for [projection](https://docs.mongodb.com/manual/reference/operator/projection/positional/) on query results. Fields in a document are used only if they match with this condition. (string, optional)
|
|
29
30
|
- **sort**: ordering of results (string, optional)
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
id "jacoco"
|
|
6
7
|
}
|
|
7
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -16,7 +17,7 @@ configurations {
|
|
|
16
17
|
provided
|
|
17
18
|
}
|
|
18
19
|
|
|
19
|
-
version = "0.3.
|
|
20
|
+
version = "0.3.1"
|
|
20
21
|
|
|
21
22
|
sourceCompatibility = 1.7
|
|
22
23
|
targetCompatibility = 1.7
|
|
@@ -38,6 +39,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
38
39
|
}
|
|
39
40
|
clean { delete "classpath" }
|
|
40
41
|
|
|
42
|
+
checkstyle {
|
|
43
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
44
|
+
toolVersion = '6.14.1'
|
|
45
|
+
}
|
|
46
|
+
checkstyleMain {
|
|
47
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
48
|
+
ignoreFailures = true
|
|
49
|
+
}
|
|
50
|
+
checkstyleTest {
|
|
51
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
52
|
+
ignoreFailures = true
|
|
53
|
+
}
|
|
54
|
+
task checkstyle(type: Checkstyle) {
|
|
55
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
56
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
57
|
+
}
|
|
58
|
+
|
|
41
59
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
42
60
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
43
61
|
script "${project.name}.gemspec"
|
|
Binary file
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="ClassTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
<module name="MethodTypeParameterName">
|
|
92
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
93
|
+
</module>
|
|
94
|
+
|
|
95
|
+
<module name="WhitespaceAround">
|
|
96
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
97
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
98
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
99
|
+
<property name="tokens" value="
|
|
100
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
101
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
102
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
103
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
104
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
105
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
106
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
107
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
108
|
+
</module>
|
|
109
|
+
</module>
|
|
110
|
+
</module>
|
|
@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
|
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -33,7 +33,9 @@ import org.embulk.spi.SchemaConfig;
|
|
|
33
33
|
import org.embulk.spi.type.Types;
|
|
34
34
|
import org.msgpack.value.Value;
|
|
35
35
|
import org.slf4j.Logger;
|
|
36
|
+
|
|
36
37
|
import javax.validation.constraints.Min;
|
|
38
|
+
|
|
37
39
|
import java.net.UnknownHostException;
|
|
38
40
|
import java.util.List;
|
|
39
41
|
|
|
@@ -66,6 +68,10 @@ public class MongodbInputPlugin
|
|
|
66
68
|
@ConfigDefault("\"{}\"")
|
|
67
69
|
String getSort();
|
|
68
70
|
|
|
71
|
+
@Config("id_field_name")
|
|
72
|
+
@ConfigDefault("\"_id\"")
|
|
73
|
+
String getIdFieldName();
|
|
74
|
+
|
|
69
75
|
@Config("batch_size")
|
|
70
76
|
@ConfigDefault("10000")
|
|
71
77
|
@Min(1)
|
|
@@ -101,7 +107,8 @@ public class MongodbInputPlugin
|
|
|
101
107
|
// Connect once to throw ConfigException in earlier stage of excecution
|
|
102
108
|
try {
|
|
103
109
|
connect(task);
|
|
104
|
-
}
|
|
110
|
+
}
|
|
111
|
+
catch (UnknownHostException | MongoException ex) {
|
|
105
112
|
throw new ConfigException(ex);
|
|
106
113
|
}
|
|
107
114
|
Schema schema = Schema.builder().add(task.getJsonColumnName(), Types.JSON).build();
|
|
@@ -141,11 +148,12 @@ public class MongodbInputPlugin
|
|
|
141
148
|
|
|
142
149
|
CodecRegistry registry = CodecRegistries.fromRegistries(
|
|
143
150
|
MongoClient.getDefaultCodecRegistry(),
|
|
144
|
-
CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord()))
|
|
151
|
+
CodecRegistries.fromCodecs(new ValueCodec(task.getStopOnInvalidRecord(), task))
|
|
145
152
|
);
|
|
146
153
|
collection = db.getCollection(task.getCollection(), Value.class)
|
|
147
154
|
.withCodecRegistry(registry);
|
|
148
|
-
}
|
|
155
|
+
}
|
|
156
|
+
catch (UnknownHostException | MongoException ex) {
|
|
149
157
|
throw new ConfigException(ex);
|
|
150
158
|
}
|
|
151
159
|
|
|
@@ -182,7 +190,8 @@ public class MongodbInputPlugin
|
|
|
182
190
|
return Exec.newConfigDiff();
|
|
183
191
|
}
|
|
184
192
|
|
|
185
|
-
private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException
|
|
193
|
+
private MongoDatabase connect(final PluginTask task) throws UnknownHostException, MongoException
|
|
194
|
+
{
|
|
186
195
|
MongoClientURI uri = new MongoClientURI(task.getUri());
|
|
187
196
|
MongoClient mongoClient = new MongoClient(uri);
|
|
188
197
|
|
|
@@ -192,10 +201,12 @@ public class MongodbInputPlugin
|
|
|
192
201
|
return db;
|
|
193
202
|
}
|
|
194
203
|
|
|
195
|
-
private void validateJsonField(String name, String jsonString)
|
|
204
|
+
private void validateJsonField(String name, String jsonString)
|
|
205
|
+
{
|
|
196
206
|
try {
|
|
197
207
|
JSON.parse(jsonString);
|
|
198
|
-
}
|
|
208
|
+
}
|
|
209
|
+
catch (JSONParseException ex) {
|
|
199
210
|
throw new ConfigException(String.format("Invalid JSON string was given for '%s' parameter. [%s]", name, jsonString));
|
|
200
211
|
}
|
|
201
212
|
}
|
|
@@ -11,15 +11,6 @@ import org.embulk.spi.Exec;
|
|
|
11
11
|
import org.msgpack.value.Value;
|
|
12
12
|
import org.slf4j.Logger;
|
|
13
13
|
|
|
14
|
-
import static org.msgpack.value.ValueFactory.newArray;
|
|
15
|
-
import static org.msgpack.value.ValueFactory.newBinary;
|
|
16
|
-
import static org.msgpack.value.ValueFactory.newBoolean;
|
|
17
|
-
import static org.msgpack.value.ValueFactory.newFloat;
|
|
18
|
-
import static org.msgpack.value.ValueFactory.newInteger;
|
|
19
|
-
import static org.msgpack.value.ValueFactory.newMap;
|
|
20
|
-
import static org.msgpack.value.ValueFactory.newNil;
|
|
21
|
-
import static org.msgpack.value.ValueFactory.newString;
|
|
22
|
-
|
|
23
14
|
import java.text.SimpleDateFormat;
|
|
24
15
|
import java.util.ArrayList;
|
|
25
16
|
import java.util.Date;
|
|
@@ -28,39 +19,44 @@ import java.util.List;
|
|
|
28
19
|
import java.util.Map;
|
|
29
20
|
import java.util.TimeZone;
|
|
30
21
|
|
|
31
|
-
|
|
22
|
+
import static org.msgpack.value.ValueFactory.*;
|
|
23
|
+
|
|
24
|
+
public class ValueCodec implements Codec<Value>
|
|
25
|
+
{
|
|
32
26
|
private final SimpleDateFormat formatter;
|
|
33
27
|
private final Logger log = Exec.getLogger(MongodbInputPlugin.class);
|
|
34
28
|
private final boolean stopOnInvalidRecord;
|
|
29
|
+
private final MongodbInputPlugin.PluginTask task;
|
|
35
30
|
|
|
36
|
-
public ValueCodec(boolean stopOnInvalidRecord)
|
|
31
|
+
public ValueCodec(boolean stopOnInvalidRecord, MongodbInputPlugin.PluginTask task)
|
|
32
|
+
{
|
|
37
33
|
this.formatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
|
|
38
34
|
formatter.setTimeZone(TimeZone.getTimeZone("UTC"));
|
|
39
35
|
this.stopOnInvalidRecord = stopOnInvalidRecord;
|
|
36
|
+
this.task = task;
|
|
40
37
|
}
|
|
41
38
|
|
|
42
39
|
@Override
|
|
43
|
-
public void encode(final BsonWriter writer, final Value value, final EncoderContext encoderContext)
|
|
40
|
+
public void encode(final BsonWriter writer, final Value value, final EncoderContext encoderContext)
|
|
41
|
+
{
|
|
44
42
|
throw new UnsupportedOperationException();
|
|
45
43
|
}
|
|
46
44
|
|
|
47
45
|
@Override
|
|
48
|
-
public Value decode(final BsonReader reader, final DecoderContext decoderContext)
|
|
46
|
+
public Value decode(final BsonReader reader, final DecoderContext decoderContext)
|
|
47
|
+
{
|
|
49
48
|
Map<Value, Value> kvs = new LinkedHashMap<>();
|
|
50
49
|
|
|
51
50
|
reader.readStartDocument();
|
|
52
|
-
boolean isTopLevelNode = false;
|
|
53
51
|
while (reader.readBsonType() != BsonType.END_OF_DOCUMENT) {
|
|
54
52
|
String fieldName = reader.readName();
|
|
55
53
|
BsonType type = reader.getCurrentBsonType();
|
|
56
|
-
|
|
57
|
-
isTopLevelNode = true;
|
|
58
|
-
}
|
|
59
|
-
fieldName = normalize(fieldName, isTopLevelNode);
|
|
54
|
+
fieldName = normalize(fieldName);
|
|
60
55
|
|
|
61
56
|
try {
|
|
62
57
|
kvs.put(newString(fieldName), readValue(reader, decoderContext));
|
|
63
|
-
}
|
|
58
|
+
}
|
|
59
|
+
catch (UnknownTypeFoundException ex) {
|
|
64
60
|
reader.skipValue();
|
|
65
61
|
if (stopOnInvalidRecord) {
|
|
66
62
|
throw ex;
|
|
@@ -74,7 +70,8 @@ public class ValueCodec implements Codec<Value> {
|
|
|
74
70
|
return newMap(kvs);
|
|
75
71
|
}
|
|
76
72
|
|
|
77
|
-
public Value decodeArray(final BsonReader reader, final DecoderContext decoderContext)
|
|
73
|
+
public Value decodeArray(final BsonReader reader, final DecoderContext decoderContext)
|
|
74
|
+
{
|
|
78
75
|
List<Value> list = new ArrayList<>();
|
|
79
76
|
|
|
80
77
|
reader.readStartArray();
|
|
@@ -86,7 +83,8 @@ public class ValueCodec implements Codec<Value> {
|
|
|
86
83
|
return newArray(list);
|
|
87
84
|
}
|
|
88
85
|
|
|
89
|
-
private Value readValue(BsonReader reader, DecoderContext decoderContext)
|
|
86
|
+
private Value readValue(BsonReader reader, DecoderContext decoderContext)
|
|
87
|
+
{
|
|
90
88
|
switch (reader.getCurrentBsonType()) {
|
|
91
89
|
// https://docs.mongodb.com/manual/reference/bson-types/
|
|
92
90
|
// https://github.com/mongodb/mongo-java-driver/tree/master/bson/src/main/org/bson/codecs
|
|
@@ -128,15 +126,15 @@ public class ValueCodec implements Codec<Value> {
|
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
@Override
|
|
131
|
-
public Class<Value> getEncoderClass()
|
|
129
|
+
public Class<Value> getEncoderClass()
|
|
130
|
+
{
|
|
132
131
|
return Value.class;
|
|
133
132
|
}
|
|
134
133
|
|
|
135
|
-
private String normalize(String key
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
return "_id";
|
|
134
|
+
private String normalize(String key)
|
|
135
|
+
{
|
|
136
|
+
if (key.equals("_id")) {
|
|
137
|
+
return task.getIdFieldName();
|
|
140
138
|
}
|
|
141
139
|
return key;
|
|
142
140
|
}
|
|
@@ -34,17 +34,20 @@ import org.junit.rules.ExpectedException;
|
|
|
34
34
|
|
|
35
35
|
import java.lang.reflect.InvocationTargetException;
|
|
36
36
|
import java.lang.reflect.Method;
|
|
37
|
+
import java.text.DateFormat;
|
|
37
38
|
import java.text.SimpleDateFormat;
|
|
38
39
|
import java.util.ArrayList;
|
|
39
40
|
import java.util.Arrays;
|
|
40
41
|
import java.util.List;
|
|
41
42
|
import java.util.Locale;
|
|
43
|
+
import java.util.TimeZone;
|
|
42
44
|
|
|
43
45
|
import static org.hamcrest.CoreMatchers.is;
|
|
44
46
|
import static org.junit.Assert.assertEquals;
|
|
45
47
|
import static org.junit.Assert.assertThat;
|
|
46
48
|
|
|
47
|
-
public class TestMongodbInputPlugin
|
|
49
|
+
public class TestMongodbInputPlugin
|
|
50
|
+
{
|
|
48
51
|
private static String MONGO_URI;
|
|
49
52
|
private static String MONGO_COLLECTION;
|
|
50
53
|
|
|
@@ -64,20 +67,23 @@ public class TestMongodbInputPlugin {
|
|
|
64
67
|
* MONGO_COLLECTION
|
|
65
68
|
*/
|
|
66
69
|
@BeforeClass
|
|
67
|
-
public static void initializeConstant()
|
|
70
|
+
public static void initializeConstant()
|
|
71
|
+
{
|
|
68
72
|
MONGO_URI = System.getenv("MONGO_URI");
|
|
69
73
|
MONGO_COLLECTION = System.getenv("MONGO_COLLECTION");
|
|
70
74
|
}
|
|
71
75
|
|
|
72
76
|
@Before
|
|
73
|
-
public void createResources() throws Exception
|
|
77
|
+
public void createResources() throws Exception
|
|
78
|
+
{
|
|
74
79
|
config = config();
|
|
75
80
|
plugin = new MongodbInputPlugin();
|
|
76
81
|
output = new MockPageOutput();
|
|
77
82
|
}
|
|
78
83
|
|
|
79
84
|
@Test
|
|
80
|
-
public void checkDefaultValues()
|
|
85
|
+
public void checkDefaultValues()
|
|
86
|
+
{
|
|
81
87
|
ConfigSource config = Exec.newConfigSource()
|
|
82
88
|
.set("uri", MONGO_URI)
|
|
83
89
|
.set("collection", MONGO_COLLECTION);
|
|
@@ -90,7 +96,8 @@ public class TestMongodbInputPlugin {
|
|
|
90
96
|
}
|
|
91
97
|
|
|
92
98
|
@Test(expected = ConfigException.class)
|
|
93
|
-
public void checkDefaultValuesUriIsNull()
|
|
99
|
+
public void checkDefaultValuesUriIsNull()
|
|
100
|
+
{
|
|
94
101
|
ConfigSource config = Exec.newConfigSource()
|
|
95
102
|
.set("uri", null)
|
|
96
103
|
.set("collection", MONGO_COLLECTION);
|
|
@@ -109,7 +116,8 @@ public class TestMongodbInputPlugin {
|
|
|
109
116
|
}
|
|
110
117
|
|
|
111
118
|
@Test(expected = ConfigException.class)
|
|
112
|
-
public void checkDefaultValuesCollectionIsNull()
|
|
119
|
+
public void checkDefaultValuesCollectionIsNull()
|
|
120
|
+
{
|
|
113
121
|
ConfigSource config = Exec.newConfigSource()
|
|
114
122
|
.set("uri", MONGO_URI)
|
|
115
123
|
.set("collection", null);
|
|
@@ -118,12 +126,14 @@ public class TestMongodbInputPlugin {
|
|
|
118
126
|
}
|
|
119
127
|
|
|
120
128
|
@Test
|
|
121
|
-
public void testResume()
|
|
129
|
+
public void testResume()
|
|
130
|
+
{
|
|
122
131
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
123
132
|
final Schema schema = getFieldSchema();
|
|
124
133
|
plugin.resume(task.dump(), schema, 0, new InputPlugin.Control() {
|
|
125
134
|
@Override
|
|
126
|
-
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
135
|
+
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
136
|
+
{
|
|
127
137
|
return emptyTaskReports(taskCount);
|
|
128
138
|
}
|
|
129
139
|
});
|
|
@@ -131,19 +141,22 @@ public class TestMongodbInputPlugin {
|
|
|
131
141
|
}
|
|
132
142
|
|
|
133
143
|
@Test
|
|
134
|
-
public void testCleanup()
|
|
144
|
+
public void testCleanup()
|
|
145
|
+
{
|
|
135
146
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
136
147
|
Schema schema = getFieldSchema();
|
|
137
148
|
plugin.cleanup(task.dump(), schema, 0, Lists.<TaskReport>newArrayList()); // no errors happens
|
|
138
149
|
}
|
|
139
150
|
|
|
140
151
|
@Test
|
|
141
|
-
public void testGuess()
|
|
152
|
+
public void testGuess()
|
|
153
|
+
{
|
|
142
154
|
plugin.guess(config); // no errors happens
|
|
143
155
|
}
|
|
144
156
|
|
|
145
157
|
@Test
|
|
146
|
-
public void testRun() throws Exception
|
|
158
|
+
public void testRun() throws Exception
|
|
159
|
+
{
|
|
147
160
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
148
161
|
|
|
149
162
|
dropCollection(task, MONGO_COLLECTION);
|
|
@@ -155,7 +168,8 @@ public class TestMongodbInputPlugin {
|
|
|
155
168
|
}
|
|
156
169
|
|
|
157
170
|
@Test(expected = ValueCodec.UnknownTypeFoundException.class)
|
|
158
|
-
public void testRunWithUnsupportedType() throws Exception
|
|
171
|
+
public void testRunWithUnsupportedType() throws Exception
|
|
172
|
+
{
|
|
159
173
|
ConfigSource config = Exec.newConfigSource()
|
|
160
174
|
.set("uri", MONGO_URI)
|
|
161
175
|
.set("collection", MONGO_COLLECTION)
|
|
@@ -176,33 +190,47 @@ public class TestMongodbInputPlugin {
|
|
|
176
190
|
}
|
|
177
191
|
|
|
178
192
|
@Test
|
|
179
|
-
public void testNormalize() throws Exception
|
|
180
|
-
|
|
193
|
+
public void testNormalize() throws Exception
|
|
194
|
+
{
|
|
195
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
196
|
+
ValueCodec codec = new ValueCodec(true, task);
|
|
181
197
|
|
|
182
|
-
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class
|
|
198
|
+
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class);
|
|
183
199
|
normalize.setAccessible(true);
|
|
184
|
-
assertEquals("_id", normalize.invoke(codec, "
|
|
185
|
-
assertEquals("
|
|
186
|
-
|
|
200
|
+
assertEquals("_id", normalize.invoke(codec, "_id").toString());
|
|
201
|
+
assertEquals("f1", normalize.invoke(codec, "f1").toString());
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@Test
|
|
205
|
+
public void testNormlizeWithIdFieldName() throws Exception
|
|
206
|
+
{
|
|
207
|
+
ConfigSource config = config().set("id_field_name", "object_id");
|
|
208
|
+
|
|
209
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
210
|
+
ValueCodec codec = new ValueCodec(true, task);
|
|
187
211
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
assertEquals("
|
|
212
|
+
Method normalize = ValueCodec.class.getDeclaredMethod("normalize", String.class);
|
|
213
|
+
normalize.setAccessible(true);
|
|
214
|
+
assertEquals("object_id", normalize.invoke(codec, "_id").toString());
|
|
215
|
+
assertEquals("f1", normalize.invoke(codec, "f1").toString());
|
|
191
216
|
}
|
|
192
217
|
|
|
193
218
|
@Test
|
|
194
|
-
public void testValidateJsonField() throws Exception
|
|
219
|
+
public void testValidateJsonField() throws Exception
|
|
220
|
+
{
|
|
195
221
|
Method validate = MongodbInputPlugin.class.getDeclaredMethod("validateJsonField", String.class, String.class);
|
|
196
222
|
validate.setAccessible(true);
|
|
197
223
|
String invalidJsonString = "{\"name\": invalid}";
|
|
198
224
|
try {
|
|
199
225
|
validate.invoke(plugin, "name", invalidJsonString);
|
|
200
|
-
}
|
|
226
|
+
}
|
|
227
|
+
catch (InvocationTargetException ex) {
|
|
201
228
|
assertEquals(ConfigException.class, ex.getCause().getClass());
|
|
202
229
|
}
|
|
203
230
|
}
|
|
204
231
|
|
|
205
|
-
static List<TaskReport> emptyTaskReports(int taskCount)
|
|
232
|
+
static List<TaskReport> emptyTaskReports(int taskCount)
|
|
233
|
+
{
|
|
206
234
|
ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
|
207
235
|
for (int i = 0; i < taskCount; i++) {
|
|
208
236
|
reports.add(Exec.newTaskReport());
|
|
@@ -211,9 +239,11 @@ public class TestMongodbInputPlugin {
|
|
|
211
239
|
}
|
|
212
240
|
|
|
213
241
|
private class Control
|
|
214
|
-
implements InputPlugin.Control
|
|
242
|
+
implements InputPlugin.Control
|
|
243
|
+
{
|
|
215
244
|
@Override
|
|
216
|
-
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
245
|
+
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
246
|
+
{
|
|
217
247
|
List<TaskReport> reports = new ArrayList<>();
|
|
218
248
|
for (int i = 0; i < taskCount; i++) {
|
|
219
249
|
reports.add(plugin.run(taskSource, schema, i, output));
|
|
@@ -222,24 +252,25 @@ public class TestMongodbInputPlugin {
|
|
|
222
252
|
}
|
|
223
253
|
}
|
|
224
254
|
|
|
225
|
-
private ConfigSource config()
|
|
255
|
+
private ConfigSource config()
|
|
256
|
+
{
|
|
226
257
|
return Exec.newConfigSource()
|
|
227
258
|
.set("uri", MONGO_URI)
|
|
228
|
-
.set("collection", MONGO_COLLECTION)
|
|
229
|
-
.set("last_path", "");
|
|
259
|
+
.set("collection", MONGO_COLLECTION);
|
|
230
260
|
}
|
|
231
261
|
|
|
232
|
-
private List<Document> createValidDocuments() throws Exception
|
|
233
|
-
|
|
262
|
+
private List<Document> createValidDocuments() throws Exception
|
|
263
|
+
{
|
|
264
|
+
DateFormat format = getUTCDateFormat();
|
|
234
265
|
|
|
235
266
|
List<Document> documents = new ArrayList<>();
|
|
236
267
|
documents.add(
|
|
237
268
|
new Document("double_field", 1.23)
|
|
238
269
|
.append("string_field", "embulk")
|
|
239
|
-
.append("array_field", Arrays.asList(1,2,3))
|
|
270
|
+
.append("array_field", Arrays.asList(1, 2, 3))
|
|
240
271
|
.append("binary_field", new BsonBinary(("test").getBytes("UTF-8")))
|
|
241
272
|
.append("boolean_field", true)
|
|
242
|
-
.append("datetime_field", format.parse("2015-01-
|
|
273
|
+
.append("datetime_field", format.parse("2015-01-27T10:23:49.000Z"))
|
|
243
274
|
.append("null_field", null)
|
|
244
275
|
.append("regex_field", new BsonRegularExpression(".+?"))
|
|
245
276
|
.append("javascript_field", new BsonJavaScript("var s = \"javascript\";"))
|
|
@@ -258,22 +289,25 @@ public class TestMongodbInputPlugin {
|
|
|
258
289
|
|
|
259
290
|
documents.add(new Document("document_field", new Document("k", "v")));
|
|
260
291
|
|
|
261
|
-
documents.add(new Document("document_field", new Document("k", format.parse("2015-02-
|
|
292
|
+
documents.add(new Document("document_field", new Document("k", format.parse("2015-02-02T23:13:45.000Z"))));
|
|
262
293
|
|
|
263
294
|
return documents;
|
|
264
295
|
}
|
|
265
296
|
|
|
266
|
-
private Schema getFieldSchema()
|
|
297
|
+
private Schema getFieldSchema()
|
|
298
|
+
{
|
|
267
299
|
ImmutableList.Builder<Column> columns = ImmutableList.builder();
|
|
268
300
|
columns.add(new Column(0, "record", Types.JSON));
|
|
269
301
|
return new Schema(columns.build());
|
|
270
302
|
}
|
|
271
303
|
|
|
272
|
-
private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception
|
|
304
|
+
private void assertValidRecords(Schema schema, MockPageOutput output) throws Exception
|
|
305
|
+
{
|
|
273
306
|
List<Object[]> records = Pages.toObjects(schema, output.pages);
|
|
274
307
|
assertEquals(5, records.size());
|
|
275
308
|
|
|
276
309
|
ObjectMapper mapper = new ObjectMapper();
|
|
310
|
+
mapper.setDateFormat(getUTCDateFormat());
|
|
277
311
|
|
|
278
312
|
{
|
|
279
313
|
JsonNode node = mapper.readTree(records.get(0)[0].toString());
|
|
@@ -314,14 +348,16 @@ public class TestMongodbInputPlugin {
|
|
|
314
348
|
}
|
|
315
349
|
}
|
|
316
350
|
|
|
317
|
-
private void createCollection(PluginTask task, String collectionName) throws Exception
|
|
351
|
+
private void createCollection(PluginTask task, String collectionName) throws Exception
|
|
352
|
+
{
|
|
318
353
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
319
354
|
method.setAccessible(true);
|
|
320
355
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
321
356
|
db.createCollection(collectionName);
|
|
322
357
|
}
|
|
323
358
|
|
|
324
|
-
private void dropCollection(PluginTask task, String collectionName) throws Exception
|
|
359
|
+
private void dropCollection(PluginTask task, String collectionName) throws Exception
|
|
360
|
+
{
|
|
325
361
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
326
362
|
method.setAccessible(true);
|
|
327
363
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
@@ -329,11 +365,18 @@ public class TestMongodbInputPlugin {
|
|
|
329
365
|
collection.drop();
|
|
330
366
|
}
|
|
331
367
|
|
|
332
|
-
private void insertDocument(PluginTask task, List<Document> documents) throws Exception
|
|
368
|
+
private void insertDocument(PluginTask task, List<Document> documents) throws Exception
|
|
369
|
+
{
|
|
333
370
|
Method method = MongodbInputPlugin.class.getDeclaredMethod("connect", PluginTask.class);
|
|
334
371
|
method.setAccessible(true);
|
|
335
372
|
MongoDatabase db = (MongoDatabase) method.invoke(plugin, task);
|
|
336
373
|
MongoCollection collection = db.getCollection(task.getCollection());
|
|
337
374
|
collection.insertMany(documents);
|
|
338
375
|
}
|
|
376
|
+
|
|
377
|
+
private DateFormat getUTCDateFormat() {
|
|
378
|
+
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", java.util.Locale.ENGLISH);
|
|
379
|
+
dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
|
|
380
|
+
return dateFormat;
|
|
381
|
+
}
|
|
339
382
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
in:
|
|
2
|
+
type: mongodb
|
|
3
|
+
uri: mongodb://localhost:27017/my_database
|
|
4
|
+
collection: "my_collection"
|
|
5
|
+
json_column_name: "json"
|
|
6
|
+
query: '{ rank: { $gte: 3 } }'
|
|
7
|
+
sort: '{ rank: -1 }'
|
|
8
|
+
id_field_name: "object_id"
|
|
9
|
+
batch_size: 100
|
|
10
|
+
out:
|
|
11
|
+
type: file
|
|
12
|
+
path_prefix: ./tmp/id_field_name
|
|
13
|
+
file_ext: csv
|
|
14
|
+
formatter:
|
|
15
|
+
type: csv
|
|
16
|
+
header_line: true
|
|
17
|
+
charset: UTF-8
|
|
18
|
+
newline: CRLF
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
json
|
|
2
|
+
"{""object_id"":""55eae883689a08361045d652"",""name"":""obj9"",""rank"":9,""value"":9.9,""created_at"":""2015-09-06T10:05:18.786Z"",""embeded"":{""key"":""value9""}}"
|
|
3
|
+
"{""object_id"":""55eae883689a08361045d651"",""name"":""obj8"",""rank"":8,""value"":8.8,""created_at"":""2015-09-06T10:05:28.786Z"",""embeded"":{""key"":""value8""}}"
|
|
4
|
+
"{""object_id"":""55eae883689a08361045d650"",""name"":""obj7"",""rank"":7,""value"":7.7,""created_at"":""2015-09-06T10:05:38.786Z"",""embeded"":{""key"":""value7""}}"
|
|
5
|
+
"{""object_id"":""55eae883689a08361045d64f"",""name"":""obj6"",""rank"":6,""value"":6.6,""created_at"":""2015-09-06T10:05:48.786Z"",""embeded"":{""key"":""value6""}}"
|
|
6
|
+
"{""object_id"":""55eae883689a08361045d64e"",""name"":""obj5"",""rank"":5,""value"":5.5,""created_at"":""2015-09-06T10:05:58.786Z"",""embeded"":{""key"":""value5""}}"
|
|
7
|
+
"{""object_id"":""55eae883689a08361045d64d"",""name"":""obj4"",""rank"":4,""value"":4.4,""created_at"":""2015-09-06T10:06:08.786Z"",""embeded"":{""key"":{""inner_key"":""value4""}}}"
|
|
8
|
+
"{""object_id"":""55eae883689a08361045d64c"",""name"":""obj3"",""rank"":3,""value"":3.3,""created_at"":""2015-09-06T10:06:18.786Z"",""embeded"":{""key"":[""v3-1"",""v3-2""]}}"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-mongodb
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Kazuyuki Honda
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-06-
|
|
11
|
+
date: 2016-06-21 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -50,6 +50,8 @@ files:
|
|
|
50
50
|
- LICENSE.txt
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
+
- config/checkstyle/checkstyle.xml
|
|
54
|
+
- config/checkstyle/default.xml
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
56
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
57
|
- gradlew
|
|
@@ -62,8 +64,10 @@ files:
|
|
|
62
64
|
- src/test/resources/basic_expected.csv
|
|
63
65
|
- src/test/resources/full.yml
|
|
64
66
|
- src/test/resources/full_expected.csv
|
|
67
|
+
- src/test/resources/id_field_name.yml
|
|
68
|
+
- src/test/resources/id_field_name_expected.csv
|
|
65
69
|
- src/test/resources/my_collection.jsonl
|
|
66
|
-
- classpath/embulk-input-mongodb-0.3.
|
|
70
|
+
- classpath/embulk-input-mongodb-0.3.1.jar
|
|
67
71
|
- classpath/mongo-java-driver-3.2.2.jar
|
|
68
72
|
homepage: https://github.com/hakobera/embulk-input-mongodb
|
|
69
73
|
licenses:
|
|
Binary file
|