embulk-output-hdfs 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/CHANGELOG.md +4 -0
- data/README.md +14 -0
- data/build.gradle +31 -3
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/example/config.yml +52 -0
- data/example/data.csv +5 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +54 -9
- data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java +275 -0
- metadata +8 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f9fa40ed9c13dbc656239643f3153f160f66ef08
|
|
4
|
+
data.tar.gz: 61fb3a7a55c94873e58f7edc4ac5e6ae1cf337ae
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e131f8221baaa36c20fcd8ee77b88cac8dde80bc20e1663f1d5dae3d54ed88aae08862a5113f928916a2f5ddd19321bbf7270d8cc43109336ffede61e2adc99f
|
|
7
|
+
data.tar.gz: 0afec8392aeb2d109ebe9beb738d6b5e09f029b5cc04ab16b5b9bc83246a13f8c0a2fd86393da2d924cb46791d44a5f9d78d84ad5f06c60d061f67e61b260c64
|
data/.travis.yml
ADDED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Hdfs file output plugin for Embulk
|
|
2
2
|
|
|
3
|
+
[](https://travis-ci.org/civitaspo/embulk-output-hdfs)
|
|
4
|
+
[](https://coveralls.io/github/civitaspo/embulk-output-hdfs?branch=master)
|
|
5
|
+
|
|
3
6
|
A File Output Plugin for Embulk to write HDFS.
|
|
4
7
|
|
|
5
8
|
## Overview
|
|
@@ -19,6 +22,17 @@ A File Output Plugin for Embulk to write HDFS.
|
|
|
19
22
|
- **rewind_seconds** When you use Date format in path_prefix property(like `/tmp/embulk/%Y-%m-%d/out`), the format is interpreted by using the time which is Now minus this property. (int, default: `0`)
|
|
20
23
|
- **overwrite** overwrite files when the same filenames already exists (boolean, default: `false`)
|
|
21
24
|
- *caution*: even if this property is `true`, this does not mean ensuring the idempotence. if you want to ensure the idempotence, you need the procedures to remove output files after or before running.
|
|
25
|
+
- **doas** username which access to Hdfs (string, default: executed user)
|
|
26
|
+
- **delete_in_advance** delete files and directories having `path_prefix` in advance (enum, default: `NONE`)
|
|
27
|
+
- `NONE`: do nothing
|
|
28
|
+
- `FILE_ONLY`: delete files
|
|
29
|
+
- `RECURSIVE`: delete files and directories
|
|
30
|
+
|
|
31
|
+
## CAUTION
|
|
32
|
+
If you use `hadoop` user (hdfs admin user) as `doas`, and if `delete_in_advance` is `RECURSIVE`,
|
|
33
|
+
`embulk-output-hdfs` can delete any files and directories you indicate as `path_prefix`,
|
|
34
|
+
this means `embulk-output-hdfs` can destroy your hdfs.
|
|
35
|
+
So, please be careful when you use `delete_in_advance` option and `doas` option ...
|
|
22
36
|
|
|
23
37
|
## Example
|
|
24
38
|
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,9 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
6
|
+
id "com.github.kt3k.coveralls" version "2.4.0"
|
|
7
|
+
id "jacoco"
|
|
5
8
|
}
|
|
6
9
|
import com.github.jrubygradle.JRubyExec
|
|
7
10
|
repositories {
|
|
@@ -12,18 +15,20 @@ configurations {
|
|
|
12
15
|
provided
|
|
13
16
|
}
|
|
14
17
|
|
|
15
|
-
version = "0.2.
|
|
18
|
+
version = "0.2.3"
|
|
16
19
|
|
|
17
20
|
sourceCompatibility = 1.7
|
|
18
21
|
targetCompatibility = 1.7
|
|
19
22
|
|
|
20
23
|
dependencies {
|
|
21
|
-
compile "org.embulk:embulk-core:0.
|
|
22
|
-
provided "org.embulk:embulk-core:0.
|
|
24
|
+
compile "org.embulk:embulk-core:0.8.8"
|
|
25
|
+
provided "org.embulk:embulk-core:0.8.8"
|
|
23
26
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
|
24
27
|
compile 'org.apache.hadoop:hadoop-client:2.6.0'
|
|
25
28
|
compile 'com.google.guava:guava:15.0'
|
|
26
29
|
testCompile "junit:junit:4.+"
|
|
30
|
+
testCompile "org.embulk:embulk-core:0.8.8:tests"
|
|
31
|
+
testCompile "org.embulk:embulk-standards:0.8.8"
|
|
27
32
|
}
|
|
28
33
|
|
|
29
34
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
@@ -33,6 +38,29 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
33
38
|
}
|
|
34
39
|
clean { delete "classpath" }
|
|
35
40
|
|
|
41
|
+
jacocoTestReport {
|
|
42
|
+
reports {
|
|
43
|
+
xml.enabled = true // coveralls plugin depends on xml format report
|
|
44
|
+
html.enabled = true
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
checkstyle {
|
|
48
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
49
|
+
toolVersion = '6.14.1'
|
|
50
|
+
}
|
|
51
|
+
checkstyleMain {
|
|
52
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
53
|
+
ignoreFailures = true
|
|
54
|
+
}
|
|
55
|
+
checkstyleTest {
|
|
56
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
57
|
+
ignoreFailures = true
|
|
58
|
+
}
|
|
59
|
+
task checkstyle(type: Checkstyle) {
|
|
60
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
61
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
62
|
+
}
|
|
63
|
+
|
|
36
64
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
37
65
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
38
66
|
script "${project.name}.gemspec"
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
data/example/config.yml
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
hdfs_example: &hdfs_example
|
|
2
|
+
config_files:
|
|
3
|
+
- /etc/hadoop/conf/core-site.xml
|
|
4
|
+
- /etc/hadoop/conf/hdfs-site.xml
|
|
5
|
+
config:
|
|
6
|
+
fs.defaultFS: 'hdfs://hadoop-nn1:8020'
|
|
7
|
+
fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
|
|
8
|
+
fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
|
|
9
|
+
|
|
10
|
+
local_fs_example: &local_fs_example
|
|
11
|
+
config:
|
|
12
|
+
fs.defaultFS: 'file:///'
|
|
13
|
+
fs.hdfs.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
|
|
14
|
+
fs.file.impl: 'org.apache.hadoop.fs.RawLocalFileSystem'
|
|
15
|
+
io.compression.codecs: 'org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec'
|
|
16
|
+
|
|
17
|
+
in:
|
|
18
|
+
type: file
|
|
19
|
+
path_prefix: example/data
|
|
20
|
+
parser:
|
|
21
|
+
charset: UTF-8
|
|
22
|
+
newline: CRLF
|
|
23
|
+
type: csv
|
|
24
|
+
delimiter: ','
|
|
25
|
+
quote: '"'
|
|
26
|
+
header_line: true
|
|
27
|
+
stop_on_invalid_record: true
|
|
28
|
+
columns:
|
|
29
|
+
- {name: id, type: long}
|
|
30
|
+
- {name: account, type: long}
|
|
31
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
|
32
|
+
- {name: purchase, type: timestamp, format: '%Y%m%d'}
|
|
33
|
+
- {name: comment, type: string}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
out:
|
|
37
|
+
type: hdfs
|
|
38
|
+
<<: *local_fs_example
|
|
39
|
+
path_prefix: /tmp/embulk-output-hdfs_example/file_
|
|
40
|
+
file_ext: csv
|
|
41
|
+
delete_in_advance: FILE_ONLY
|
|
42
|
+
formatter:
|
|
43
|
+
type: csv
|
|
44
|
+
newline: CRLF
|
|
45
|
+
newline_in_field: LF
|
|
46
|
+
header_line: true
|
|
47
|
+
charset: UTF-8
|
|
48
|
+
quote_policy: NONE
|
|
49
|
+
quote: '"'
|
|
50
|
+
escape: '\'
|
|
51
|
+
null_string: ''
|
|
52
|
+
default_timezone: UTC
|
data/example/data.csv
ADDED
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -3,11 +3,13 @@ package org.embulk.output.hdfs;
|
|
|
3
3
|
import com.google.common.base.Optional;
|
|
4
4
|
import com.google.common.base.Throwables;
|
|
5
5
|
import org.apache.hadoop.conf.Configuration;
|
|
6
|
+
import org.apache.hadoop.fs.FileStatus;
|
|
6
7
|
import org.apache.hadoop.fs.FileSystem;
|
|
7
8
|
import org.apache.hadoop.fs.Path;
|
|
8
9
|
import org.embulk.config.Config;
|
|
9
10
|
import org.embulk.config.ConfigDefault;
|
|
10
11
|
import org.embulk.config.ConfigDiff;
|
|
12
|
+
import org.embulk.config.ConfigException;
|
|
11
13
|
import org.embulk.config.ConfigSource;
|
|
12
14
|
import org.embulk.config.Task;
|
|
13
15
|
import org.embulk.config.TaskReport;
|
|
@@ -27,6 +29,8 @@ import java.util.ArrayList;
|
|
|
27
29
|
import java.util.List;
|
|
28
30
|
import java.util.Map;
|
|
29
31
|
|
|
32
|
+
import static org.embulk.output.hdfs.HdfsFileOutputPlugin.PluginTask.*;
|
|
33
|
+
|
|
30
34
|
public class HdfsFileOutputPlugin
|
|
31
35
|
implements FileOutputPlugin
|
|
32
36
|
{
|
|
@@ -37,33 +41,38 @@ public class HdfsFileOutputPlugin
|
|
|
37
41
|
{
|
|
38
42
|
@Config("config_files")
|
|
39
43
|
@ConfigDefault("[]")
|
|
40
|
-
|
|
44
|
+
List<String> getConfigFiles();
|
|
41
45
|
|
|
42
46
|
@Config("config")
|
|
43
47
|
@ConfigDefault("{}")
|
|
44
|
-
|
|
48
|
+
Map<String, String> getConfig();
|
|
45
49
|
|
|
46
50
|
@Config("path_prefix")
|
|
47
|
-
|
|
51
|
+
String getPathPrefix();
|
|
48
52
|
|
|
49
53
|
@Config("file_ext")
|
|
50
|
-
|
|
54
|
+
String getFileExt();
|
|
51
55
|
|
|
52
56
|
@Config("sequence_format")
|
|
53
57
|
@ConfigDefault("\"%03d.%02d.\"")
|
|
54
|
-
|
|
58
|
+
String getSequenceFormat();
|
|
55
59
|
|
|
56
60
|
@Config("rewind_seconds")
|
|
57
61
|
@ConfigDefault("0")
|
|
58
|
-
|
|
62
|
+
int getRewindSeconds();
|
|
59
63
|
|
|
60
64
|
@Config("overwrite")
|
|
61
65
|
@ConfigDefault("false")
|
|
62
|
-
|
|
66
|
+
boolean getOverwrite();
|
|
63
67
|
|
|
64
68
|
@Config("doas")
|
|
65
69
|
@ConfigDefault("null")
|
|
66
|
-
|
|
70
|
+
Optional<String> getDoas();
|
|
71
|
+
|
|
72
|
+
enum DeleteInAdvancePolicy{ NONE, FILE_ONLY, RECURSIVE}
|
|
73
|
+
@Config("delete_in_advance")
|
|
74
|
+
@ConfigDefault("\"NONE\"")
|
|
75
|
+
DeleteInAdvancePolicy getDeleteInAdvance();
|
|
67
76
|
}
|
|
68
77
|
|
|
69
78
|
@Override
|
|
@@ -72,6 +81,15 @@ public class HdfsFileOutputPlugin
|
|
|
72
81
|
{
|
|
73
82
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
74
83
|
|
|
84
|
+
try {
|
|
85
|
+
String pathPrefix = strftime(task.getPathPrefix(), task.getRewindSeconds());
|
|
86
|
+
FileSystem fs = getFs(task);
|
|
87
|
+
deleteInAdvance(fs, pathPrefix, task.getDeleteInAdvance());
|
|
88
|
+
}
|
|
89
|
+
catch (IOException e) {
|
|
90
|
+
throw Throwables.propagate(e);
|
|
91
|
+
}
|
|
92
|
+
|
|
75
93
|
control.run(task.dump());
|
|
76
94
|
return Exec.newConfigDiff();
|
|
77
95
|
}
|
|
@@ -97,7 +115,7 @@ public class HdfsFileOutputPlugin
|
|
|
97
115
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
98
116
|
|
|
99
117
|
final String pathPrefix = strftime(task.getPathPrefix(), task.getRewindSeconds());
|
|
100
|
-
final String pathSuffix = task.
|
|
118
|
+
final String pathSuffix = task.getFileExt();
|
|
101
119
|
final String sequenceFormat = task.getSequenceFormat();
|
|
102
120
|
|
|
103
121
|
return new TransactionalFileOutput()
|
|
@@ -211,4 +229,31 @@ public class HdfsFileOutputPlugin
|
|
|
211
229
|
String.format("(Time.now - %s).strftime('%s')", String.valueOf(rewind_seconds), raw));
|
|
212
230
|
return resolved.toString();
|
|
213
231
|
}
|
|
232
|
+
|
|
233
|
+
private void deleteInAdvance(FileSystem fs, String pathPrefix, DeleteInAdvancePolicy deleteInAdvancePolicy)
|
|
234
|
+
throws IOException
|
|
235
|
+
{
|
|
236
|
+
final Path globPath = new Path(pathPrefix + "*");
|
|
237
|
+
switch (deleteInAdvancePolicy) {
|
|
238
|
+
case NONE:
|
|
239
|
+
// do nothing
|
|
240
|
+
break;
|
|
241
|
+
case FILE_ONLY:
|
|
242
|
+
for (FileStatus status : fs.globStatus(globPath)) {
|
|
243
|
+
if (status.isFile()) {
|
|
244
|
+
logger.debug("delete in advance: {}", status.getPath());
|
|
245
|
+
fs.delete(status.getPath(), false);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
break;
|
|
249
|
+
case RECURSIVE:
|
|
250
|
+
for (FileStatus status : fs.globStatus(globPath)) {
|
|
251
|
+
logger.debug("delete in advance: {}", status.getPath());
|
|
252
|
+
fs.delete(status.getPath(), true);
|
|
253
|
+
}
|
|
254
|
+
break;
|
|
255
|
+
default:
|
|
256
|
+
throw new ConfigException("`delete_in_advance` must not null.");
|
|
257
|
+
}
|
|
258
|
+
}
|
|
214
259
|
}
|
|
@@ -1,5 +1,280 @@
|
|
|
1
1
|
package org.embulk.output.hdfs;
|
|
2
2
|
|
|
3
|
+
import com.google.common.base.Charsets;
|
|
4
|
+
import com.google.common.base.Optional;
|
|
5
|
+
import com.google.common.collect.Lists;
|
|
6
|
+
import com.google.common.collect.Maps;
|
|
7
|
+
import org.embulk.EmbulkTestRuntime;
|
|
8
|
+
import org.embulk.config.ConfigException;
|
|
9
|
+
import org.embulk.config.ConfigSource;
|
|
10
|
+
import org.embulk.config.TaskReport;
|
|
11
|
+
import org.embulk.config.TaskSource;
|
|
12
|
+
import org.embulk.spi.Exec;
|
|
13
|
+
import org.embulk.spi.FileOutputRunner;
|
|
14
|
+
import org.embulk.spi.OutputPlugin.Control;
|
|
15
|
+
import org.embulk.spi.Page;
|
|
16
|
+
import org.embulk.spi.PageTestUtils;
|
|
17
|
+
import org.embulk.spi.Schema;
|
|
18
|
+
import org.embulk.spi.TransactionalPageOutput;
|
|
19
|
+
import org.embulk.spi.time.Timestamp;
|
|
20
|
+
import org.junit.Before;
|
|
21
|
+
import org.junit.Rule;
|
|
22
|
+
import org.junit.Test;
|
|
23
|
+
import org.junit.rules.ExpectedException;
|
|
24
|
+
import org.junit.rules.TemporaryFolder;
|
|
25
|
+
import org.slf4j.Logger;
|
|
26
|
+
|
|
27
|
+
import java.io.File;
|
|
28
|
+
import java.io.IOException;
|
|
29
|
+
import java.nio.file.DirectoryStream;
|
|
30
|
+
import java.nio.file.Files;
|
|
31
|
+
import java.nio.file.Paths;
|
|
32
|
+
import java.util.List;
|
|
33
|
+
|
|
34
|
+
import static com.google.common.io.Files.readLines;
|
|
35
|
+
import static org.embulk.output.hdfs.HdfsFileOutputPlugin.*;
|
|
36
|
+
import static org.embulk.spi.type.Types.*;
|
|
37
|
+
import static org.hamcrest.CoreMatchers.containsString;
|
|
38
|
+
import static org.hamcrest.CoreMatchers.hasItem;
|
|
39
|
+
import static org.hamcrest.CoreMatchers.not;
|
|
40
|
+
import static org.junit.Assert.assertEquals;
|
|
41
|
+
import static org.junit.Assert.assertNotEquals;
|
|
42
|
+
import static org.junit.Assert.assertThat;
|
|
43
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
|
44
|
+
import static org.msgpack.value.ValueFactory.newString;
|
|
45
|
+
|
|
3
46
|
public class TestHdfsFileOutputPlugin
|
|
4
47
|
{
|
|
48
|
+
@Rule
|
|
49
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
50
|
+
|
|
51
|
+
@Rule
|
|
52
|
+
public ExpectedException exception = ExpectedException.none();
|
|
53
|
+
|
|
54
|
+
@Rule
|
|
55
|
+
public TemporaryFolder tmpFolder = new TemporaryFolder();
|
|
56
|
+
|
|
57
|
+
private Logger logger = runtime.getExec().getLogger(TestHdfsFileOutputPlugin.class);
|
|
58
|
+
private HdfsFileOutputPlugin plugin;
|
|
59
|
+
private FileOutputRunner runner;
|
|
60
|
+
private String pathPrefix;
|
|
61
|
+
|
|
62
|
+
private static final Schema SCHEMA = new Schema.Builder()
|
|
63
|
+
.add("_c0", BOOLEAN)
|
|
64
|
+
.add("_c1", LONG)
|
|
65
|
+
.add("_c2", DOUBLE)
|
|
66
|
+
.add("_c3", STRING)
|
|
67
|
+
.add("_c4", TIMESTAMP)
|
|
68
|
+
.add("_c5", JSON)
|
|
69
|
+
.build();
|
|
70
|
+
|
|
71
|
+
@Before
|
|
72
|
+
public void createResources()
|
|
73
|
+
throws IOException
|
|
74
|
+
{
|
|
75
|
+
plugin = new HdfsFileOutputPlugin();
|
|
76
|
+
runner = new FileOutputRunner(runtime.getInstance(HdfsFileOutputPlugin.class));
|
|
77
|
+
pathPrefix = tmpFolder.getRoot().getAbsolutePath() + "/embulk-output-hdfs_";
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
private ConfigSource getBaseConfigSource()
|
|
81
|
+
{
|
|
82
|
+
return Exec.newConfigSource()
|
|
83
|
+
.set("type", "hdfs")
|
|
84
|
+
.set("path_prefix", pathPrefix)
|
|
85
|
+
.set("file_ext", "csv")
|
|
86
|
+
.setNested("formatter", Exec.newConfigSource()
|
|
87
|
+
.set("type", "csv")
|
|
88
|
+
.set("newline", "CRLF")
|
|
89
|
+
.set("newline_in_field", "LF")
|
|
90
|
+
.set("header_line", true)
|
|
91
|
+
.set("charset", "UTF-8")
|
|
92
|
+
.set("quote_policy", "NONE")
|
|
93
|
+
.set("quote", "\"")
|
|
94
|
+
.set("escape", "\\")
|
|
95
|
+
.set("null_string", "")
|
|
96
|
+
.set("default_timezone", "UTC"));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
@Test
|
|
100
|
+
public void testDefaultValues()
|
|
101
|
+
{
|
|
102
|
+
ConfigSource config = getBaseConfigSource();
|
|
103
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
104
|
+
assertEquals(pathPrefix, task.getPathPrefix());
|
|
105
|
+
assertEquals("csv", task.getFileExt());
|
|
106
|
+
assertEquals("%03d.%02d.", task.getSequenceFormat());
|
|
107
|
+
assertEquals(Lists.newArrayList(), task.getConfigFiles());
|
|
108
|
+
assertEquals(Maps.newHashMap(), task.getConfig());
|
|
109
|
+
assertEquals(0, task.getRewindSeconds());
|
|
110
|
+
assertEquals(false, task.getOverwrite());
|
|
111
|
+
assertEquals(Optional.absent(), task.getDoas());
|
|
112
|
+
assertEquals(PluginTask.DeleteInAdvancePolicy.NONE, task.getDeleteInAdvance());
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
@Test(expected = ConfigException.class)
|
|
116
|
+
public void testRequiredValues()
|
|
117
|
+
{
|
|
118
|
+
ConfigSource config = Exec.newConfigSource();
|
|
119
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private List<String> lsR(List<String> names, java.nio.file.Path dir)
|
|
123
|
+
{
|
|
124
|
+
try (DirectoryStream<java.nio.file.Path> stream = Files.newDirectoryStream(dir)) {
|
|
125
|
+
for (java.nio.file.Path path : stream) {
|
|
126
|
+
if (path.toFile().isDirectory()) {
|
|
127
|
+
logger.debug("[lsR] find a directory: {}", path.toAbsolutePath().toString());
|
|
128
|
+
names.add(path.toAbsolutePath().toString());
|
|
129
|
+
lsR(names, path);
|
|
130
|
+
}
|
|
131
|
+
else {
|
|
132
|
+
logger.debug("[lsR] find a file: {}", path.toAbsolutePath().toString());
|
|
133
|
+
names.add(path.toAbsolutePath().toString());
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
catch (IOException e) {
|
|
138
|
+
logger.debug(e.getMessage(), e);
|
|
139
|
+
}
|
|
140
|
+
return names;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private void run(ConfigSource config)
|
|
144
|
+
{
|
|
145
|
+
runner.transaction(config, SCHEMA, 1, new Control()
|
|
146
|
+
{
|
|
147
|
+
@Override
|
|
148
|
+
public List<TaskReport> run(TaskSource taskSource)
|
|
149
|
+
{
|
|
150
|
+
TransactionalPageOutput pageOutput = runner.open(taskSource, SCHEMA, 1);
|
|
151
|
+
boolean committed = false;
|
|
152
|
+
try {
|
|
153
|
+
// Result:
|
|
154
|
+
// _c0,_c1,_c2,_c3,_c4,_c5
|
|
155
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
156
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
157
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
|
158
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
|
|
159
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
|
|
160
|
+
pageOutput.add(page);
|
|
161
|
+
}
|
|
162
|
+
pageOutput.commit();
|
|
163
|
+
committed = true;
|
|
164
|
+
}
|
|
165
|
+
finally {
|
|
166
|
+
if (!committed) {
|
|
167
|
+
pageOutput.abort();
|
|
168
|
+
}
|
|
169
|
+
pageOutput.close();
|
|
170
|
+
}
|
|
171
|
+
return Lists.newArrayList();
|
|
172
|
+
}
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private void assertRecordsInFile(String filePath)
|
|
177
|
+
{
|
|
178
|
+
try {
|
|
179
|
+
List<String> lines = readLines(new File(filePath),
|
|
180
|
+
Charsets.UTF_8);
|
|
181
|
+
for (int i = 0; i < lines.size(); i++) {
|
|
182
|
+
String[] record = lines.get(i).split(",");
|
|
183
|
+
if (i == 0) {
|
|
184
|
+
for (int j = 0; j <= 4; j++) {
|
|
185
|
+
assertEquals("_c" + j, record[j]);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
|
190
|
+
assertEquals("true", record[0]);
|
|
191
|
+
assertEquals("2", record[1]);
|
|
192
|
+
assertEquals("3.0", record[2]);
|
|
193
|
+
assertEquals("45", record[3]);
|
|
194
|
+
assertEquals("1970-01-01 00:00:00.678000 +0000", record[4]);
|
|
195
|
+
assertEquals("{\"k\":\"v\"}", record[5]);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
catch (IOException e) {
|
|
200
|
+
logger.debug(e.getMessage(), e);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
@Test
|
|
205
|
+
public void testBulkLoad()
|
|
206
|
+
{
|
|
207
|
+
ConfigSource config = getBaseConfigSource()
|
|
208
|
+
.setNested("config", Exec.newConfigSource()
|
|
209
|
+
.set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
210
|
+
.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
211
|
+
.set("fs.defaultFS", "file:///"));
|
|
212
|
+
|
|
213
|
+
run(config);
|
|
214
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
|
|
215
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.csv")));
|
|
216
|
+
assertRecordsInFile(String.format("%s/%s001.00.csv",
|
|
217
|
+
tmpFolder.getRoot().getAbsolutePath(),
|
|
218
|
+
pathPrefix));
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
@Test
|
|
222
|
+
public void testDeleteRECURSIVEInAdvance()
|
|
223
|
+
throws IOException
|
|
224
|
+
{
|
|
225
|
+
for (int n = 0; n <= 10; n++) {
|
|
226
|
+
tmpFolder.newFile("embulk-output-hdfs_file_" + n + ".txt");
|
|
227
|
+
tmpFolder.newFolder("embulk-output-hdfs_directory_" + n);
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
|
|
231
|
+
|
|
232
|
+
ConfigSource config = getBaseConfigSource()
|
|
233
|
+
.setNested("config", Exec.newConfigSource()
|
|
234
|
+
.set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
235
|
+
.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
236
|
+
.set("fs.defaultFS", "file:///"))
|
|
237
|
+
.set("delete_in_advance", "RECURSIVE");
|
|
238
|
+
|
|
239
|
+
run(config);
|
|
240
|
+
|
|
241
|
+
List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
|
|
242
|
+
assertNotEquals(fileListBeforeRun, fileListAfterRun);
|
|
243
|
+
assertThat(fileListAfterRun, not(hasItem(containsString("embulk-output-hdfs_directory"))));
|
|
244
|
+
assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
|
|
245
|
+
assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
|
|
246
|
+
assertRecordsInFile(String.format("%s/%s001.00.csv",
|
|
247
|
+
tmpFolder.getRoot().getAbsolutePath(),
|
|
248
|
+
pathPrefix));
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
@Test
|
|
252
|
+
public void testDeleteFILE_ONLYInAdvance()
|
|
253
|
+
throws IOException
|
|
254
|
+
{
|
|
255
|
+
for (int n = 0; n <= 10; n++) {
|
|
256
|
+
tmpFolder.newFile("embulk-output-hdfs_file_" + n + ".txt");
|
|
257
|
+
tmpFolder.newFolder("embulk-output-hdfs_directory_" + n);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
List<String> fileListBeforeRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
|
|
261
|
+
|
|
262
|
+
ConfigSource config = getBaseConfigSource()
|
|
263
|
+
.setNested("config", Exec.newConfigSource()
|
|
264
|
+
.set("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
265
|
+
.set("fs.file.impl", "org.apache.hadoop.fs.RawLocalFileSystem")
|
|
266
|
+
.set("fs.defaultFS", "file:///"))
|
|
267
|
+
.set("delete_in_advance", "FILE_ONLY");
|
|
268
|
+
|
|
269
|
+
run(config);
|
|
270
|
+
|
|
271
|
+
List<String> fileListAfterRun = lsR(Lists.<String>newArrayList(), Paths.get(tmpFolder.getRoot().getAbsolutePath()));
|
|
272
|
+
assertNotEquals(fileListBeforeRun, fileListAfterRun);
|
|
273
|
+
assertThat(fileListAfterRun, not(hasItem(containsString("txt"))));
|
|
274
|
+
assertThat(fileListAfterRun, hasItem(containsString("embulk-output-hdfs_directory")));
|
|
275
|
+
assertThat(fileListAfterRun, hasItem(containsString(pathPrefix + "001.00.csv")));
|
|
276
|
+
assertRecordsInFile(String.format("%s/%s001.00.csv",
|
|
277
|
+
tmpFolder.getRoot().getAbsolutePath(),
|
|
278
|
+
pathPrefix));
|
|
279
|
+
}
|
|
5
280
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-hdfs
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Civitaspo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-04-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -46,10 +46,15 @@ extensions: []
|
|
|
46
46
|
extra_rdoc_files: []
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
|
+
- .travis.yml
|
|
49
50
|
- CHANGELOG.md
|
|
50
51
|
- LICENSE.txt
|
|
51
52
|
- README.md
|
|
52
53
|
- build.gradle
|
|
54
|
+
- config/checkstyle/checkstyle.xml
|
|
55
|
+
- config/checkstyle/default.xml
|
|
56
|
+
- example/config.yml
|
|
57
|
+
- example/data.csv
|
|
53
58
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
59
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
60
|
- gradlew
|
|
@@ -80,7 +85,7 @@ files:
|
|
|
80
85
|
- classpath/curator-client-2.6.0.jar
|
|
81
86
|
- classpath/curator-framework-2.6.0.jar
|
|
82
87
|
- classpath/curator-recipes-2.6.0.jar
|
|
83
|
-
- classpath/embulk-output-hdfs-0.2.
|
|
88
|
+
- classpath/embulk-output-hdfs-0.2.3.jar
|
|
84
89
|
- classpath/gson-2.2.4.jar
|
|
85
90
|
- classpath/hadoop-annotations-2.6.0.jar
|
|
86
91
|
- classpath/hadoop-auth-2.6.0.jar
|