embulk-output-hdfs 0.2.4 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +6 -1
- data/CHANGELOG.md +9 -0
- data/README.md +38 -9
- data/build.gradle +10 -8
- data/example/config.yml +3 -1
- data/example/config_deprecated_option.yml +52 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +1 -2
- data/gradlew +43 -35
- data/gradlew.bat +4 -10
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java +160 -0
- data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +55 -175
- data/src/main/java/org/embulk/output/hdfs/ModeTask.java +111 -0
- data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +269 -0
- data/src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java +76 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java +6 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java +53 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java +10 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java +22 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java +22 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java +11 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java +62 -0
- data/src/main/java/org/embulk/output/hdfs/transaction/Tx.java +13 -0
- data/src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java +21 -0
- data/src/main/java/org/embulk/output/hdfs/util/SamplePath.java +21 -0
- data/src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java +23 -0
- data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java +153 -22
- metadata +87 -70
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-hdfs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-12-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
41
|
description: Stores files on Hdfs.
|
42
42
|
email:
|
43
43
|
- civitaspo@gmail.com
|
@@ -55,86 +55,103 @@ files:
|
|
55
55
|
- config/checkstyle/default.xml
|
56
56
|
- example/config.yml
|
57
57
|
- example/config_avoid_create_0byte_file.yml
|
58
|
+
- example/config_deprecated_option.yml
|
58
59
|
- example/data.csv
|
59
60
|
- gradle/wrapper/gradle-wrapper.jar
|
60
61
|
- gradle/wrapper/gradle-wrapper.properties
|
61
62
|
- gradlew
|
62
63
|
- gradlew.bat
|
63
64
|
- lib/embulk/output/hdfs.rb
|
65
|
+
- settings.gradle
|
66
|
+
- src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java
|
64
67
|
- src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java
|
68
|
+
- src/main/java/org/embulk/output/hdfs/ModeTask.java
|
69
|
+
- src/main/java/org/embulk/output/hdfs/client/HdfsClient.java
|
70
|
+
- src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java
|
71
|
+
- src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java
|
72
|
+
- src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java
|
73
|
+
- src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java
|
74
|
+
- src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java
|
75
|
+
- src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java
|
76
|
+
- src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java
|
77
|
+
- src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java
|
78
|
+
- src/main/java/org/embulk/output/hdfs/transaction/Tx.java
|
79
|
+
- src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java
|
80
|
+
- src/main/java/org/embulk/output/hdfs/util/SamplePath.java
|
81
|
+
- src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java
|
65
82
|
- src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java
|
83
|
+
- classpath/jaxb-impl-2.2.3-1.jar
|
84
|
+
- classpath/jsr305-1.3.9.jar
|
66
85
|
- classpath/activation-1.1.jar
|
67
|
-
- classpath/
|
68
|
-
- classpath/apacheds-kerberos-codec-2.0.0-M15.jar
|
69
|
-
- classpath/api-asn1-api-1.0.0-M20.jar
|
70
|
-
- classpath/api-util-1.0.0-M20.jar
|
71
|
-
- classpath/asm-3.1.jar
|
72
|
-
- classpath/avro-1.7.4.jar
|
73
|
-
- classpath/commons-beanutils-1.7.0.jar
|
74
|
-
- classpath/commons-cli-1.2.jar
|
75
|
-
- classpath/commons-codec-1.6.jar
|
76
|
-
- classpath/commons-collections-3.2.1.jar
|
77
|
-
- classpath/commons-compress-1.4.1.jar
|
86
|
+
- classpath/curator-recipes-2.6.0.jar
|
78
87
|
- classpath/commons-configuration-1.6.jar
|
79
|
-
- classpath/commons-
|
88
|
+
- classpath/commons-beanutils-1.7.0.jar
|
89
|
+
- classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
|
90
|
+
- classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
|
91
|
+
- classpath/xz-1.0.jar
|
92
|
+
- classpath/hadoop-yarn-common-2.6.0.jar
|
80
93
|
- classpath/commons-httpclient-3.1.jar
|
81
|
-
- classpath/
|
82
|
-
- classpath/
|
94
|
+
- classpath/stax-api-1.0-2.jar
|
95
|
+
- classpath/slf4j-log4j12-1.7.5.jar
|
96
|
+
- classpath/apacheds-i18n-2.0.0-M15.jar
|
97
|
+
- classpath/httpclient-4.2.5.jar
|
98
|
+
- classpath/jline-0.9.94.jar
|
99
|
+
- classpath/jaxb-api-2.2.2.jar
|
100
|
+
- classpath/jackson-jaxrs-1.9.13.jar
|
101
|
+
- classpath/xercesImpl-2.9.1.jar
|
83
102
|
- classpath/commons-logging-1.1.3.jar
|
84
|
-
- classpath/commons-math3-3.1.1.jar
|
85
|
-
- classpath/commons-net-3.1.jar
|
86
|
-
- classpath/curator-client-2.6.0.jar
|
87
|
-
- classpath/curator-framework-2.6.0.jar
|
88
|
-
- classpath/curator-recipes-2.6.0.jar
|
89
|
-
- classpath/embulk-output-hdfs-0.2.4.jar
|
90
|
-
- classpath/gson-2.2.4.jar
|
91
|
-
- classpath/hadoop-annotations-2.6.0.jar
|
92
|
-
- classpath/hadoop-auth-2.6.0.jar
|
93
|
-
- classpath/hadoop-client-2.6.0.jar
|
94
|
-
- classpath/hadoop-common-2.6.0.jar
|
95
|
-
- classpath/hadoop-hdfs-2.6.0.jar
|
96
|
-
- classpath/hadoop-mapreduce-client-app-2.6.0.jar
|
97
103
|
- classpath/hadoop-mapreduce-client-common-2.6.0.jar
|
98
|
-
- classpath/hadoop-mapreduce-client-core-2.6.0.jar
|
99
|
-
- classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
|
100
|
-
- classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
|
101
|
-
- classpath/hadoop-yarn-api-2.6.0.jar
|
102
104
|
- classpath/hadoop-yarn-client-2.6.0.jar
|
103
|
-
- classpath/
|
105
|
+
- classpath/embulk-output-hdfs-0.3.0.jar
|
106
|
+
- classpath/jersey-json-1.9.jar
|
107
|
+
- classpath/avro-1.7.4.jar
|
108
|
+
- classpath/log4j-1.2.17.jar
|
109
|
+
- classpath/hadoop-auth-2.6.0.jar
|
110
|
+
- classpath/commons-cli-1.2.jar
|
111
|
+
- classpath/xml-apis-1.3.04.jar
|
112
|
+
- classpath/commons-digester-1.8.jar
|
113
|
+
- classpath/servlet-api-2.5.jar
|
114
|
+
- classpath/hadoop-hdfs-2.6.0.jar
|
115
|
+
- classpath/protobuf-java-2.5.0.jar
|
104
116
|
- classpath/hadoop-yarn-server-common-2.6.0.jar
|
105
|
-
- classpath/
|
117
|
+
- classpath/xmlenc-0.52.jar
|
118
|
+
- classpath/jackson-xc-1.9.13.jar
|
119
|
+
- classpath/jetty-util-6.1.26.jar
|
120
|
+
- classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
|
121
|
+
- classpath/hadoop-annotations-2.6.0.jar
|
122
|
+
- classpath/commons-compress-1.4.1.jar
|
106
123
|
- classpath/htrace-core-3.0.4.jar
|
107
|
-
- classpath/
|
108
|
-
- classpath/httpcore-4.2.4.jar
|
124
|
+
- classpath/commons-io-2.4.jar
|
109
125
|
- classpath/jackson-core-asl-1.9.13.jar
|
110
|
-
- classpath/jackson-jaxrs-1.9.13.jar
|
111
|
-
- classpath/jackson-mapper-asl-1.9.13.jar
|
112
|
-
- classpath/jackson-xc-1.9.13.jar
|
113
|
-
- classpath/jaxb-api-2.2.2.jar
|
114
|
-
- classpath/jaxb-impl-2.2.3-1.jar
|
115
|
-
- classpath/jersey-client-1.9.jar
|
116
126
|
- classpath/jersey-core-1.9.jar
|
117
|
-
- classpath/
|
118
|
-
- classpath/
|
127
|
+
- classpath/commons-codec-1.6.jar
|
128
|
+
- classpath/curator-framework-2.6.0.jar
|
129
|
+
- classpath/hadoop-client-2.6.0.jar
|
119
130
|
- classpath/jersey-server-1.9.jar
|
120
|
-
- classpath/jettison-1.1.jar
|
121
|
-
- classpath/jetty-util-6.1.26.jar
|
122
|
-
- classpath/jline-0.9.94.jar
|
123
|
-
- classpath/jsr305-1.3.9.jar
|
124
|
-
- classpath/leveldbjni-all-1.8.jar
|
125
|
-
- classpath/log4j-1.2.17.jar
|
126
131
|
- classpath/netty-3.7.0.Final.jar
|
132
|
+
- classpath/jersey-client-1.9.jar
|
133
|
+
- classpath/jersey-guice-1.9.jar
|
134
|
+
- classpath/curator-client-2.6.0.jar
|
127
135
|
- classpath/paranamer-2.3.jar
|
128
|
-
- classpath/protobuf-java-2.5.0.jar
|
129
|
-
- classpath/servlet-api-2.5.jar
|
130
|
-
- classpath/slf4j-log4j12-1.7.5.jar
|
131
|
-
- classpath/snappy-java-1.0.4.1.jar
|
132
|
-
- classpath/stax-api-1.0-2.jar
|
133
|
-
- classpath/xercesImpl-2.9.1.jar
|
134
|
-
- classpath/xml-apis-1.3.04.jar
|
135
|
-
- classpath/xmlenc-0.52.jar
|
136
|
-
- classpath/xz-1.0.jar
|
137
136
|
- classpath/zookeeper-3.4.6.jar
|
137
|
+
- classpath/jettison-1.1.jar
|
138
|
+
- classpath/api-asn1-api-1.0.0-M20.jar
|
139
|
+
- classpath/apacheds-kerberos-codec-2.0.0-M15.jar
|
140
|
+
- classpath/hadoop-mapreduce-client-app-2.6.0.jar
|
141
|
+
- classpath/commons-collections-3.2.1.jar
|
142
|
+
- classpath/asm-3.1.jar
|
143
|
+
- classpath/hadoop-yarn-api-2.6.0.jar
|
144
|
+
- classpath/api-util-1.0.0-M20.jar
|
145
|
+
- classpath/commons-net-3.1.jar
|
146
|
+
- classpath/gson-2.2.4.jar
|
147
|
+
- classpath/commons-lang-2.6.jar
|
148
|
+
- classpath/hadoop-mapreduce-client-core-2.6.0.jar
|
149
|
+
- classpath/leveldbjni-all-1.8.jar
|
150
|
+
- classpath/httpcore-4.2.4.jar
|
151
|
+
- classpath/snappy-java-1.0.4.1.jar
|
152
|
+
- classpath/jackson-mapper-asl-1.9.13.jar
|
153
|
+
- classpath/commons-math3-3.1.1.jar
|
154
|
+
- classpath/hadoop-common-2.6.0.jar
|
138
155
|
homepage: https://github.com/civitaspo/embulk-output-hdfs
|
139
156
|
licenses:
|
140
157
|
- MIT
|