embulk-output-hdfs 0.2.4 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -1
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +38 -9
  5. data/build.gradle +10 -8
  6. data/example/config.yml +3 -1
  7. data/example/config_deprecated_option.yml +52 -0
  8. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  9. data/gradle/wrapper/gradle-wrapper.properties +1 -2
  10. data/gradlew +43 -35
  11. data/gradlew.bat +4 -10
  12. data/settings.gradle +1 -0
  13. data/src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java +160 -0
  14. data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +55 -175
  15. data/src/main/java/org/embulk/output/hdfs/ModeTask.java +111 -0
  16. data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +269 -0
  17. data/src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java +76 -0
  18. data/src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java +6 -0
  19. data/src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java +53 -0
  20. data/src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java +10 -0
  21. data/src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java +22 -0
  22. data/src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java +22 -0
  23. data/src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java +11 -0
  24. data/src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java +62 -0
  25. data/src/main/java/org/embulk/output/hdfs/transaction/Tx.java +13 -0
  26. data/src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java +21 -0
  27. data/src/main/java/org/embulk/output/hdfs/util/SamplePath.java +21 -0
  28. data/src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java +23 -0
  29. data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java +153 -22
  30. metadata +87 -70
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-hdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.4
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-27 00:00:00.000000000 Z
11
+ date: 2017-12-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - ~>
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - ~>
32
25
  - !ruby/object:Gem::Version
33
- version: '10.0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: '10.0'
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
41
  description: Stores files on Hdfs.
42
42
  email:
43
43
  - civitaspo@gmail.com
@@ -55,86 +55,103 @@ files:
55
55
  - config/checkstyle/default.xml
56
56
  - example/config.yml
57
57
  - example/config_avoid_create_0byte_file.yml
58
+ - example/config_deprecated_option.yml
58
59
  - example/data.csv
59
60
  - gradle/wrapper/gradle-wrapper.jar
60
61
  - gradle/wrapper/gradle-wrapper.properties
61
62
  - gradlew
62
63
  - gradlew.bat
63
64
  - lib/embulk/output/hdfs.rb
65
+ - settings.gradle
66
+ - src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java
64
67
  - src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java
68
+ - src/main/java/org/embulk/output/hdfs/ModeTask.java
69
+ - src/main/java/org/embulk/output/hdfs/client/HdfsClient.java
70
+ - src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java
71
+ - src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java
72
+ - src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java
73
+ - src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java
74
+ - src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java
75
+ - src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java
76
+ - src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java
77
+ - src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java
78
+ - src/main/java/org/embulk/output/hdfs/transaction/Tx.java
79
+ - src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java
80
+ - src/main/java/org/embulk/output/hdfs/util/SamplePath.java
81
+ - src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java
65
82
  - src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java
83
+ - classpath/jaxb-impl-2.2.3-1.jar
84
+ - classpath/jsr305-1.3.9.jar
66
85
  - classpath/activation-1.1.jar
67
- - classpath/apacheds-i18n-2.0.0-M15.jar
68
- - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
69
- - classpath/api-asn1-api-1.0.0-M20.jar
70
- - classpath/api-util-1.0.0-M20.jar
71
- - classpath/asm-3.1.jar
72
- - classpath/avro-1.7.4.jar
73
- - classpath/commons-beanutils-1.7.0.jar
74
- - classpath/commons-cli-1.2.jar
75
- - classpath/commons-codec-1.6.jar
76
- - classpath/commons-collections-3.2.1.jar
77
- - classpath/commons-compress-1.4.1.jar
86
+ - classpath/curator-recipes-2.6.0.jar
78
87
  - classpath/commons-configuration-1.6.jar
79
- - classpath/commons-digester-1.8.jar
88
+ - classpath/commons-beanutils-1.7.0.jar
89
+ - classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
90
+ - classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
91
+ - classpath/xz-1.0.jar
92
+ - classpath/hadoop-yarn-common-2.6.0.jar
80
93
  - classpath/commons-httpclient-3.1.jar
81
- - classpath/commons-io-2.4.jar
82
- - classpath/commons-lang-2.6.jar
94
+ - classpath/stax-api-1.0-2.jar
95
+ - classpath/slf4j-log4j12-1.7.5.jar
96
+ - classpath/apacheds-i18n-2.0.0-M15.jar
97
+ - classpath/httpclient-4.2.5.jar
98
+ - classpath/jline-0.9.94.jar
99
+ - classpath/jaxb-api-2.2.2.jar
100
+ - classpath/jackson-jaxrs-1.9.13.jar
101
+ - classpath/xercesImpl-2.9.1.jar
83
102
  - classpath/commons-logging-1.1.3.jar
84
- - classpath/commons-math3-3.1.1.jar
85
- - classpath/commons-net-3.1.jar
86
- - classpath/curator-client-2.6.0.jar
87
- - classpath/curator-framework-2.6.0.jar
88
- - classpath/curator-recipes-2.6.0.jar
89
- - classpath/embulk-output-hdfs-0.2.4.jar
90
- - classpath/gson-2.2.4.jar
91
- - classpath/hadoop-annotations-2.6.0.jar
92
- - classpath/hadoop-auth-2.6.0.jar
93
- - classpath/hadoop-client-2.6.0.jar
94
- - classpath/hadoop-common-2.6.0.jar
95
- - classpath/hadoop-hdfs-2.6.0.jar
96
- - classpath/hadoop-mapreduce-client-app-2.6.0.jar
97
103
  - classpath/hadoop-mapreduce-client-common-2.6.0.jar
98
- - classpath/hadoop-mapreduce-client-core-2.6.0.jar
99
- - classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
100
- - classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
101
- - classpath/hadoop-yarn-api-2.6.0.jar
102
104
  - classpath/hadoop-yarn-client-2.6.0.jar
103
- - classpath/hadoop-yarn-common-2.6.0.jar
105
+ - classpath/embulk-output-hdfs-0.3.0.jar
106
+ - classpath/jersey-json-1.9.jar
107
+ - classpath/avro-1.7.4.jar
108
+ - classpath/log4j-1.2.17.jar
109
+ - classpath/hadoop-auth-2.6.0.jar
110
+ - classpath/commons-cli-1.2.jar
111
+ - classpath/xml-apis-1.3.04.jar
112
+ - classpath/commons-digester-1.8.jar
113
+ - classpath/servlet-api-2.5.jar
114
+ - classpath/hadoop-hdfs-2.6.0.jar
115
+ - classpath/protobuf-java-2.5.0.jar
104
116
  - classpath/hadoop-yarn-server-common-2.6.0.jar
105
- - classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
117
+ - classpath/xmlenc-0.52.jar
118
+ - classpath/jackson-xc-1.9.13.jar
119
+ - classpath/jetty-util-6.1.26.jar
120
+ - classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
121
+ - classpath/hadoop-annotations-2.6.0.jar
122
+ - classpath/commons-compress-1.4.1.jar
106
123
  - classpath/htrace-core-3.0.4.jar
107
- - classpath/httpclient-4.2.5.jar
108
- - classpath/httpcore-4.2.4.jar
124
+ - classpath/commons-io-2.4.jar
109
125
  - classpath/jackson-core-asl-1.9.13.jar
110
- - classpath/jackson-jaxrs-1.9.13.jar
111
- - classpath/jackson-mapper-asl-1.9.13.jar
112
- - classpath/jackson-xc-1.9.13.jar
113
- - classpath/jaxb-api-2.2.2.jar
114
- - classpath/jaxb-impl-2.2.3-1.jar
115
- - classpath/jersey-client-1.9.jar
116
126
  - classpath/jersey-core-1.9.jar
117
- - classpath/jersey-guice-1.9.jar
118
- - classpath/jersey-json-1.9.jar
127
+ - classpath/commons-codec-1.6.jar
128
+ - classpath/curator-framework-2.6.0.jar
129
+ - classpath/hadoop-client-2.6.0.jar
119
130
  - classpath/jersey-server-1.9.jar
120
- - classpath/jettison-1.1.jar
121
- - classpath/jetty-util-6.1.26.jar
122
- - classpath/jline-0.9.94.jar
123
- - classpath/jsr305-1.3.9.jar
124
- - classpath/leveldbjni-all-1.8.jar
125
- - classpath/log4j-1.2.17.jar
126
131
  - classpath/netty-3.7.0.Final.jar
132
+ - classpath/jersey-client-1.9.jar
133
+ - classpath/jersey-guice-1.9.jar
134
+ - classpath/curator-client-2.6.0.jar
127
135
  - classpath/paranamer-2.3.jar
128
- - classpath/protobuf-java-2.5.0.jar
129
- - classpath/servlet-api-2.5.jar
130
- - classpath/slf4j-log4j12-1.7.5.jar
131
- - classpath/snappy-java-1.0.4.1.jar
132
- - classpath/stax-api-1.0-2.jar
133
- - classpath/xercesImpl-2.9.1.jar
134
- - classpath/xml-apis-1.3.04.jar
135
- - classpath/xmlenc-0.52.jar
136
- - classpath/xz-1.0.jar
137
136
  - classpath/zookeeper-3.4.6.jar
137
+ - classpath/jettison-1.1.jar
138
+ - classpath/api-asn1-api-1.0.0-M20.jar
139
+ - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
140
+ - classpath/hadoop-mapreduce-client-app-2.6.0.jar
141
+ - classpath/commons-collections-3.2.1.jar
142
+ - classpath/asm-3.1.jar
143
+ - classpath/hadoop-yarn-api-2.6.0.jar
144
+ - classpath/api-util-1.0.0-M20.jar
145
+ - classpath/commons-net-3.1.jar
146
+ - classpath/gson-2.2.4.jar
147
+ - classpath/commons-lang-2.6.jar
148
+ - classpath/hadoop-mapreduce-client-core-2.6.0.jar
149
+ - classpath/leveldbjni-all-1.8.jar
150
+ - classpath/httpcore-4.2.4.jar
151
+ - classpath/snappy-java-1.0.4.1.jar
152
+ - classpath/jackson-mapper-asl-1.9.13.jar
153
+ - classpath/commons-math3-3.1.1.jar
154
+ - classpath/hadoop-common-2.6.0.jar
138
155
  homepage: https://github.com/civitaspo/embulk-output-hdfs
139
156
  licenses:
140
157
  - MIT