embulk-output-hdfs 0.2.4 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +6 -1
  3. data/CHANGELOG.md +9 -0
  4. data/README.md +38 -9
  5. data/build.gradle +10 -8
  6. data/example/config.yml +3 -1
  7. data/example/config_deprecated_option.yml +52 -0
  8. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  9. data/gradle/wrapper/gradle-wrapper.properties +1 -2
  10. data/gradlew +43 -35
  11. data/gradlew.bat +4 -10
  12. data/settings.gradle +1 -0
  13. data/src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java +160 -0
  14. data/src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java +55 -175
  15. data/src/main/java/org/embulk/output/hdfs/ModeTask.java +111 -0
  16. data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +269 -0
  17. data/src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java +76 -0
  18. data/src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java +6 -0
  19. data/src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java +53 -0
  20. data/src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java +10 -0
  21. data/src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java +22 -0
  22. data/src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java +22 -0
  23. data/src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java +11 -0
  24. data/src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java +62 -0
  25. data/src/main/java/org/embulk/output/hdfs/transaction/Tx.java +13 -0
  26. data/src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java +21 -0
  27. data/src/main/java/org/embulk/output/hdfs/util/SamplePath.java +21 -0
  28. data/src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java +23 -0
  29. data/src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java +153 -22
  30. metadata +87 -70
metadata CHANGED
@@ -1,43 +1,43 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-hdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.4
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-04-27 00:00:00.000000000 Z
11
+ date: 2017-12-03 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - ~>
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - ~>
32
25
  - !ruby/object:Gem::Version
33
- version: '10.0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: '10.0'
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
41
  description: Stores files on Hdfs.
42
42
  email:
43
43
  - civitaspo@gmail.com
@@ -55,86 +55,103 @@ files:
55
55
  - config/checkstyle/default.xml
56
56
  - example/config.yml
57
57
  - example/config_avoid_create_0byte_file.yml
58
+ - example/config_deprecated_option.yml
58
59
  - example/data.csv
59
60
  - gradle/wrapper/gradle-wrapper.jar
60
61
  - gradle/wrapper/gradle-wrapper.properties
61
62
  - gradlew
62
63
  - gradlew.bat
63
64
  - lib/embulk/output/hdfs.rb
65
+ - settings.gradle
66
+ - src/main/java/org/embulk/output/hdfs/HdfsFileOutput.java
64
67
  - src/main/java/org/embulk/output/hdfs/HdfsFileOutputPlugin.java
68
+ - src/main/java/org/embulk/output/hdfs/ModeTask.java
69
+ - src/main/java/org/embulk/output/hdfs/client/HdfsClient.java
70
+ - src/main/java/org/embulk/output/hdfs/compat/ModeCompat.java
71
+ - src/main/java/org/embulk/output/hdfs/transaction/AbortIfExistTx.java
72
+ - src/main/java/org/embulk/output/hdfs/transaction/AbstractTx.java
73
+ - src/main/java/org/embulk/output/hdfs/transaction/ControlRun.java
74
+ - src/main/java/org/embulk/output/hdfs/transaction/DeleteFilesInAdvanceTx.java
75
+ - src/main/java/org/embulk/output/hdfs/transaction/DeleteRecursiveInAdvanceTx.java
76
+ - src/main/java/org/embulk/output/hdfs/transaction/OverwriteTx.java
77
+ - src/main/java/org/embulk/output/hdfs/transaction/ReplaceTx.java
78
+ - src/main/java/org/embulk/output/hdfs/transaction/Tx.java
79
+ - src/main/java/org/embulk/output/hdfs/util/SafeWorkspaceName.java
80
+ - src/main/java/org/embulk/output/hdfs/util/SamplePath.java
81
+ - src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java
65
82
  - src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java
83
+ - classpath/jaxb-impl-2.2.3-1.jar
84
+ - classpath/jsr305-1.3.9.jar
66
85
  - classpath/activation-1.1.jar
67
- - classpath/apacheds-i18n-2.0.0-M15.jar
68
- - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
69
- - classpath/api-asn1-api-1.0.0-M20.jar
70
- - classpath/api-util-1.0.0-M20.jar
71
- - classpath/asm-3.1.jar
72
- - classpath/avro-1.7.4.jar
73
- - classpath/commons-beanutils-1.7.0.jar
74
- - classpath/commons-cli-1.2.jar
75
- - classpath/commons-codec-1.6.jar
76
- - classpath/commons-collections-3.2.1.jar
77
- - classpath/commons-compress-1.4.1.jar
86
+ - classpath/curator-recipes-2.6.0.jar
78
87
  - classpath/commons-configuration-1.6.jar
79
- - classpath/commons-digester-1.8.jar
88
+ - classpath/commons-beanutils-1.7.0.jar
89
+ - classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
90
+ - classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
91
+ - classpath/xz-1.0.jar
92
+ - classpath/hadoop-yarn-common-2.6.0.jar
80
93
  - classpath/commons-httpclient-3.1.jar
81
- - classpath/commons-io-2.4.jar
82
- - classpath/commons-lang-2.6.jar
94
+ - classpath/stax-api-1.0-2.jar
95
+ - classpath/slf4j-log4j12-1.7.5.jar
96
+ - classpath/apacheds-i18n-2.0.0-M15.jar
97
+ - classpath/httpclient-4.2.5.jar
98
+ - classpath/jline-0.9.94.jar
99
+ - classpath/jaxb-api-2.2.2.jar
100
+ - classpath/jackson-jaxrs-1.9.13.jar
101
+ - classpath/xercesImpl-2.9.1.jar
83
102
  - classpath/commons-logging-1.1.3.jar
84
- - classpath/commons-math3-3.1.1.jar
85
- - classpath/commons-net-3.1.jar
86
- - classpath/curator-client-2.6.0.jar
87
- - classpath/curator-framework-2.6.0.jar
88
- - classpath/curator-recipes-2.6.0.jar
89
- - classpath/embulk-output-hdfs-0.2.4.jar
90
- - classpath/gson-2.2.4.jar
91
- - classpath/hadoop-annotations-2.6.0.jar
92
- - classpath/hadoop-auth-2.6.0.jar
93
- - classpath/hadoop-client-2.6.0.jar
94
- - classpath/hadoop-common-2.6.0.jar
95
- - classpath/hadoop-hdfs-2.6.0.jar
96
- - classpath/hadoop-mapreduce-client-app-2.6.0.jar
97
103
  - classpath/hadoop-mapreduce-client-common-2.6.0.jar
98
- - classpath/hadoop-mapreduce-client-core-2.6.0.jar
99
- - classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
100
- - classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
101
- - classpath/hadoop-yarn-api-2.6.0.jar
102
104
  - classpath/hadoop-yarn-client-2.6.0.jar
103
- - classpath/hadoop-yarn-common-2.6.0.jar
105
+ - classpath/embulk-output-hdfs-0.3.0.jar
106
+ - classpath/jersey-json-1.9.jar
107
+ - classpath/avro-1.7.4.jar
108
+ - classpath/log4j-1.2.17.jar
109
+ - classpath/hadoop-auth-2.6.0.jar
110
+ - classpath/commons-cli-1.2.jar
111
+ - classpath/xml-apis-1.3.04.jar
112
+ - classpath/commons-digester-1.8.jar
113
+ - classpath/servlet-api-2.5.jar
114
+ - classpath/hadoop-hdfs-2.6.0.jar
115
+ - classpath/protobuf-java-2.5.0.jar
104
116
  - classpath/hadoop-yarn-server-common-2.6.0.jar
105
- - classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
117
+ - classpath/xmlenc-0.52.jar
118
+ - classpath/jackson-xc-1.9.13.jar
119
+ - classpath/jetty-util-6.1.26.jar
120
+ - classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
121
+ - classpath/hadoop-annotations-2.6.0.jar
122
+ - classpath/commons-compress-1.4.1.jar
106
123
  - classpath/htrace-core-3.0.4.jar
107
- - classpath/httpclient-4.2.5.jar
108
- - classpath/httpcore-4.2.4.jar
124
+ - classpath/commons-io-2.4.jar
109
125
  - classpath/jackson-core-asl-1.9.13.jar
110
- - classpath/jackson-jaxrs-1.9.13.jar
111
- - classpath/jackson-mapper-asl-1.9.13.jar
112
- - classpath/jackson-xc-1.9.13.jar
113
- - classpath/jaxb-api-2.2.2.jar
114
- - classpath/jaxb-impl-2.2.3-1.jar
115
- - classpath/jersey-client-1.9.jar
116
126
  - classpath/jersey-core-1.9.jar
117
- - classpath/jersey-guice-1.9.jar
118
- - classpath/jersey-json-1.9.jar
127
+ - classpath/commons-codec-1.6.jar
128
+ - classpath/curator-framework-2.6.0.jar
129
+ - classpath/hadoop-client-2.6.0.jar
119
130
  - classpath/jersey-server-1.9.jar
120
- - classpath/jettison-1.1.jar
121
- - classpath/jetty-util-6.1.26.jar
122
- - classpath/jline-0.9.94.jar
123
- - classpath/jsr305-1.3.9.jar
124
- - classpath/leveldbjni-all-1.8.jar
125
- - classpath/log4j-1.2.17.jar
126
131
  - classpath/netty-3.7.0.Final.jar
132
+ - classpath/jersey-client-1.9.jar
133
+ - classpath/jersey-guice-1.9.jar
134
+ - classpath/curator-client-2.6.0.jar
127
135
  - classpath/paranamer-2.3.jar
128
- - classpath/protobuf-java-2.5.0.jar
129
- - classpath/servlet-api-2.5.jar
130
- - classpath/slf4j-log4j12-1.7.5.jar
131
- - classpath/snappy-java-1.0.4.1.jar
132
- - classpath/stax-api-1.0-2.jar
133
- - classpath/xercesImpl-2.9.1.jar
134
- - classpath/xml-apis-1.3.04.jar
135
- - classpath/xmlenc-0.52.jar
136
- - classpath/xz-1.0.jar
137
136
  - classpath/zookeeper-3.4.6.jar
137
+ - classpath/jettison-1.1.jar
138
+ - classpath/api-asn1-api-1.0.0-M20.jar
139
+ - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
140
+ - classpath/hadoop-mapreduce-client-app-2.6.0.jar
141
+ - classpath/commons-collections-3.2.1.jar
142
+ - classpath/asm-3.1.jar
143
+ - classpath/hadoop-yarn-api-2.6.0.jar
144
+ - classpath/api-util-1.0.0-M20.jar
145
+ - classpath/commons-net-3.1.jar
146
+ - classpath/gson-2.2.4.jar
147
+ - classpath/commons-lang-2.6.jar
148
+ - classpath/hadoop-mapreduce-client-core-2.6.0.jar
149
+ - classpath/leveldbjni-all-1.8.jar
150
+ - classpath/httpcore-4.2.4.jar
151
+ - classpath/snappy-java-1.0.4.1.jar
152
+ - classpath/jackson-mapper-asl-1.9.13.jar
153
+ - classpath/commons-math3-3.1.1.jar
154
+ - classpath/hadoop-common-2.6.0.jar
138
155
  homepage: https://github.com/civitaspo/embulk-output-hdfs
139
156
  licenses:
140
157
  - MIT