embulk-output-hdfs 0.3.0 → 0.4.0.pre

Sign up to get free protection for your applications and to get access to all the features.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/.github/FUNDING.yml +1 -0
  3. data/.github/workflows/release.yml +40 -0
  4. data/.github/workflows/test.yml +25 -0
  5. data/CHANGELOG.md +6 -0
  6. data/README.md +2 -3
  7. data/build.gradle +15 -14
  8. data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java +2 -0
  9. metadata +106 -84
  10. data/.travis.yml +0 -14
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dd16ecf369d77f2a3dab20ebbb305dfbfca1a2a9
4
- data.tar.gz: e89f1b6e282e461abc6116c42dfe393abff4032e
3
+ metadata.gz: d56dc552885c22e4e7117d1545049e25728c4a3f
4
+ data.tar.gz: 5afe15a90f563621d5453d9f6cf6f2617090737e
5
5
  SHA512:
6
- metadata.gz: e8e335c24da1f14cd05492baaddb1c2de0c57d0cfa9a15eef17b85c288e1bb0f57ba91cf95f1006a612b16b624a57ac7d422a7853fe6ccf44d66caabc9984523
7
- data.tar.gz: 0049d9041ee1796b7e6e1cbc8dd1f7e0e738509f67abceaf0e7220ada5c95d9b63820cd38a5d37c720ee1e303e6504d3d38ba2524e9f9412143c5329cdba3186
6
+ metadata.gz: 785c4619d0128e1ee80aab2cad4da422a4a01876b680eb89c671001f363fd333ac9e30b6c7275a8114e01f243ef1f6d99119efe944ecc931080096f067f0457c
7
+ data.tar.gz: d1eb6e5c2b60a5c0df436684ba177197b0f4c809a436d41c91ef6a1fd2ce1cc9275237af54c61436506ab09323fe3b628ed70b79599deb055383d6acfbd3680e
data/.github/FUNDING.yml ADDED
@@ -0,0 +1 @@
1
+ github: civitaspo
data/.github/workflows/release.yml ADDED
@@ -0,0 +1,40 @@
1
+ name: Release CI
2
+
3
+ on:
4
+ pull_request:
5
+ branches:
6
+ - master
7
+ types:
8
+ - closed
9
+
10
+ jobs:
11
+ release:
12
+ runs-on: ubuntu-latest
13
+
14
+ steps:
15
+ - uses: actions/checkout@v1
16
+ - name: Set up JDK 1.8
17
+ uses: actions/setup-java@v1
18
+ with:
19
+ java-version: 1.8
20
+ - name: Test with Gradle
21
+ if: github.event.pull_request.merged == true
22
+ run: ./gradlew test
23
+ - name: Coveralls Test Report
24
+ if: github.event.pull_request.merged == true
25
+ run: ./gradlew jacocoTestReport coveralls
26
+ env:
27
+ COVERALLS_REPO_TOKEN: ${{secrets.COVERALLS_REPO_TOKEN}}
28
+ CI_NAME: Github Actions
29
+ CI_BRANCH: ${{github.ref}}
30
+ CI_PULL_REQUEST: ${{github.event.pull_request.html_url}}
31
+ - name: Release the new gem
32
+ if: github.event.pull_request.merged == true
33
+ run: |
34
+ mkdir -p $HOME/.gem
35
+ touch $HOME/.gem/credentials
36
+ chmod 0600 $HOME/.gem/credentials
37
+ printf -- "---\n:rubygems_api_key: ${RUBYGEMS_API_KEY}\n" > $HOME/.gem/credentials
38
+ ./gradlew gemPush
39
+ env:
40
+ RUBYGEMS_API_KEY: ${{secrets.RUBYGEMS_API_KEY}}
data/.github/workflows/test.yml ADDED
@@ -0,0 +1,25 @@
1
+ name: Test CI
2
+
3
+ on:
4
+ - pull_request
5
+
6
+ jobs:
7
+ test:
8
+ runs-on: ubuntu-latest
9
+
10
+ steps:
11
+ - uses: actions/checkout@v1
12
+ - name: Set up JDK 1.8
13
+ uses: actions/setup-java@v1
14
+ with:
15
+ java-version: 1.8
16
+ - name: Test with Gradle
17
+ run: ./gradlew test
18
+ - name: Coveralls Test Report
19
+ run: ./gradlew jacocoTestReport coveralls
20
+ env:
21
+ COVERALLS_REPO_TOKEN: ${{secrets.COVERALLS_REPO_TOKEN}}
22
+ CI_NAME: Github Actions
23
+ CI_BRANCH: ${{github.ref}}
24
+ CI_PULL_REQUEST: ${{github.event.pull_request.html_url}}
25
+
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ 0.4.0.pre (2019-12-07)
2
+ ======================
3
+ * Use Github Actions instead of TravisCI
4
+ * Update dependencies (java 7 -> 8, embulk v0.8.38 -> v0.9.20, hadoop-client 2.6.0 -> 2.9.2)
5
+ * Add `UserGroupInformation#setConfiguration` for kerberos authentication
6
+
1
7
  0.3.0 (2017-12-03)
2
8
  ==================
3
9
  * Add: `mode` option.
data/README.md CHANGED
@@ -1,7 +1,6 @@
1
1
  # Hdfs file output plugin for Embulk
2
2
 
3
- [![Build Status](https://travis-ci.org/civitaspo/embulk-output-hdfs.svg)](https://travis-ci.org/civitaspo/embulk-output-hdfs)
4
- [![Coverage Status](https://coveralls.io/repos/github/civitaspo/embulk-output-hdfs/badge.svg?branch=master)](https://coveralls.io/github/civitaspo/embulk-output-hdfs?branch=master)
3
+ ![Release CI Status Badge](https://github.com/civitaspo/embulk-output-hdfs/workflows/Release%20CI/badge.svg) ![Test CI Status Badge](https://github.com/civitaspo/embulk-output-hdfs/workflows/Test%20CI/badge.svg) [![Coverage Status](https://coveralls.io/repos/github/civitaspo/embulk-output-hdfs/badge.svg?branch=master)](https://coveralls.io/github/civitaspo/embulk-output-hdfs?branch=master)
5
4
 
6
5
  A File Output Plugin for Embulk to write HDFS.
7
6
 
@@ -94,5 +93,5 @@ $ ./gradlew gem
94
93
 
95
94
  ```
96
95
  $ ./gradlew classpath
97
- $ bundle exec embulk run -I lib example.yml
96
+ $ embulk run -I lib example/config.yml
98
97
  ```
data/build.gradle CHANGED
@@ -1,6 +1,6 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
3
+ id "com.github.jruby-gradle.base" version "1.5.0"
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  id "com.github.kt3k.coveralls" version "2.4.0"
@@ -15,20 +15,19 @@ configurations {
15
15
  provided
16
16
  }
17
17
 
18
- version = "0.3.0"
18
+ version = "0.4.0.pre"
19
19
 
20
- sourceCompatibility = 1.7
21
- targetCompatibility = 1.7
20
+ sourceCompatibility = 1.8
21
+ targetCompatibility = 1.8
22
22
 
23
23
  dependencies {
24
- compile "org.embulk:embulk-core:0.8.38"
25
- provided "org.embulk:embulk-core:0.8.38"
24
+ compile "org.embulk:embulk-core:0.9.20"
25
+ provided "org.embulk:embulk-core:0.9.20"
26
26
  // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
27
- compile 'org.apache.hadoop:hadoop-client:2.6.0'
28
- compile 'com.google.guava:guava:15.0'
27
+ compile 'org.apache.hadoop:hadoop-client:2.9.2'
29
28
  testCompile "junit:junit:4.+"
30
- testCompile "org.embulk:embulk-core:0.8.38:tests"
31
- testCompile "org.embulk:embulk-standards:0.8.38"
29
+ testCompile "org.embulk:embulk-core:0.9.20:tests"
30
+ testCompile "org.embulk:embulk-standards:0.9.20"
32
31
  }
33
32
 
34
33
  task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -62,14 +61,16 @@ task checkstyle(type: Checkstyle) {
62
61
  }
63
62
 
64
63
  task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
65
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
66
- script "${project.name}.gemspec"
64
+ jrubyArgs "-S"
65
+ script "gem"
66
+ scriptArgs "build", "${project.name}.gemspec"
67
67
  doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
68
68
  }
69
69
 
70
70
  task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
71
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
72
- script "pkg/${project.name}-${project.version}.gem"
71
+ jrubyArgs "-S"
72
+ script "gem"
73
+ scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
73
74
  }
74
75
 
75
76
  task "package"(dependsOn: ["gemspec", "classpath"]) {
data/src/main/java/org/embulk/output/hdfs/client/HdfsClient.java CHANGED
@@ -8,6 +8,7 @@ import org.apache.hadoop.fs.FileSystem;
8
8
  import org.apache.hadoop.fs.Options;
9
9
  import org.apache.hadoop.fs.Path;
10
10
  import org.apache.hadoop.fs.Trash;
11
+ import org.apache.hadoop.security.UserGroupInformation;
11
12
  import org.embulk.config.ConfigException;
12
13
  import org.embulk.output.hdfs.HdfsFileOutputPlugin;
13
14
  import org.embulk.spi.DataException;
@@ -47,6 +48,7 @@ public class HdfsClient
47
48
  for (Map.Entry<String, String> config : configs.entrySet()) {
48
49
  c.set(config.getKey(), config.getValue());
49
50
  }
51
+ UserGroupInformation.setConfiguration(c);
50
52
  return c;
51
53
  }
52
54
 
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-hdfs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0.pre
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-12-03 00:00:00.000000000 Z
11
+ date: 2019-12-07 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ~>
16
+ - - "~>"
17
17
  - !ruby/object:Gem::Version
18
18
  version: '1.0'
19
19
  name: bundler
@@ -21,13 +21,13 @@ dependencies:
21
21
  type: :development
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - '>='
30
+ - - ">="
31
31
  - !ruby/object:Gem::Version
32
32
  version: '10.0'
33
33
  name: rake
@@ -35,7 +35,7 @@ dependencies:
35
35
  type: :development
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '10.0'
41
41
  description: Stores files on Hdfs.
@@ -45,12 +45,106 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
- - .gitignore
49
- - .travis.yml
48
+ - ".github/FUNDING.yml"
49
+ - ".github/workflows/release.yml"
50
+ - ".github/workflows/test.yml"
51
+ - ".gitignore"
50
52
  - CHANGELOG.md
51
53
  - LICENSE.txt
52
54
  - README.md
53
55
  - build.gradle
56
+ - classpath/HikariCP-java7-2.4.12.jar
57
+ - classpath/accessors-smart-1.2.jar
58
+ - classpath/activation-1.1.jar
59
+ - classpath/apacheds-i18n-2.0.0-M15.jar
60
+ - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
61
+ - classpath/api-asn1-api-1.0.0-M20.jar
62
+ - classpath/api-util-1.0.0-M20.jar
63
+ - classpath/asm-3.1.jar
64
+ - classpath/asm-5.0.4.jar
65
+ - classpath/avro-1.7.7.jar
66
+ - classpath/commons-beanutils-1.7.0.jar
67
+ - classpath/commons-cli-1.2.jar
68
+ - classpath/commons-codec-1.9.jar
69
+ - classpath/commons-collections-3.2.2.jar
70
+ - classpath/commons-compress-1.4.1.jar
71
+ - classpath/commons-configuration-1.6.jar
72
+ - classpath/commons-digester-1.8.jar
73
+ - classpath/commons-io-2.4.jar
74
+ - classpath/commons-lang-2.6.jar
75
+ - classpath/commons-logging-1.2.jar
76
+ - classpath/commons-math3-3.1.1.jar
77
+ - classpath/commons-net-3.1.jar
78
+ - classpath/curator-client-2.7.1.jar
79
+ - classpath/curator-framework-2.7.1.jar
80
+ - classpath/curator-recipes-2.7.1.jar
81
+ - classpath/ehcache-3.3.1.jar
82
+ - classpath/embulk-output-hdfs-0.4.0.pre.jar
83
+ - classpath/geronimo-jcache_1.0_spec-1.0-alpha-1.jar
84
+ - classpath/gson-2.2.4.jar
85
+ - classpath/hadoop-annotations-2.9.2.jar
86
+ - classpath/hadoop-auth-2.9.2.jar
87
+ - classpath/hadoop-client-2.9.2.jar
88
+ - classpath/hadoop-common-2.9.2.jar
89
+ - classpath/hadoop-hdfs-client-2.9.2.jar
90
+ - classpath/hadoop-mapreduce-client-app-2.9.2.jar
91
+ - classpath/hadoop-mapreduce-client-common-2.9.2.jar
92
+ - classpath/hadoop-mapreduce-client-core-2.9.2.jar
93
+ - classpath/hadoop-mapreduce-client-jobclient-2.9.2.jar
94
+ - classpath/hadoop-mapreduce-client-shuffle-2.9.2.jar
95
+ - classpath/hadoop-yarn-api-2.9.2.jar
96
+ - classpath/hadoop-yarn-client-2.9.2.jar
97
+ - classpath/hadoop-yarn-common-2.9.2.jar
98
+ - classpath/hadoop-yarn-registry-2.9.2.jar
99
+ - classpath/hadoop-yarn-server-common-2.9.2.jar
100
+ - classpath/hadoop-yarn-server-nodemanager-2.9.2.jar
101
+ - classpath/htrace-core4-4.1.0-incubating.jar
102
+ - classpath/httpclient-4.5.2.jar
103
+ - classpath/httpcore-4.4.4.jar
104
+ - classpath/jackson-core-asl-1.9.13.jar
105
+ - classpath/jackson-jaxrs-1.9.13.jar
106
+ - classpath/jackson-mapper-asl-1.9.13.jar
107
+ - classpath/jackson-xc-1.9.13.jar
108
+ - classpath/java-xmlbuilder-0.4.jar
109
+ - classpath/jaxb-api-2.2.2.jar
110
+ - classpath/jaxb-impl-2.2.3-1.jar
111
+ - classpath/jcip-annotations-1.0-1.jar
112
+ - classpath/jersey-client-1.9.jar
113
+ - classpath/jersey-core-1.9.jar
114
+ - classpath/jersey-guice-1.9.jar
115
+ - classpath/jersey-json-1.9.jar
116
+ - classpath/jersey-server-1.9.jar
117
+ - classpath/jets3t-0.9.0.jar
118
+ - classpath/jettison-1.1.jar
119
+ - classpath/jetty-6.1.26.jar
120
+ - classpath/jetty-sslengine-6.1.26.jar
121
+ - classpath/jetty-util-6.1.26.jar
122
+ - classpath/jline-0.9.94.jar
123
+ - classpath/jsch-0.1.54.jar
124
+ - classpath/json-smart-2.3.jar
125
+ - classpath/jsp-api-2.1.jar
126
+ - classpath/jsr305-3.0.0.jar
127
+ - classpath/leveldbjni-all-1.8.jar
128
+ - classpath/log4j-1.2.17.jar
129
+ - classpath/metrics-core-3.0.1.jar
130
+ - classpath/mssql-jdbc-6.2.1.jre7.jar
131
+ - classpath/netty-3.7.0.Final.jar
132
+ - classpath/nimbus-jose-jwt-4.41.1.jar
133
+ - classpath/okhttp-2.7.5.jar
134
+ - classpath/okio-1.6.0.jar
135
+ - classpath/paranamer-2.3.jar
136
+ - classpath/protobuf-java-2.5.0.jar
137
+ - classpath/servlet-api-2.5-20081211.jar
138
+ - classpath/servlet-api-2.5.jar
139
+ - classpath/slf4j-api-1.7.25.jar
140
+ - classpath/slf4j-log4j12-1.7.25.jar
141
+ - classpath/snappy-java-1.0.5.jar
142
+ - classpath/stax-api-1.0-2.jar
143
+ - classpath/stax2-api-3.1.4.jar
144
+ - classpath/woodstox-core-5.0.3.jar
145
+ - classpath/xmlenc-0.52.jar
146
+ - classpath/xz-1.0.jar
147
+ - classpath/zookeeper-3.4.6.jar
54
148
  - config/checkstyle/checkstyle.xml
55
149
  - config/checkstyle/default.xml
56
150
  - example/config.yml
@@ -80,78 +174,6 @@ files:
80
174
  - src/main/java/org/embulk/output/hdfs/util/SamplePath.java
81
175
  - src/main/java/org/embulk/output/hdfs/util/StrftimeUtil.java
82
176
  - src/test/java/org/embulk/output/hdfs/TestHdfsFileOutputPlugin.java
83
- - classpath/jaxb-impl-2.2.3-1.jar
84
- - classpath/jsr305-1.3.9.jar
85
- - classpath/activation-1.1.jar
86
- - classpath/curator-recipes-2.6.0.jar
87
- - classpath/commons-configuration-1.6.jar
88
- - classpath/commons-beanutils-1.7.0.jar
89
- - classpath/hadoop-yarn-server-nodemanager-2.6.0.jar
90
- - classpath/hadoop-mapreduce-client-shuffle-2.6.0.jar
91
- - classpath/xz-1.0.jar
92
- - classpath/hadoop-yarn-common-2.6.0.jar
93
- - classpath/commons-httpclient-3.1.jar
94
- - classpath/stax-api-1.0-2.jar
95
- - classpath/slf4j-log4j12-1.7.5.jar
96
- - classpath/apacheds-i18n-2.0.0-M15.jar
97
- - classpath/httpclient-4.2.5.jar
98
- - classpath/jline-0.9.94.jar
99
- - classpath/jaxb-api-2.2.2.jar
100
- - classpath/jackson-jaxrs-1.9.13.jar
101
- - classpath/xercesImpl-2.9.1.jar
102
- - classpath/commons-logging-1.1.3.jar
103
- - classpath/hadoop-mapreduce-client-common-2.6.0.jar
104
- - classpath/hadoop-yarn-client-2.6.0.jar
105
- - classpath/embulk-output-hdfs-0.3.0.jar
106
- - classpath/jersey-json-1.9.jar
107
- - classpath/avro-1.7.4.jar
108
- - classpath/log4j-1.2.17.jar
109
- - classpath/hadoop-auth-2.6.0.jar
110
- - classpath/commons-cli-1.2.jar
111
- - classpath/xml-apis-1.3.04.jar
112
- - classpath/commons-digester-1.8.jar
113
- - classpath/servlet-api-2.5.jar
114
- - classpath/hadoop-hdfs-2.6.0.jar
115
- - classpath/protobuf-java-2.5.0.jar
116
- - classpath/hadoop-yarn-server-common-2.6.0.jar
117
- - classpath/xmlenc-0.52.jar
118
- - classpath/jackson-xc-1.9.13.jar
119
- - classpath/jetty-util-6.1.26.jar
120
- - classpath/hadoop-mapreduce-client-jobclient-2.6.0.jar
121
- - classpath/hadoop-annotations-2.6.0.jar
122
- - classpath/commons-compress-1.4.1.jar
123
- - classpath/htrace-core-3.0.4.jar
124
- - classpath/commons-io-2.4.jar
125
- - classpath/jackson-core-asl-1.9.13.jar
126
- - classpath/jersey-core-1.9.jar
127
- - classpath/commons-codec-1.6.jar
128
- - classpath/curator-framework-2.6.0.jar
129
- - classpath/hadoop-client-2.6.0.jar
130
- - classpath/jersey-server-1.9.jar
131
- - classpath/netty-3.7.0.Final.jar
132
- - classpath/jersey-client-1.9.jar
133
- - classpath/jersey-guice-1.9.jar
134
- - classpath/curator-client-2.6.0.jar
135
- - classpath/paranamer-2.3.jar
136
- - classpath/zookeeper-3.4.6.jar
137
- - classpath/jettison-1.1.jar
138
- - classpath/api-asn1-api-1.0.0-M20.jar
139
- - classpath/apacheds-kerberos-codec-2.0.0-M15.jar
140
- - classpath/hadoop-mapreduce-client-app-2.6.0.jar
141
- - classpath/commons-collections-3.2.1.jar
142
- - classpath/asm-3.1.jar
143
- - classpath/hadoop-yarn-api-2.6.0.jar
144
- - classpath/api-util-1.0.0-M20.jar
145
- - classpath/commons-net-3.1.jar
146
- - classpath/gson-2.2.4.jar
147
- - classpath/commons-lang-2.6.jar
148
- - classpath/hadoop-mapreduce-client-core-2.6.0.jar
149
- - classpath/leveldbjni-all-1.8.jar
150
- - classpath/httpcore-4.2.4.jar
151
- - classpath/snappy-java-1.0.4.1.jar
152
- - classpath/jackson-mapper-asl-1.9.13.jar
153
- - classpath/commons-math3-3.1.1.jar
154
- - classpath/hadoop-common-2.6.0.jar
155
177
  homepage: https://github.com/civitaspo/embulk-output-hdfs
156
178
  licenses:
157
179
  - MIT
@@ -162,17 +184,17 @@ require_paths:
162
184
  - lib
163
185
  required_ruby_version: !ruby/object:Gem::Requirement
164
186
  requirements:
165
- - - '>='
187
+ - - ">="
166
188
  - !ruby/object:Gem::Version
167
189
  version: '0'
168
190
  required_rubygems_version: !ruby/object:Gem::Requirement
169
191
  requirements:
170
- - - '>='
192
+ - - ">"
171
193
  - !ruby/object:Gem::Version
172
- version: '0'
194
+ version: 1.3.1
173
195
  requirements: []
174
196
  rubyforge_project:
175
- rubygems_version: 2.1.9
197
+ rubygems_version: 2.6.8
176
198
  signing_key:
177
199
  specification_version: 4
178
200
  summary: Hdfs file output plugin for Embulk
data/.travis.yml DELETED
@@ -1,14 +0,0 @@
1
- dist: precise
2
- language: java
3
- jdk:
4
- - openjdk7
5
- - oraclejdk7
6
- - oraclejdk8
7
- script:
8
- - ./gradlew test
9
- after_success:
10
- - ./gradlew jacocoTestReport coveralls
11
- addons:
12
- hosts:
13
- - example.com
14
- hostname: example.com