embulk-output-orc 0.3.2 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -6
- data/build.gradle +17 -11
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -1
- data/src/main/java/org/embulk/output/orc/OrcColumnVisitor.java +24 -5
- data/src/main/java/org/embulk/output/orc/OrcOutputPlugin.java +8 -4
- data/src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java +90 -7
- data/src/test/java/org/embulk/output/orc/OrcOutputPluginHelperTest.java +71 -0
- data/src/test/resources/example-null.yml +25 -0
- data/src/test/resources/example.yml +25 -0
- metadata +44 -37
- data/src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f90685be3f76457be27d9ee129b56276a66dd42b
|
4
|
+
data.tar.gz: ac33ac212a7bb6352aef8eb0caf310f5afd25bbe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: aecde246d738967d2a91906560d9735ea11981e8f7349da57d7418a3ab08a7796fb5573f1e9b72b54ff6eee1a08f4ade83194024bfcf67ff6113ff6f570fb7b6
|
7
|
+
data.tar.gz: 34b3eb0dc0c01388edf4310f5f55d00b0eab65b95b2def0a888867df8f79e0087c66d70e099d2af6327e57be03a80a75a6135cf281a58f5c4a0b8e22ed7caa1a
|
data/README.md
CHANGED
@@ -13,15 +13,16 @@
|
|
13
13
|
## Configuration
|
14
14
|
|
15
15
|
- **path_prefix**: A prefix of output path. (string, required)
|
16
|
-
- support: `file`, `s3n` and `s3a`.
|
16
|
+
- support: `file`, `s3`, `s3n` and `s3a`.
|
17
17
|
- **file_ext**: An extension of output file. (string, default: `.orc`)
|
18
18
|
- **sequence_format**: (string, default: `.%03d`)
|
19
|
-
- **buffer_size**: Set the ORC buffer size (integer, default: `262144`)
|
20
|
-
- **strip_size**: Set the ORC strip size (integer, default: `67108864`)
|
21
|
-
- **block_size**: Set the ORC block size (integer, default: `268435456`)
|
19
|
+
- **buffer_size**: Set the ORC buffer size (integer, default: `262144(256KB)` )
|
20
|
+
- **strip_size**: Set the ORC strip size (integer, default: `67108864(64MB)` )
|
21
|
+
- **block_size**: Set the ORC block size (integer, default: `268435456(256MB)`)
|
22
22
|
- **compression_kind**: description (string, default: `'ZLIB'`)
|
23
|
-
- `NONE`, `ZLIB`, `SNAPPY`
|
24
|
-
- **overwrite**:
|
23
|
+
- `NONE`, `ZLIB`, `SNAPPY`, `LZO`, `LZ4`
|
24
|
+
- **overwrite**: Overwrite if output files already exist. (boolean, default: `false`)
|
25
|
+
- Support: `LocalFileSystem`, `S3(s3, s3a, s3n)`
|
25
26
|
- **default_from_timezone** Time zone of timestamp columns. This can be overwritten for each column using column_options (DateTimeZone, default: `UTC`)
|
26
27
|
|
27
28
|
- **auth_method**: name of mechanism to authenticate requests (basic, env, instance, profile, properties, anonymous, or session. default: basic)
|
@@ -42,6 +43,11 @@ out:
|
|
42
43
|
|
43
44
|
## ChangeLog
|
44
45
|
|
46
|
+
### ver 0.3.3
|
47
|
+
|
48
|
+
- bugfix
|
49
|
+
- Bump `orc` library to `1.4.4`
|
50
|
+
|
45
51
|
### ver 0.3.2
|
46
52
|
|
47
53
|
- Update `orc` libraries to `1.4.3`
|
data/build.gradle
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
plugins {
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
3
|
-
id "com.github.jruby-gradle.base" version "
|
3
|
+
id "com.github.jruby-gradle.base" version "1.5.0"
|
4
4
|
id "java"
|
5
5
|
id "checkstyle"
|
6
6
|
id "org.sonarqube" version "2.5"
|
@@ -18,7 +18,7 @@ configurations {
|
|
18
18
|
runtime.exclude group: "org.slf4j", module: "slf4j-log4j12"
|
19
19
|
}
|
20
20
|
|
21
|
-
version = "0.3.
|
21
|
+
version = "0.3.3"
|
22
22
|
|
23
23
|
sourceCompatibility = 1.8
|
24
24
|
targetCompatibility = 1.8
|
@@ -27,15 +27,19 @@ dependencies {
|
|
27
27
|
compile "org.embulk:embulk-core:0.8.34"
|
28
28
|
provided "org.embulk:embulk-core:0.8.34"
|
29
29
|
|
30
|
-
compile "org.apache.orc:orc:1.4.
|
31
|
-
compile "org.apache.orc:orc-core:1.4.
|
32
|
-
compile "org.apache.hadoop:hadoop-hdfs:2.
|
30
|
+
compile "org.apache.orc:orc:1.4.4"
|
31
|
+
compile "org.apache.orc:orc-core:1.4.4"
|
32
|
+
compile "org.apache.hadoop:hadoop-hdfs:2.7.5"
|
33
33
|
|
34
34
|
compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8'
|
35
35
|
compile "com.amazonaws:aws-java-sdk-s3:1.10.33"
|
36
|
-
compile "org.apache.hadoop:hadoop-aws:2.7.
|
36
|
+
compile "org.apache.hadoop:hadoop-aws:2.7.5"
|
37
|
+
compile 'com.google.guava:guava:24.1-jre'
|
37
38
|
|
38
|
-
testCompile
|
39
|
+
testCompile 'org.jmockit:jmockit:1.38'
|
40
|
+
// testCompile "junit:junit:4.+"
|
41
|
+
testCompile 'org.hamcrest:hamcrest-core:1.3'
|
42
|
+
testCompile 'org.testng:testng:6.14.2'
|
39
43
|
testCompile "org.embulk:embulk-core:0.8.34:tests"
|
40
44
|
testCompile "org.embulk:embulk-standards:0.8.34"
|
41
45
|
}
|
@@ -65,14 +69,16 @@ task checkstyle(type: Checkstyle) {
|
|
65
69
|
}
|
66
70
|
|
67
71
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
68
|
-
jrubyArgs "-
|
69
|
-
script "
|
72
|
+
jrubyArgs "-S"
|
73
|
+
script "gem"
|
74
|
+
scriptArgs "build", "${project.name}.gemspec"
|
70
75
|
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
71
76
|
}
|
72
77
|
|
73
78
|
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
|
74
|
-
jrubyArgs "-
|
75
|
-
script "
|
79
|
+
jrubyArgs "-S"
|
80
|
+
script "gem"
|
81
|
+
scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
|
76
82
|
}
|
77
83
|
|
78
84
|
task "package"(dependsOn: ["gemspec", "classpath"]) {
|
Binary file
|
@@ -1,5 +1,6 @@
|
|
1
|
+
#Wed Jan 09 23:08:09 JST 2019
|
1
2
|
distributionBase=GRADLE_USER_HOME
|
2
3
|
distributionPath=wrapper/dists
|
3
4
|
zipStoreBase=GRADLE_USER_HOME
|
4
5
|
zipStorePath=wrapper/dists
|
5
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-4.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-all.zip
|
@@ -30,7 +30,8 @@ public class OrcColumnVisitor
|
|
30
30
|
public void booleanColumn(Column column)
|
31
31
|
{
|
32
32
|
if (reader.isNull(column)) {
|
33
|
-
|
33
|
+
batch.cols[column.getIndex()].noNulls = false;
|
34
|
+
batch.cols[column.getIndex()].isNull[i] = true;
|
34
35
|
}
|
35
36
|
else {
|
36
37
|
if (reader.getBoolean(column)) {
|
@@ -45,20 +46,38 @@ public class OrcColumnVisitor
|
|
45
46
|
@Override
|
46
47
|
public void longColumn(Column column)
|
47
48
|
{
|
48
|
-
|
49
|
+
if (reader.isNull(column)) {
|
50
|
+
batch.cols[column.getIndex()].noNulls = false;
|
51
|
+
batch.cols[column.getIndex()].isNull[i] = true;
|
52
|
+
}
|
53
|
+
else {
|
54
|
+
((LongColumnVector) batch.cols[column.getIndex()]).vector[i] = reader.getLong(column);
|
55
|
+
}
|
49
56
|
}
|
50
57
|
|
51
58
|
@Override
|
52
59
|
public void doubleColumn(Column column)
|
53
60
|
{
|
54
|
-
|
61
|
+
if (reader.isNull(column)) {
|
62
|
+
batch.cols[column.getIndex()].noNulls = false;
|
63
|
+
batch.cols[column.getIndex()].isNull[i] = true;
|
64
|
+
}
|
65
|
+
else {
|
66
|
+
((DoubleColumnVector) batch.cols[column.getIndex()]).vector[i] = reader.getDouble(column);
|
67
|
+
}
|
55
68
|
}
|
56
69
|
|
57
70
|
@Override
|
58
71
|
public void stringColumn(Column column)
|
59
72
|
{
|
60
|
-
((
|
61
|
-
|
73
|
+
if (!reader.isNull(column)) {
|
74
|
+
((BytesColumnVector) batch.cols[column.getIndex()])
|
75
|
+
.setVal(i, reader.getString(column).getBytes(StandardCharsets.UTF_8));
|
76
|
+
}
|
77
|
+
else {
|
78
|
+
batch.cols[column.getIndex()].noNulls = false;
|
79
|
+
batch.cols[column.getIndex()].isNull[i] = true;
|
80
|
+
}
|
62
81
|
}
|
63
82
|
|
64
83
|
@Override
|
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.output.orc;
|
2
2
|
|
3
|
+
import com.amazonaws.auth.AWSCredentials;
|
3
4
|
import com.google.common.base.Throwables;
|
4
5
|
import org.apache.hadoop.conf.Configuration;
|
5
6
|
import org.apache.hadoop.fs.LocalFileSystem;
|
@@ -60,7 +61,6 @@ public class OrcOutputPlugin
|
|
60
61
|
public void cleanup(TaskSource taskSource,
|
61
62
|
Schema schema, int taskCount,
|
62
63
|
List<TaskReport> successTaskReports)
|
63
|
-
|
64
64
|
{
|
65
65
|
}
|
66
66
|
|
@@ -70,7 +70,8 @@ public class OrcOutputPlugin
|
|
70
70
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
71
71
|
|
72
72
|
if (task.getOverwrite()) {
|
73
|
-
|
73
|
+
AWSCredentials credentials = AwsCredentials.getAWSCredentialsProvider(task).getCredentials();
|
74
|
+
OrcOutputPluginHelper.removeOldFile(buildPath(task, taskIndex), task);
|
74
75
|
}
|
75
76
|
|
76
77
|
final PageReader reader = new PageReader(schema);
|
@@ -137,6 +138,7 @@ public class OrcOutputPlugin
|
|
137
138
|
}
|
138
139
|
if (task.getEndpoint().isPresent()) {
|
139
140
|
conf.set("fs.s3a.endpoint", task.getEndpoint().get());
|
141
|
+
conf.set("fs.s3n.endpoint", task.getEndpoint().get());
|
140
142
|
}
|
141
143
|
return conf;
|
142
144
|
}
|
@@ -158,9 +160,11 @@ public class OrcOutputPlugin
|
|
158
160
|
OrcFile.WriterOptions writerOptions = createWriterOptions(task, conf);
|
159
161
|
// see: https://stackoverflow.com/questions/9256733/how-to-connect-hive-in-ireport
|
160
162
|
// see: https://community.hortonworks.com/content/kbentry/73458/connecting-dbvisualizer-and-datagrip-to-hive-with.html
|
161
|
-
writer = OrcFile.createWriter(
|
163
|
+
writer = OrcFile.createWriter(
|
164
|
+
new Path(buildPath(task, processorIndex)),
|
162
165
|
writerOptions.setSchema(oschema)
|
163
|
-
.version(OrcFile.Version.V_0_12)
|
166
|
+
.version(OrcFile.Version.V_0_12)
|
167
|
+
);
|
164
168
|
}
|
165
169
|
catch (IOException e) {
|
166
170
|
Throwables.propagate(e);
|
@@ -1,11 +1,19 @@
|
|
1
1
|
package org.embulk.output.orc;
|
2
2
|
|
3
|
+
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
|
4
|
+
import com.amazonaws.services.s3.AmazonS3;
|
5
|
+
import com.amazonaws.services.s3.AmazonS3Client;
|
6
|
+
import com.amazonaws.services.s3.model.DeleteObjectRequest;
|
7
|
+
import com.google.common.base.Joiner;
|
8
|
+
import com.google.common.base.Splitter;
|
3
9
|
import com.google.common.base.Throwables;
|
4
10
|
|
5
11
|
import java.io.IOException;
|
6
12
|
import java.nio.file.Files;
|
7
13
|
import java.nio.file.Path;
|
8
14
|
import java.nio.file.Paths;
|
15
|
+
import java.util.Arrays;
|
16
|
+
import java.util.List;
|
9
17
|
|
10
18
|
class OrcOutputPluginHelper
|
11
19
|
{
|
@@ -14,15 +22,90 @@ class OrcOutputPluginHelper
|
|
14
22
|
throw new UnsupportedOperationException();
|
15
23
|
}
|
16
24
|
|
17
|
-
static void removeOldFile(String fpath)
|
25
|
+
static void removeOldFile(String fpath, PluginTask task)
|
18
26
|
{
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
27
|
+
// NOTE: Delete a file if local-filesystem, not HDFS or S3.
|
28
|
+
String schema = getSchema(fpath);
|
29
|
+
if (isDeleteTarget(schema)) {
|
30
|
+
switch (schema) {
|
31
|
+
case "file":
|
32
|
+
try {
|
33
|
+
Files.deleteIfExists(Paths.get(fpath));
|
34
|
+
}
|
35
|
+
catch (IOException e) {
|
36
|
+
Throwables.propagate(e);
|
37
|
+
}
|
38
|
+
break;
|
39
|
+
case "s3":
|
40
|
+
case "s3n":
|
41
|
+
case "s3a":
|
42
|
+
AmazonS3URILikeObject s3Url = parseS3Url(fpath);
|
43
|
+
AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
|
44
|
+
if (task.getEndpoint().isPresent()) {
|
45
|
+
s3client.setEndpoint(task.getEndpoint().get());
|
46
|
+
}
|
47
|
+
s3client.deleteObject(new DeleteObjectRequest(s3Url.getBucket(), s3Url.getKey()));
|
48
|
+
default:
|
49
|
+
// TODO: Unsupported
|
50
|
+
}
|
23
51
|
}
|
24
|
-
|
25
|
-
|
52
|
+
}
|
53
|
+
|
54
|
+
public static boolean isDeleteTarget(String schema)
|
55
|
+
{
|
56
|
+
switch (schema) {
|
57
|
+
case "file":
|
58
|
+
return true;
|
59
|
+
case "s3":
|
60
|
+
case "s3a":
|
61
|
+
case "s3n":
|
62
|
+
return true;
|
63
|
+
default:
|
64
|
+
return false;
|
65
|
+
}
|
66
|
+
}
|
67
|
+
|
68
|
+
static String getSchema(String fpath)
|
69
|
+
{
|
70
|
+
String schema = Splitter.on("://")
|
71
|
+
.splitToList(fpath).get(0);
|
72
|
+
if (schema.equals("s3a") || schema.equals("s3n") || schema.equals("s3")) {
|
73
|
+
return schema;
|
74
|
+
}
|
75
|
+
else {
|
76
|
+
Path path = Paths.get(fpath);
|
77
|
+
return path.getFileSystem().provider().getScheme();
|
78
|
+
}
|
79
|
+
}
|
80
|
+
|
81
|
+
static AmazonS3URILikeObject parseS3Url(String s3url)
|
82
|
+
{
|
83
|
+
List<String> parts = Arrays.asList(
|
84
|
+
s3url.split("(://|/)"));
|
85
|
+
String bucket = parts.get(1);
|
86
|
+
String key = Joiner.on("/").join(parts.subList(2, parts.size()));
|
87
|
+
return new AmazonS3URILikeObject(bucket, key);
|
88
|
+
}
|
89
|
+
|
90
|
+
static class AmazonS3URILikeObject
|
91
|
+
{
|
92
|
+
String bucket;
|
93
|
+
String key;
|
94
|
+
|
95
|
+
public AmazonS3URILikeObject(String bucket, String key)
|
96
|
+
{
|
97
|
+
this.bucket = bucket;
|
98
|
+
this.key = key;
|
99
|
+
}
|
100
|
+
|
101
|
+
public String getBucket()
|
102
|
+
{
|
103
|
+
return bucket;
|
104
|
+
}
|
105
|
+
|
106
|
+
public String getKey()
|
107
|
+
{
|
108
|
+
return key;
|
26
109
|
}
|
27
110
|
}
|
28
111
|
}
|
@@ -0,0 +1,71 @@
|
|
1
|
+
package org.embulk.output.orc;
|
2
|
+
|
3
|
+
import org.testng.annotations.DataProvider;
|
4
|
+
import org.testng.annotations.Test;
|
5
|
+
|
6
|
+
import static org.hamcrest.MatcherAssert.assertThat;
|
7
|
+
import static org.hamcrest.core.Is.is;
|
8
|
+
|
9
|
+
public class OrcOutputPluginHelperTest
|
10
|
+
{
|
11
|
+
@DataProvider(name = "url-provider")
|
12
|
+
public Object[][] dataProvider()
|
13
|
+
{
|
14
|
+
return new Object[][] {
|
15
|
+
{"file://tmp/output.orc", "file"},
|
16
|
+
{"/tmp/output.000.orc", "file"},
|
17
|
+
{"s3n://embulk-test/output.0001.orc", "s3n"},
|
18
|
+
{"s3a://embulk-test/output.0001.orc", "s3a"},
|
19
|
+
{"s3://embulk-test/output.0001.orc", "s3"},
|
20
|
+
};
|
21
|
+
}
|
22
|
+
|
23
|
+
@Test(dataProvider = "url-provider")
|
24
|
+
public void getFPathTest(String file, String expect)
|
25
|
+
{
|
26
|
+
String schema = OrcOutputPluginHelper.getSchema(file);
|
27
|
+
assertThat(schema, is(expect));
|
28
|
+
}
|
29
|
+
|
30
|
+
@DataProvider(name = "schema-provider")
|
31
|
+
public Object[][] schemaProvider()
|
32
|
+
{
|
33
|
+
return new Object[][] {
|
34
|
+
{"file", true},
|
35
|
+
{"s3", true},
|
36
|
+
{"s3n", true},
|
37
|
+
{"s3a", true},
|
38
|
+
{"hdfs", false},
|
39
|
+
};
|
40
|
+
}
|
41
|
+
|
42
|
+
@Test(dataProvider = "schema-provider")
|
43
|
+
public void isDeleteTargetTest(String schema, boolean expect)
|
44
|
+
{
|
45
|
+
boolean result = OrcOutputPluginHelper.isDeleteTarget(schema);
|
46
|
+
assertThat(result, is(expect));
|
47
|
+
}
|
48
|
+
|
49
|
+
@DataProvider(name = "parserTest-provider")
|
50
|
+
public Object[][] parserTestProvider()
|
51
|
+
{
|
52
|
+
String baseurl = "demo-bucket/test/output.000.orc";
|
53
|
+
String bucket = "demo-bucket";
|
54
|
+
String keyname = "test/output.000.orc";
|
55
|
+
|
56
|
+
return new Object[][] {
|
57
|
+
{"s3://" + baseurl, bucket, keyname},
|
58
|
+
{"s3a://" + baseurl, bucket, keyname},
|
59
|
+
{"s3n://" + baseurl, bucket, keyname},
|
60
|
+
};
|
61
|
+
}
|
62
|
+
|
63
|
+
@Test(dataProvider = "parserTest-provider")
|
64
|
+
public void parseS3UrlTest(String url, String bucket, String key)
|
65
|
+
{
|
66
|
+
OrcOutputPluginHelper.AmazonS3URILikeObject parts =
|
67
|
+
OrcOutputPluginHelper.parseS3Url(url);
|
68
|
+
assertThat(parts.getBucket(), is(bucket));
|
69
|
+
assertThat(parts.getKey(), is(key));
|
70
|
+
}
|
71
|
+
}
|
@@ -0,0 +1,25 @@
|
|
1
|
+
---
|
2
|
+
in:
|
3
|
+
type: randomj
|
4
|
+
rows: 8
|
5
|
+
threads: 1
|
6
|
+
# default_timezone: Asia/Tokyo
|
7
|
+
primary_key: myid
|
8
|
+
schema:
|
9
|
+
- {name: myid, type: long}
|
10
|
+
- {name: named, type: string, null_rate: 10000}
|
11
|
+
- {name: x_flag, type: boolean, null_rate: 10000}
|
12
|
+
- {name: pit_rate, type: double, null_rate: 10000}
|
13
|
+
- {name: score, type: long, null_rate: 10000}
|
14
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S', null_rate: 10000}
|
15
|
+
- {name: purchase, type: timestamp, format: '%Y/%m/%d', null_rate: 10000}
|
16
|
+
|
17
|
+
exec:
|
18
|
+
max_threads: 2 # run at most 8 tasks concurrently
|
19
|
+
min_output_tasks: 1 # disable page scattering
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: orc
|
23
|
+
overwrite: true
|
24
|
+
path_prefix: "/tmp/output"
|
25
|
+
compression_kind: ZLIB
|
@@ -0,0 +1,25 @@
|
|
1
|
+
---
|
2
|
+
in:
|
3
|
+
type: randomj
|
4
|
+
rows: 1024
|
5
|
+
threads: 1
|
6
|
+
# default_timezone: Asia/Tokyo
|
7
|
+
primary_key: myid
|
8
|
+
schema:
|
9
|
+
- {name: myid, type: long}
|
10
|
+
- {name: named, type: string, null_rate: 1000}
|
11
|
+
- {name: x_flag, type: boolean, null_rate: 1000}
|
12
|
+
- {name: pit_rate, type: double, null_rate: 1000}
|
13
|
+
- {name: score, type: long, null_rate: 1000}
|
14
|
+
- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
|
15
|
+
- {name: purchase, type: timestamp, format: '%Y/%m/%d'}
|
16
|
+
|
17
|
+
exec:
|
18
|
+
max_threads: 2 # run at most 8 tasks concurrently
|
19
|
+
min_output_tasks: 1 # disable page scattering
|
20
|
+
|
21
|
+
out:
|
22
|
+
type: orc
|
23
|
+
overwrite: true
|
24
|
+
path_prefix: "/tmp/output"
|
25
|
+
compression_kind: ZLIB
|
metadata
CHANGED
@@ -1,19 +1,19 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-orc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yuokada
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-01-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
|
-
- - ~>
|
16
|
+
- - "~>"
|
17
17
|
- !ruby/object:Gem::Version
|
18
18
|
version: '1.0'
|
19
19
|
name: bundler
|
@@ -21,13 +21,13 @@ dependencies:
|
|
21
21
|
type: :development
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
29
29
|
requirements:
|
30
|
-
- -
|
30
|
+
- - ">="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '10.0'
|
33
33
|
name: rake
|
@@ -35,7 +35,7 @@ dependencies:
|
|
35
35
|
type: :development
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '10.0'
|
41
41
|
description: Dumps records to Orc format file.
|
@@ -45,27 +45,14 @@ executables: []
|
|
45
45
|
extensions: []
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
|
-
- .gitignore
|
49
|
-
- .travis.yml
|
48
|
+
- ".gitignore"
|
49
|
+
- ".travis.yml"
|
50
50
|
- LICENSE.txt
|
51
51
|
- README.md
|
52
52
|
- build.gradle
|
53
|
-
- config/checkstyle/checkstyle.xml
|
54
|
-
- config/checkstyle/default.xml
|
55
|
-
- example/example.yml
|
56
|
-
- gradle/wrapper/gradle-wrapper.jar
|
57
|
-
- gradle/wrapper/gradle-wrapper.properties
|
58
|
-
- gradlew
|
59
|
-
- gradlew.bat
|
60
|
-
- lib/embulk/output/orc.rb
|
61
|
-
- src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
|
62
|
-
- src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
|
63
|
-
- src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
|
64
|
-
- src/main/java/org/embulk/output/orc/PluginTask.java
|
65
|
-
- src/main/java/org/embulk/output/orc/TimestampColumnOption.java
|
66
|
-
- src/test/java/org/embulk/output/orc/TestOrcOutputPlugin.java
|
67
53
|
- classpath/activation-1.1.jar
|
68
54
|
- classpath/aircompressor-0.8.jar
|
55
|
+
- classpath/animal-sniffer-annotations-1.14.jar
|
69
56
|
- classpath/apacheds-i18n-2.0.0-M15.jar
|
70
57
|
- classpath/apacheds-kerberos-codec-2.0.0-M15.jar
|
71
58
|
- classpath/api-asn1-api-1.0.0-M20.jar
|
@@ -76,6 +63,7 @@ files:
|
|
76
63
|
- classpath/aws-java-sdk-core-1.10.33.jar
|
77
64
|
- classpath/aws-java-sdk-kms-1.10.33.jar
|
78
65
|
- classpath/aws-java-sdk-s3-1.10.33.jar
|
66
|
+
- classpath/checker-compat-qual-2.0.0.jar
|
79
67
|
- classpath/commons-beanutils-1.7.0.jar
|
80
68
|
- classpath/commons-cli-1.2.jar
|
81
69
|
- classpath/commons-codec-1.6.jar
|
@@ -84,7 +72,6 @@ files:
|
|
84
72
|
- classpath/commons-configuration-1.6.jar
|
85
73
|
- classpath/commons-daemon-1.0.13.jar
|
86
74
|
- classpath/commons-digester-1.8.jar
|
87
|
-
- classpath/commons-el-1.0.jar
|
88
75
|
- classpath/commons-httpclient-3.1.jar
|
89
76
|
- classpath/commons-io-2.4.jar
|
90
77
|
- classpath/commons-lang-2.6.jar
|
@@ -94,24 +81,25 @@ files:
|
|
94
81
|
- classpath/curator-client-2.7.1.jar
|
95
82
|
- classpath/curator-framework-2.7.1.jar
|
96
83
|
- classpath/curator-recipes-2.7.1.jar
|
97
|
-
- classpath/embulk-output-orc-0.3.
|
84
|
+
- classpath/embulk-output-orc-0.3.3.jar
|
98
85
|
- classpath/embulk-util-aws-credentials-0.2.8.jar
|
86
|
+
- classpath/error_prone_annotations-2.1.3.jar
|
99
87
|
- classpath/gson-2.2.4.jar
|
100
|
-
- classpath/
|
101
|
-
- classpath/hadoop-
|
102
|
-
- classpath/hadoop-
|
103
|
-
- classpath/hadoop-
|
104
|
-
- classpath/hadoop-
|
88
|
+
- classpath/guava-24.1-jre.jar
|
89
|
+
- classpath/hadoop-annotations-2.7.5.jar
|
90
|
+
- classpath/hadoop-auth-2.7.5.jar
|
91
|
+
- classpath/hadoop-aws-2.7.5.jar
|
92
|
+
- classpath/hadoop-common-2.7.5.jar
|
93
|
+
- classpath/hadoop-hdfs-2.7.5.jar
|
105
94
|
- classpath/hive-storage-api-2.2.1.jar
|
106
|
-
- classpath/htrace-core-3.0.4.jar
|
107
95
|
- classpath/htrace-core-3.1.0-incubating.jar
|
108
96
|
- classpath/httpclient-4.3.6.jar
|
109
97
|
- classpath/httpcore-4.3.3.jar
|
98
|
+
- classpath/j2objc-annotations-1.1.jar
|
110
99
|
- classpath/jackson-core-asl-1.9.13.jar
|
111
100
|
- classpath/jackson-jaxrs-1.8.3.jar
|
112
101
|
- classpath/jackson-mapper-asl-1.9.13.jar
|
113
102
|
- classpath/jackson-xc-1.8.3.jar
|
114
|
-
- classpath/jasper-runtime-5.5.23.jar
|
115
103
|
- classpath/java-xmlbuilder-0.4.jar
|
116
104
|
- classpath/jaxb-api-2.2.2.jar
|
117
105
|
- classpath/jaxb-impl-2.2.3-1.jar
|
@@ -122,17 +110,20 @@ files:
|
|
122
110
|
- classpath/jets3t-0.9.0.jar
|
123
111
|
- classpath/jettison-1.1.jar
|
124
112
|
- classpath/jetty-6.1.26.jar
|
113
|
+
- classpath/jetty-sslengine-6.1.26.jar
|
125
114
|
- classpath/jetty-util-6.1.26.jar
|
126
115
|
- classpath/jline-0.9.94.jar
|
127
|
-
- classpath/
|
128
|
-
- classpath/jsch-0.1.42.jar
|
116
|
+
- classpath/jsch-0.1.54.jar
|
129
117
|
- classpath/jsp-api-2.1.jar
|
130
118
|
- classpath/jsr305-3.0.0.jar
|
119
|
+
- classpath/leveldbjni-all-1.8.jar
|
131
120
|
- classpath/log4j-1.2.17.jar
|
132
121
|
- classpath/netty-3.7.0.Final.jar
|
133
|
-
- classpath/
|
122
|
+
- classpath/netty-all-4.0.23.Final.jar
|
123
|
+
- classpath/orc-core-1.4.4.jar
|
134
124
|
- classpath/paranamer-2.3.jar
|
135
125
|
- classpath/protobuf-java-2.5.0.jar
|
126
|
+
- classpath/servlet-api-2.5-20081211.jar
|
136
127
|
- classpath/servlet-api-2.5.jar
|
137
128
|
- classpath/snappy-java-1.0.4.1.jar
|
138
129
|
- classpath/stax-api-1.0-2.jar
|
@@ -141,6 +132,22 @@ files:
|
|
141
132
|
- classpath/xmlenc-0.52.jar
|
142
133
|
- classpath/xz-1.0.jar
|
143
134
|
- classpath/zookeeper-3.4.6.jar
|
135
|
+
- config/checkstyle/checkstyle.xml
|
136
|
+
- config/checkstyle/default.xml
|
137
|
+
- example/example.yml
|
138
|
+
- gradle/wrapper/gradle-wrapper.jar
|
139
|
+
- gradle/wrapper/gradle-wrapper.properties
|
140
|
+
- gradlew
|
141
|
+
- gradlew.bat
|
142
|
+
- lib/embulk/output/orc.rb
|
143
|
+
- src/main/java/org/embulk/output/orc/OrcColumnVisitor.java
|
144
|
+
- src/main/java/org/embulk/output/orc/OrcOutputPlugin.java
|
145
|
+
- src/main/java/org/embulk/output/orc/OrcOutputPluginHelper.java
|
146
|
+
- src/main/java/org/embulk/output/orc/PluginTask.java
|
147
|
+
- src/main/java/org/embulk/output/orc/TimestampColumnOption.java
|
148
|
+
- src/test/java/org/embulk/output/orc/OrcOutputPluginHelperTest.java
|
149
|
+
- src/test/resources/example-null.yml
|
150
|
+
- src/test/resources/example.yml
|
144
151
|
homepage: https://github.com/yuokada/embulk-output-orc
|
145
152
|
licenses:
|
146
153
|
- MIT
|
@@ -151,17 +158,17 @@ require_paths:
|
|
151
158
|
- lib
|
152
159
|
required_ruby_version: !ruby/object:Gem::Requirement
|
153
160
|
requirements:
|
154
|
-
- -
|
161
|
+
- - ">="
|
155
162
|
- !ruby/object:Gem::Version
|
156
163
|
version: '0'
|
157
164
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
158
165
|
requirements:
|
159
|
-
- -
|
166
|
+
- - ">="
|
160
167
|
- !ruby/object:Gem::Version
|
161
168
|
version: '0'
|
162
169
|
requirements: []
|
163
170
|
rubyforge_project:
|
164
|
-
rubygems_version: 2.
|
171
|
+
rubygems_version: 2.6.8
|
165
172
|
signing_key:
|
166
173
|
specification_version: 4
|
167
174
|
summary: Orc output plugin for Embulk
|