embulk-input-athena 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +12 -7
- data/build.gradle +8 -6
- data/docker-compose.yml +7 -1
- data/src/main/java/org/embulk/input/athena/AthenaInputConnection.java +2 -0
- data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java +93 -44
- metadata +22 -23
- data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1 +0 -192
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fcbb0e1c3535380f35815607c16d771e144766fa
|
|
4
|
+
data.tar.gz: 8d69e048e15ef7f534f20abaeff9e94fb864ed36
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: abf0271e011bffc189ca6a2e44f71464bd14ed6129e1562381988c1b20440bbc6c14d6e81fd240a09eb3efdb0b9d8b50e85348e648483609be031bd489f17bf6
|
|
7
|
+
data.tar.gz: ece58b6ed17f7fd5c9da23c8528fb6c480e3b46dde0ff7064eb4e2222fe8e01f3f86044d000b744af9e95088324082376a14724736bed62aeb293346cbe13d88
|
data/README.md
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
# Athena input plugin for Embulk
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/embulk-input-athena)
|
|
4
|
+
[](LICENSE)
|
|
5
|
+
|
|
3
6
|
Athena input plugin for Embulk loads records from Athena(AWS).
|
|
4
7
|
|
|
5
8
|
## Overview
|
|
@@ -12,14 +15,15 @@ Athena input plugin for Embulk loads records from Athena(AWS).
|
|
|
12
15
|
## Configuration
|
|
13
16
|
|
|
14
17
|
* **driver_path**: path to the jar file of the Athena JDBC driver. If not set, the bundled JDBC driver(AthenaJDBC41-1.1.0.jar) will be used. (string)
|
|
15
|
-
* **database**:
|
|
16
|
-
* **athena_url**:
|
|
18
|
+
* **database**: database name (string, required)
|
|
19
|
+
* **athena_url**: Athena url (string, required)
|
|
17
20
|
* **s3_staging_dir**: The S3 location to which your query output is written, for example s3://query-results-bucket/folder/. (string, required)
|
|
18
|
-
* **access_key**:
|
|
19
|
-
* **secret_key**:
|
|
20
|
-
* **query**:
|
|
21
|
-
* **columns**:
|
|
22
|
-
* **options**:
|
|
21
|
+
* **access_key**: AWS access key (string, required)
|
|
22
|
+
* **secret_key**: AWS secret key (string, required)
|
|
23
|
+
* **query**: SQL to run (string, required)
|
|
24
|
+
* **columns**: columns (string, required)
|
|
25
|
+
* **options**: extra JDBC properties (string, default: {})
|
|
26
|
+
* **null_to_zero**: if true, convert long, double and boolean value from null to zero (boolean, default: false)
|
|
23
27
|
|
|
24
28
|
## Example
|
|
25
29
|
|
|
@@ -36,6 +40,7 @@ in:
|
|
|
36
40
|
columns:
|
|
37
41
|
- {name: uid, type: string}
|
|
38
42
|
- {name: created_at, type: timestamp}
|
|
43
|
+
null_to_zero: true
|
|
39
44
|
```
|
|
40
45
|
|
|
41
46
|
## Build
|
data/build.gradle
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
plugins {
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
|
-
id "com.github.jruby-gradle.base" version "
|
|
3
|
+
id "com.github.jruby-gradle.base" version "1.5.0"
|
|
4
4
|
id "java"
|
|
5
5
|
id "checkstyle"
|
|
6
6
|
// for task download
|
|
@@ -16,7 +16,7 @@ configurations {
|
|
|
16
16
|
provided
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
version = "0.1.
|
|
19
|
+
version = "0.1.4"
|
|
20
20
|
|
|
21
21
|
sourceCompatibility = 1.8
|
|
22
22
|
targetCompatibility = 1.8
|
|
@@ -64,14 +64,16 @@ task checkstyle(type: Checkstyle) {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
67
|
-
jrubyArgs "-
|
|
68
|
-
script "
|
|
67
|
+
jrubyArgs "-S"
|
|
68
|
+
script "gem"
|
|
69
|
+
scriptArgs "build", "${project.name}.gemspec"
|
|
69
70
|
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
|
|
73
|
-
jrubyArgs "-
|
|
74
|
-
script "
|
|
74
|
+
jrubyArgs "-S"
|
|
75
|
+
script "gem"
|
|
76
|
+
scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
|
|
75
77
|
}
|
|
76
78
|
|
|
77
79
|
task "package"(dependsOn: ["gemspec", "classpath"]) {
|
data/docker-compose.yml
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
package org.embulk.input.athena;
|
|
2
2
|
|
|
3
|
+
import com.google.common.base.Optional;
|
|
4
|
+
|
|
3
5
|
import java.io.File;
|
|
4
6
|
import java.io.FileFilter;
|
|
5
7
|
import java.net.MalformedURLException;
|
|
@@ -10,14 +12,11 @@ import java.nio.file.Paths;
|
|
|
10
12
|
import java.sql.Connection;
|
|
11
13
|
import java.sql.DriverManager;
|
|
12
14
|
import java.sql.ResultSet;
|
|
13
|
-
import java.sql.ResultSetMetaData;
|
|
14
15
|
import java.sql.SQLException;
|
|
15
16
|
import java.sql.Statement;
|
|
16
17
|
import java.util.List;
|
|
17
18
|
import java.util.Properties;
|
|
18
19
|
|
|
19
|
-
import com.google.common.base.Optional;
|
|
20
|
-
|
|
21
20
|
import org.embulk.config.Config;
|
|
22
21
|
import org.embulk.config.ConfigDefault;
|
|
23
22
|
import org.embulk.config.ConfigDiff;
|
|
@@ -41,15 +40,16 @@ import org.embulk.spi.SchemaConfig;
|
|
|
41
40
|
import org.embulk.spi.time.Timestamp;
|
|
42
41
|
import org.slf4j.Logger;
|
|
43
42
|
|
|
44
|
-
public class AthenaInputPlugin implements InputPlugin
|
|
45
|
-
|
|
43
|
+
public class AthenaInputPlugin implements InputPlugin
|
|
44
|
+
{
|
|
46
45
|
protected final Logger logger = Exec.getLogger(getClass());
|
|
47
46
|
|
|
48
|
-
public interface PluginTask extends Task
|
|
47
|
+
public interface PluginTask extends Task
|
|
48
|
+
{
|
|
49
49
|
@Config("driver_path")
|
|
50
50
|
@ConfigDefault("null")
|
|
51
|
-
public Optional
|
|
52
|
-
|
|
51
|
+
public Optional<String> getDriverPath();
|
|
52
|
+
|
|
53
53
|
// database (required string)
|
|
54
54
|
@Config("database")
|
|
55
55
|
public String getDatabase();
|
|
@@ -82,12 +82,17 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
82
82
|
@ConfigDefault("{}")
|
|
83
83
|
public ToStringMap getOptions();
|
|
84
84
|
|
|
85
|
+
@Config("null_to_zero")
|
|
86
|
+
@ConfigDefault("false")
|
|
87
|
+
public boolean getNullToZero();
|
|
88
|
+
|
|
85
89
|
@ConfigInject
|
|
86
90
|
BufferAllocator getBufferAllocator();
|
|
87
91
|
}
|
|
88
92
|
|
|
89
93
|
@Override
|
|
90
|
-
public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control)
|
|
94
|
+
public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control)
|
|
95
|
+
{
|
|
91
96
|
PluginTask task = config.loadConfig(PluginTask.class);
|
|
92
97
|
|
|
93
98
|
Schema schema = task.getColumns().toSchema();
|
|
@@ -97,17 +102,20 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
97
102
|
}
|
|
98
103
|
|
|
99
104
|
@Override
|
|
100
|
-
public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control)
|
|
105
|
+
public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control)
|
|
106
|
+
{
|
|
101
107
|
control.run(taskSource, schema, taskCount);
|
|
102
108
|
return Exec.newConfigDiff();
|
|
103
109
|
}
|
|
104
110
|
|
|
105
111
|
@Override
|
|
106
|
-
public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
|
|
112
|
+
public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
|
|
113
|
+
{
|
|
107
114
|
}
|
|
108
115
|
|
|
109
116
|
@Override
|
|
110
|
-
public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
|
|
117
|
+
public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
|
|
118
|
+
{
|
|
111
119
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
112
120
|
BufferAllocator allocator = task.getBufferAllocator();
|
|
113
121
|
PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
|
|
@@ -120,58 +128,88 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
120
128
|
connection = getAthenaConnection(task);
|
|
121
129
|
statement = connection.createStatement();
|
|
122
130
|
ResultSet resultSet = statement.executeQuery(task.getQuery());
|
|
131
|
+
boolean nullToZero = task.getNullToZero();
|
|
123
132
|
|
|
124
|
-
ResultSetMetaData m = resultSet.getMetaData();
|
|
125
133
|
while (resultSet.next()) {
|
|
126
|
-
schema.visitColumns(new ColumnVisitor()
|
|
134
|
+
schema.visitColumns(new ColumnVisitor()
|
|
135
|
+
{
|
|
127
136
|
@Override
|
|
128
|
-
public void timestampColumn(Column column)
|
|
137
|
+
public void timestampColumn(Column column)
|
|
138
|
+
{
|
|
129
139
|
try {
|
|
130
140
|
java.sql.Timestamp t = resultSet.getTimestamp(column.getName());
|
|
131
141
|
pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(t.getTime()));
|
|
132
|
-
}
|
|
142
|
+
}
|
|
143
|
+
catch (SQLException e) {
|
|
133
144
|
e.printStackTrace();
|
|
134
145
|
}
|
|
135
146
|
}
|
|
136
147
|
|
|
137
148
|
@Override
|
|
138
|
-
public void stringColumn(Column column)
|
|
149
|
+
public void stringColumn(Column column)
|
|
150
|
+
{
|
|
139
151
|
try {
|
|
140
152
|
pageBuilder.setString(column, resultSet.getString(column.getName()));
|
|
141
|
-
}
|
|
153
|
+
}
|
|
154
|
+
catch (SQLException e) {
|
|
142
155
|
e.printStackTrace();
|
|
143
156
|
}
|
|
144
157
|
}
|
|
145
158
|
|
|
146
159
|
@Override
|
|
147
|
-
public void longColumn(Column column)
|
|
160
|
+
public void longColumn(Column column)
|
|
161
|
+
{
|
|
148
162
|
try {
|
|
149
|
-
|
|
150
|
-
|
|
163
|
+
long ret = resultSet.getLong(column.getName());
|
|
164
|
+
if (resultSet.wasNull() && !nullToZero){
|
|
165
|
+
pageBuilder.setNull(column);
|
|
166
|
+
}
|
|
167
|
+
else {
|
|
168
|
+
pageBuilder.setLong(column, ret);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
catch (SQLException e) {
|
|
151
172
|
e.printStackTrace();
|
|
152
173
|
}
|
|
153
174
|
}
|
|
154
175
|
|
|
155
176
|
@Override
|
|
156
|
-
public void doubleColumn(Column column)
|
|
177
|
+
public void doubleColumn(Column column)
|
|
178
|
+
{
|
|
157
179
|
try {
|
|
158
|
-
|
|
159
|
-
|
|
180
|
+
double ret = resultSet.getDouble(column.getName());
|
|
181
|
+
if (resultSet.wasNull() && !nullToZero){
|
|
182
|
+
pageBuilder.setNull(column);
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
pageBuilder.setDouble(column, ret);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
catch (SQLException e) {
|
|
160
189
|
e.printStackTrace();
|
|
161
190
|
}
|
|
162
191
|
}
|
|
163
192
|
|
|
164
193
|
@Override
|
|
165
|
-
public void booleanColumn(Column column)
|
|
194
|
+
public void booleanColumn(Column column)
|
|
195
|
+
{
|
|
166
196
|
try {
|
|
167
|
-
|
|
168
|
-
|
|
197
|
+
boolean ret = resultSet.getBoolean(column.getName());
|
|
198
|
+
if (resultSet.wasNull() && !nullToZero){
|
|
199
|
+
pageBuilder.setNull(column);
|
|
200
|
+
}
|
|
201
|
+
else {
|
|
202
|
+
pageBuilder.setBoolean(column, ret);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch (SQLException e) {
|
|
169
206
|
e.printStackTrace();
|
|
170
207
|
}
|
|
171
208
|
}
|
|
172
209
|
|
|
173
210
|
@Override
|
|
174
|
-
public void jsonColumn(Column column)
|
|
211
|
+
public void jsonColumn(Column column)
|
|
212
|
+
{
|
|
175
213
|
// TODO:
|
|
176
214
|
}
|
|
177
215
|
});
|
|
@@ -183,19 +221,23 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
183
221
|
pageBuilder.close();
|
|
184
222
|
resultSet.close();
|
|
185
223
|
connection.close();
|
|
186
|
-
}
|
|
224
|
+
}
|
|
225
|
+
catch (Exception e) {
|
|
187
226
|
e.printStackTrace();
|
|
188
|
-
}
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
189
229
|
try {
|
|
190
|
-
if (statement != null)
|
|
230
|
+
if (statement != null) {
|
|
191
231
|
statement.close();
|
|
192
|
-
|
|
193
|
-
|
|
232
|
+
}
|
|
194
233
|
}
|
|
234
|
+
catch (Exception ex) { }
|
|
195
235
|
try {
|
|
196
|
-
if (connection != null)
|
|
236
|
+
if (connection != null) {
|
|
197
237
|
connection.close();
|
|
198
|
-
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
catch (Exception ex) {
|
|
199
241
|
ex.printStackTrace();
|
|
200
242
|
}
|
|
201
243
|
}
|
|
@@ -204,11 +246,13 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
204
246
|
}
|
|
205
247
|
|
|
206
248
|
@Override
|
|
207
|
-
public ConfigDiff guess(ConfigSource config)
|
|
249
|
+
public ConfigDiff guess(ConfigSource config)
|
|
250
|
+
{
|
|
208
251
|
return Exec.newConfigDiff();
|
|
209
252
|
}
|
|
210
253
|
|
|
211
|
-
protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException
|
|
254
|
+
protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException
|
|
255
|
+
{
|
|
212
256
|
loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
|
|
213
257
|
Properties properties = new Properties();
|
|
214
258
|
properties.put("s3_staging_dir", task.getS3StagingDir());
|
|
@@ -227,23 +271,26 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
227
271
|
{
|
|
228
272
|
if (driverPath.isPresent()) {
|
|
229
273
|
addDriverJarToClasspath(driverPath.get());
|
|
230
|
-
}
|
|
274
|
+
}
|
|
275
|
+
else {
|
|
231
276
|
try {
|
|
232
277
|
// Gradle test task will add JDBC driver to classpath
|
|
233
278
|
Class.forName(className);
|
|
234
|
-
|
|
235
|
-
|
|
279
|
+
}
|
|
280
|
+
catch (ClassNotFoundException ex) {
|
|
236
281
|
File root = findPluginRoot();
|
|
237
282
|
File driverLib = new File(root, "default_jdbc_driver");
|
|
238
283
|
File[] files = driverLib.listFiles(new FileFilter() {
|
|
239
284
|
@Override
|
|
240
|
-
public boolean accept(File file)
|
|
285
|
+
public boolean accept(File file)
|
|
286
|
+
{
|
|
241
287
|
return file.isFile() && file.getName().endsWith(".jar");
|
|
242
288
|
}
|
|
243
289
|
});
|
|
244
290
|
if (files == null || files.length == 0) {
|
|
245
291
|
throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
|
|
246
|
-
}
|
|
292
|
+
}
|
|
293
|
+
else {
|
|
247
294
|
for (File file : files) {
|
|
248
295
|
logger.info("JDBC Driver = " + file.getAbsolutePath());
|
|
249
296
|
addDriverJarToClasspath(file.getAbsolutePath());
|
|
@@ -255,7 +302,8 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
255
302
|
// Load JDBC Driver
|
|
256
303
|
try {
|
|
257
304
|
Class.forName(className);
|
|
258
|
-
}
|
|
305
|
+
}
|
|
306
|
+
catch (ClassNotFoundException ex) {
|
|
259
307
|
throw new RuntimeException(ex);
|
|
260
308
|
}
|
|
261
309
|
}
|
|
@@ -289,7 +337,8 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
|
289
337
|
return folder;
|
|
290
338
|
}
|
|
291
339
|
}
|
|
292
|
-
}
|
|
340
|
+
}
|
|
341
|
+
catch (MalformedURLException | URISyntaxException e) {
|
|
293
342
|
throw new RuntimeException(e);
|
|
294
343
|
}
|
|
295
344
|
}
|
metadata
CHANGED
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-athena
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.4
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- shinji19
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-08-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
15
15
|
requirements:
|
|
16
|
-
- - ~>
|
|
16
|
+
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
18
|
version: '1.0'
|
|
19
19
|
name: bundler
|
|
@@ -21,13 +21,13 @@ dependencies:
|
|
|
21
21
|
type: :development
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
|
-
- - ~>
|
|
24
|
+
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
26
|
version: '1.0'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
29
29
|
requirements:
|
|
30
|
-
- -
|
|
30
|
+
- - ">="
|
|
31
31
|
- !ruby/object:Gem::Version
|
|
32
32
|
version: '10.0'
|
|
33
33
|
name: rake
|
|
@@ -35,7 +35,7 @@ dependencies:
|
|
|
35
35
|
type: :development
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
|
-
- -
|
|
38
|
+
- - ">="
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
40
|
version: '10.0'
|
|
41
41
|
description: Loads records from Athena.
|
|
@@ -45,23 +45,11 @@ executables: []
|
|
|
45
45
|
extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
|
47
47
|
files:
|
|
48
|
-
- .gitignore
|
|
48
|
+
- ".gitignore"
|
|
49
49
|
- Dockerfile
|
|
50
50
|
- LICENSE
|
|
51
51
|
- README.md
|
|
52
52
|
- build.gradle
|
|
53
|
-
- config/checkstyle/checkstyle.xml
|
|
54
|
-
- config/checkstyle/default.xml
|
|
55
|
-
- docker-compose.yml
|
|
56
|
-
- gradle/wrapper/gradle-wrapper.jar
|
|
57
|
-
- gradle/wrapper/gradle-wrapper.properties
|
|
58
|
-
- gradlew
|
|
59
|
-
- gradlew.bat
|
|
60
|
-
- lib/embulk/input/athena.rb
|
|
61
|
-
- src/main/java/org/embulk/input/athena/AthenaInputConnection.java
|
|
62
|
-
- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
|
|
63
|
-
- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
|
|
64
|
-
- src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
|
|
65
53
|
- classpath/AthenaJDBC41-1.1.0.jar
|
|
66
54
|
- classpath/aws-java-sdk-1.11.301.jar
|
|
67
55
|
- classpath/aws-java-sdk-acm-1.11.301.jar
|
|
@@ -192,7 +180,7 @@ files:
|
|
|
192
180
|
- classpath/aws-java-sdk-xray-1.11.301.jar
|
|
193
181
|
- classpath/commons-codec-1.10.jar
|
|
194
182
|
- classpath/commons-logging-1.2.jar
|
|
195
|
-
- classpath/embulk-input-athena-0.1.
|
|
183
|
+
- classpath/embulk-input-athena-0.1.4.jar
|
|
196
184
|
- classpath/embulk-input-jdbc-0.9.1.jar
|
|
197
185
|
- classpath/httpclient-4.5.5.jar
|
|
198
186
|
- classpath/httpcore-4.4.9.jar
|
|
@@ -207,6 +195,17 @@ files:
|
|
|
207
195
|
- classpath/netty-handler-4.1.17.Final.jar
|
|
208
196
|
- classpath/netty-resolver-4.1.17.Final.jar
|
|
209
197
|
- classpath/netty-transport-4.1.17.Final.jar
|
|
198
|
+
- config/checkstyle/checkstyle.xml
|
|
199
|
+
- config/checkstyle/default.xml
|
|
200
|
+
- docker-compose.yml
|
|
201
|
+
- gradle/wrapper/gradle-wrapper.jar
|
|
202
|
+
- gradle/wrapper/gradle-wrapper.properties
|
|
203
|
+
- gradlew
|
|
204
|
+
- gradlew.bat
|
|
205
|
+
- lib/embulk/input/athena.rb
|
|
206
|
+
- src/main/java/org/embulk/input/athena/AthenaInputConnection.java
|
|
207
|
+
- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
|
|
208
|
+
- src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
|
|
210
209
|
homepage: https://github.com/shinji19/embulk-input-athena
|
|
211
210
|
licenses:
|
|
212
211
|
- MIT
|
|
@@ -217,17 +216,17 @@ require_paths:
|
|
|
217
216
|
- lib
|
|
218
217
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
219
218
|
requirements:
|
|
220
|
-
- -
|
|
219
|
+
- - ">="
|
|
221
220
|
- !ruby/object:Gem::Version
|
|
222
221
|
version: '0'
|
|
223
222
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
224
223
|
requirements:
|
|
225
|
-
- -
|
|
224
|
+
- - ">="
|
|
226
225
|
- !ruby/object:Gem::Version
|
|
227
226
|
version: '0'
|
|
228
227
|
requirements: []
|
|
229
228
|
rubyforge_project:
|
|
230
|
-
rubygems_version: 2.
|
|
229
|
+
rubygems_version: 2.6.8
|
|
231
230
|
signing_key:
|
|
232
231
|
specification_version: 4
|
|
233
232
|
summary: Athena input plugin for Embulk
|
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
package org.embulk.input.athena;
|
|
2
|
-
|
|
3
|
-
import java.sql.Connection;
|
|
4
|
-
import java.sql.DriverManager;
|
|
5
|
-
import java.sql.ResultSet;
|
|
6
|
-
import java.sql.SQLException;
|
|
7
|
-
import java.sql.Statement;
|
|
8
|
-
import java.util.List;
|
|
9
|
-
import java.util.Properties;
|
|
10
|
-
|
|
11
|
-
import com.google.common.base.Optional;
|
|
12
|
-
|
|
13
|
-
import org.embulk.config.Config;
|
|
14
|
-
import org.embulk.config.ConfigDefault;
|
|
15
|
-
import org.embulk.config.ConfigDiff;
|
|
16
|
-
import org.embulk.config.ConfigSource;
|
|
17
|
-
import org.embulk.config.Task;
|
|
18
|
-
import org.embulk.config.TaskReport;
|
|
19
|
-
import org.embulk.config.TaskSource;
|
|
20
|
-
import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
|
|
21
|
-
import org.embulk.input.jdbc.JdbcInputConnection;
|
|
22
|
-
import org.embulk.spi.Exec;
|
|
23
|
-
import org.embulk.spi.InputPlugin;
|
|
24
|
-
import org.embulk.spi.PageOutput;
|
|
25
|
-
import org.embulk.spi.Schema;
|
|
26
|
-
import org.embulk.spi.SchemaConfig;
|
|
27
|
-
|
|
28
|
-
public class AthenaInputPlugin
|
|
29
|
-
extends AbstractJdbcInputPlugin
|
|
30
|
-
{
|
|
31
|
-
public interface AthenaPluginTask
|
|
32
|
-
extends AbstractJdbcInputPlugin.PluginTask
|
|
33
|
-
{
|
|
34
|
-
@Config("driver_path")
|
|
35
|
-
@ConfigDefault("null")
|
|
36
|
-
public Optional<String> getDriverPath();
|
|
37
|
-
|
|
38
|
-
// athena_url (required string)
|
|
39
|
-
@Config("athena_url")
|
|
40
|
-
public String getAthenaUrl();
|
|
41
|
-
|
|
42
|
-
// s3_staging_dir (required string)
|
|
43
|
-
@Config("s3_staging_dir")
|
|
44
|
-
public String getS3StagingDir();
|
|
45
|
-
|
|
46
|
-
// access_key (required string)
|
|
47
|
-
@Config("access_key")
|
|
48
|
-
public String getAccessKey();
|
|
49
|
-
|
|
50
|
-
// secret_key (required string)
|
|
51
|
-
@Config("secret_key")
|
|
52
|
-
public String getSecretKey();
|
|
53
|
-
|
|
54
|
-
// configuration option 2 (optional string, null is not allowed)
|
|
55
|
-
// @Config("option2")
|
|
56
|
-
// @ConfigDefault("\"myvalue\"")
|
|
57
|
-
// public String getOption2();
|
|
58
|
-
|
|
59
|
-
// configuration option 3 (optional string, null is allowed)
|
|
60
|
-
// @Config("option3")
|
|
61
|
-
// @ConfigDefault("null")
|
|
62
|
-
// public Optional<String> getOption3();
|
|
63
|
-
|
|
64
|
-
// if you get schema from config
|
|
65
|
-
// @Config("columns")
|
|
66
|
-
// public SchemaConfig getColumns();
|
|
67
|
-
}
|
|
68
|
-
|
|
69
|
-
@Override
|
|
70
|
-
protected Class<? extends PluginTask> getTaskClass() {
|
|
71
|
-
return AthenaPluginTask.class;
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
@Override
|
|
75
|
-
protected AthenaInputConnection newConnection(PluginTask pluginTask) throws SQLException {
|
|
76
|
-
AthenaPluginTask task = (AthenaPluginTask) pluginTask;
|
|
77
|
-
loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
|
|
78
|
-
//Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
|
|
79
|
-
Properties properties = new Properties();
|
|
80
|
-
properties.put("s3_staging_dir", task.getS3StagingDir());
|
|
81
|
-
properties.put("user", task.getAccessKey());
|
|
82
|
-
properties.put("password", task.getSecretKey());
|
|
83
|
-
properties.putAll(task.getOptions());
|
|
84
|
-
|
|
85
|
-
Connection connection = DriverManager.getConnection(task.getAthenaUrl(), properties);
|
|
86
|
-
try {
|
|
87
|
-
AthenaInputConnection c = new AthenaInputConnection(connection);
|
|
88
|
-
connection = null;
|
|
89
|
-
return c;
|
|
90
|
-
} finally {
|
|
91
|
-
if (connection != null) {
|
|
92
|
-
connection.close();
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
/*
|
|
98
|
-
@Override
|
|
99
|
-
public ConfigDiff transaction(ConfigSource config,
|
|
100
|
-
InputPlugin.Control control)
|
|
101
|
-
{
|
|
102
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
|
103
|
-
|
|
104
|
-
// Schema schema = task.getColumns().toSchema();
|
|
105
|
-
Schema schema = Schema.builder().build();
|
|
106
|
-
int taskCount = 1; // number of run() method calls
|
|
107
|
-
|
|
108
|
-
return resume(task.dump(), schema, taskCount, control);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
@Override
|
|
112
|
-
public ConfigDiff resume(TaskSource taskSource,
|
|
113
|
-
Schema schema, int taskCount,
|
|
114
|
-
InputPlugin.Control control)
|
|
115
|
-
{
|
|
116
|
-
control.run(taskSource, schema, taskCount);
|
|
117
|
-
return Exec.newConfigDiff();
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
@Override
|
|
121
|
-
public void cleanup(TaskSource taskSource,
|
|
122
|
-
Schema schema, int taskCount,
|
|
123
|
-
List<TaskReport> successTaskReports)
|
|
124
|
-
{
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
@Override
|
|
128
|
-
public TaskReport run(TaskSource taskSource,
|
|
129
|
-
Schema schema, int taskIndex,
|
|
130
|
-
PageOutput output)
|
|
131
|
-
{
|
|
132
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
133
|
-
|
|
134
|
-
// Write your code here :)
|
|
135
|
-
|
|
136
|
-
Connection connection = null;
|
|
137
|
-
Statement statement = null;
|
|
138
|
-
try {
|
|
139
|
-
connection = getAthenaConnection(task);
|
|
140
|
-
statement = connection.createStatement();
|
|
141
|
-
ResultSet resultSet = statement.executeQuery("select * from default.sample");
|
|
142
|
-
|
|
143
|
-
while(resultSet.next()){
|
|
144
|
-
String statusCode = resultSet.getString("created_at");
|
|
145
|
-
System.out.println("st code" + statusCode);
|
|
146
|
-
}
|
|
147
|
-
resultSet.close();
|
|
148
|
-
connection.close();
|
|
149
|
-
} catch (Exception e){
|
|
150
|
-
e.printStackTrace();
|
|
151
|
-
} finally {
|
|
152
|
-
try {
|
|
153
|
-
if (statement != null)
|
|
154
|
-
statement.close();
|
|
155
|
-
} catch (Exception ex) {
|
|
156
|
-
|
|
157
|
-
}
|
|
158
|
-
try {
|
|
159
|
-
if (connection != null)
|
|
160
|
-
connection.close();
|
|
161
|
-
} catch (Exception ex) {
|
|
162
|
-
ex.printStackTrace();
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
return Exec.newTaskReport();
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
@Override
|
|
170
|
-
public ConfigDiff guess(ConfigSource config)
|
|
171
|
-
{
|
|
172
|
-
return Exec.newConfigDiff();
|
|
173
|
-
}
|
|
174
|
-
*/
|
|
175
|
-
/*
|
|
176
|
-
protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException{
|
|
177
|
-
Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
|
|
178
|
-
Properties properties = new Properties();
|
|
179
|
-
properties.put("s3_staging_dir", task.getS3StagingDir());
|
|
180
|
-
properties.put("user", task.getAccessKey());
|
|
181
|
-
properties.put("password", task.getSecretKey());
|
|
182
|
-
if (task.getLogPath() != null){
|
|
183
|
-
properties.put("log_path", task.getLogPath());
|
|
184
|
-
}
|
|
185
|
-
if (task.getLogLevel() != null){
|
|
186
|
-
properties.put("log_level", task.getLogLevel());
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
return DriverManager.getConnection(task.getAthenaUrl(), properties);
|
|
190
|
-
}
|
|
191
|
-
*/
|
|
192
|
-
}
|