embulk-input-athena 0.1.2 → 0.1.4

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7deb74a8401b375ad2a48e2e0400d34334e0984f
4
- data.tar.gz: ab085cbbd9db5902183e979df7fd88a5111696db
3
+ metadata.gz: fcbb0e1c3535380f35815607c16d771e144766fa
4
+ data.tar.gz: 8d69e048e15ef7f534f20abaeff9e94fb864ed36
5
5
  SHA512:
6
- metadata.gz: 1c048ae148d4c2758986a1705776537646e423b32f26e7c0c07f4de6568403b43108f627e7c530961983a9492d974c6cb2e1d4048d4470c99c20fcdc6f25e592
7
- data.tar.gz: a2f42184c6ac26cc0e9f8d253de0ef7071c017aa45e6654524b5cd0f1c69b020c7ceed3a8685f1cfb868e68ec07bd8af29988c9b7cfb86411bc116105c9c2640
6
+ metadata.gz: abf0271e011bffc189ca6a2e44f71464bd14ed6129e1562381988c1b20440bbc6c14d6e81fd240a09eb3efdb0b9d8b50e85348e648483609be031bd489f17bf6
7
+ data.tar.gz: ece58b6ed17f7fd5c9da23c8528fb6c480e3b46dde0ff7064eb4e2222fe8e01f3f86044d000b744af9e95088324082376a14724736bed62aeb293346cbe13d88
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Athena input plugin for Embulk
2
2
 
3
+ [![Gem Version](https://badge.fury.io/rb/embulk-input-athena.svg)](https://badge.fury.io/rb/embulk-input-athena)
4
+ [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
5
+
3
6
  Athena input plugin for Embulk loads records from Athena(AWS).
4
7
 
5
8
  ## Overview
@@ -12,14 +15,15 @@ Athena input plugin for Embulk loads records from Athena(AWS).
12
15
  ## Configuration
13
16
 
14
17
  * **driver_path**: path to the jar file of the Athena JDBC driver. If not set, the bundled JDBC driver(AthenaJDBC41-1.1.0.jar) will be used. (string)
15
- * **database**: description (string, required)
16
- * **athena_url**: description (string, required)
18
+ * **database**: database name (string, required)
19
+ * **athena_url**: Athena url (string, required)
17
20
  * **s3_staging_dir**: The S3 location to which your query output is written, for example s3://query-results-bucket/folder/. (string, required)
18
- * **access_key**: description (string, required)
19
- * **secret_key**: description (string, required)
20
- * **query**: description (string, required)
21
- * **columns**: description (string, required)
22
- * **options**: description (string, default: {})
21
+ * **access_key**: AWS access key (string, required)
22
+ * **secret_key**: AWS secret key (string, required)
23
+ * **query**: SQL to run (string, required)
24
+ * **columns**: columns (string, required)
25
+ * **options**: extra JDBC properties (string, default: {})
26
+ * **null_to_zero**: if true, convert long, double and boolean value from null to zero (boolean, default: false)
23
27
 
24
28
  ## Example
25
29
 
@@ -36,6 +40,7 @@ in:
36
40
  columns:
37
41
  - {name: uid, type: string}
38
42
  - {name: created_at, type: timestamp}
43
+ null_to_zero: true
39
44
  ```
40
45
 
41
46
  ## Build
@@ -1,6 +1,6 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
3
+ id "com.github.jruby-gradle.base" version "1.5.0"
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  // for task download
@@ -16,7 +16,7 @@ configurations {
16
16
  provided
17
17
  }
18
18
 
19
- version = "0.1.2"
19
+ version = "0.1.4"
20
20
 
21
21
  sourceCompatibility = 1.8
22
22
  targetCompatibility = 1.8
@@ -64,14 +64,16 @@ task checkstyle(type: Checkstyle) {
64
64
  }
65
65
 
66
66
  task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
67
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
68
- script "${project.name}.gemspec"
67
+ jrubyArgs "-S"
68
+ script "gem"
69
+ scriptArgs "build", "${project.name}.gemspec"
69
70
  doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
70
71
  }
71
72
 
72
73
  task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
73
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
74
- script "pkg/${project.name}-${project.version}.gem"
74
+ jrubyArgs "-S"
75
+ script "gem"
76
+ scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
75
77
  }
76
78
 
77
79
  task "package"(dependsOn: ["gemspec", "classpath"]) {
@@ -7,4 +7,10 @@ services:
7
7
  working_dir: /root/embulk-input-athena
8
8
  volumes:
9
9
  - ./:/root/embulk-input-athena
10
- command: tail -f /dev/null
10
+ tty: true
11
+ postgres:
12
+ image: postgres
13
+ ports:
14
+ - 5432:5432
15
+ environment:
16
+ - POSTGRES_PASSWORD=postgres
@@ -1,3 +1,4 @@
1
+ /*
1
2
  package org.embulk.input.athena;
2
3
 
3
4
  import java.util.List;
@@ -48,3 +49,4 @@ public class AthenaInputConnection
48
49
  return new SingleSelect(stmt);
49
50
  }
50
51
  }
52
+ */
@@ -1,5 +1,7 @@
1
1
  package org.embulk.input.athena;
2
2
 
3
+ import com.google.common.base.Optional;
4
+
3
5
  import java.io.File;
4
6
  import java.io.FileFilter;
5
7
  import java.net.MalformedURLException;
@@ -10,14 +12,11 @@ import java.nio.file.Paths;
10
12
  import java.sql.Connection;
11
13
  import java.sql.DriverManager;
12
14
  import java.sql.ResultSet;
13
- import java.sql.ResultSetMetaData;
14
15
  import java.sql.SQLException;
15
16
  import java.sql.Statement;
16
17
  import java.util.List;
17
18
  import java.util.Properties;
18
19
 
19
- import com.google.common.base.Optional;
20
-
21
20
  import org.embulk.config.Config;
22
21
  import org.embulk.config.ConfigDefault;
23
22
  import org.embulk.config.ConfigDiff;
@@ -41,15 +40,16 @@ import org.embulk.spi.SchemaConfig;
41
40
  import org.embulk.spi.time.Timestamp;
42
41
  import org.slf4j.Logger;
43
42
 
44
- public class AthenaInputPlugin implements InputPlugin {
45
-
43
+ public class AthenaInputPlugin implements InputPlugin
44
+ {
46
45
  protected final Logger logger = Exec.getLogger(getClass());
47
46
 
48
- public interface PluginTask extends Task {
47
+ public interface PluginTask extends Task
48
+ {
49
49
  @Config("driver_path")
50
50
  @ConfigDefault("null")
51
- public Optional <String> getDriverPath();
52
-
51
+ public Optional<String> getDriverPath();
52
+
53
53
  // database (required string)
54
54
  @Config("database")
55
55
  public String getDatabase();
@@ -82,12 +82,17 @@ public class AthenaInputPlugin implements InputPlugin {
82
82
  @ConfigDefault("{}")
83
83
  public ToStringMap getOptions();
84
84
 
85
+ @Config("null_to_zero")
86
+ @ConfigDefault("false")
87
+ public boolean getNullToZero();
88
+
85
89
  @ConfigInject
86
90
  BufferAllocator getBufferAllocator();
87
91
  }
88
92
 
89
93
  @Override
90
- public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) {
94
+ public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control)
95
+ {
91
96
  PluginTask task = config.loadConfig(PluginTask.class);
92
97
 
93
98
  Schema schema = task.getColumns().toSchema();
@@ -97,17 +102,20 @@ public class AthenaInputPlugin implements InputPlugin {
97
102
  }
98
103
 
99
104
  @Override
100
- public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control) {
105
+ public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control)
106
+ {
101
107
  control.run(taskSource, schema, taskCount);
102
108
  return Exec.newConfigDiff();
103
109
  }
104
110
 
105
111
  @Override
106
- public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
112
+ public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
113
+ {
107
114
  }
108
115
 
109
116
  @Override
110
- public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
117
+ public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
118
+ {
111
119
  PluginTask task = taskSource.loadTask(PluginTask.class);
112
120
  BufferAllocator allocator = task.getBufferAllocator();
113
121
  PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
@@ -120,58 +128,88 @@ public class AthenaInputPlugin implements InputPlugin {
120
128
  connection = getAthenaConnection(task);
121
129
  statement = connection.createStatement();
122
130
  ResultSet resultSet = statement.executeQuery(task.getQuery());
131
+ boolean nullToZero = task.getNullToZero();
123
132
 
124
- ResultSetMetaData m = resultSet.getMetaData();
125
133
  while (resultSet.next()) {
126
- schema.visitColumns(new ColumnVisitor() {
134
+ schema.visitColumns(new ColumnVisitor()
135
+ {
127
136
  @Override
128
- public void timestampColumn(Column column) {
137
+ public void timestampColumn(Column column)
138
+ {
129
139
  try {
130
140
  java.sql.Timestamp t = resultSet.getTimestamp(column.getName());
131
141
  pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(t.getTime()));
132
- } catch (SQLException e) {
142
+ }
143
+ catch (SQLException e) {
133
144
  e.printStackTrace();
134
145
  }
135
146
  }
136
147
 
137
148
  @Override
138
- public void stringColumn(Column column) {
149
+ public void stringColumn(Column column)
150
+ {
139
151
  try {
140
152
  pageBuilder.setString(column, resultSet.getString(column.getName()));
141
- } catch (SQLException e) {
153
+ }
154
+ catch (SQLException e) {
142
155
  e.printStackTrace();
143
156
  }
144
157
  }
145
158
 
146
159
  @Override
147
- public void longColumn(Column column) {
160
+ public void longColumn(Column column)
161
+ {
148
162
  try {
149
- pageBuilder.setLong(column, resultSet.getLong(column.getName()));
150
- } catch (SQLException e) {
163
+ long ret = resultSet.getLong(column.getName());
164
+ if (resultSet.wasNull() && !nullToZero){
165
+ pageBuilder.setNull(column);
166
+ }
167
+ else {
168
+ pageBuilder.setLong(column, ret);
169
+ }
170
+ }
171
+ catch (SQLException e) {
151
172
  e.printStackTrace();
152
173
  }
153
174
  }
154
175
 
155
176
  @Override
156
- public void doubleColumn(Column column) {
177
+ public void doubleColumn(Column column)
178
+ {
157
179
  try {
158
- pageBuilder.setDouble(column, resultSet.getDouble(column.getName()));
159
- } catch (SQLException e) {
180
+ double ret = resultSet.getDouble(column.getName());
181
+ if (resultSet.wasNull() && !nullToZero){
182
+ pageBuilder.setNull(column);
183
+ }
184
+ else {
185
+ pageBuilder.setDouble(column, ret);
186
+ }
187
+ }
188
+ catch (SQLException e) {
160
189
  e.printStackTrace();
161
190
  }
162
191
  }
163
192
 
164
193
  @Override
165
- public void booleanColumn(Column column) {
194
+ public void booleanColumn(Column column)
195
+ {
166
196
  try {
167
- pageBuilder.setBoolean(column, resultSet.getBoolean(column.getName()));
168
- } catch (SQLException e) {
197
+ boolean ret = resultSet.getBoolean(column.getName());
198
+ if (resultSet.wasNull() && !nullToZero){
199
+ pageBuilder.setNull(column);
200
+ }
201
+ else {
202
+ pageBuilder.setBoolean(column, ret);
203
+ }
204
+ }
205
+ catch (SQLException e) {
169
206
  e.printStackTrace();
170
207
  }
171
208
  }
172
209
 
173
210
  @Override
174
- public void jsonColumn(Column column) {
211
+ public void jsonColumn(Column column)
212
+ {
175
213
  // TODO:
176
214
  }
177
215
  });
@@ -183,19 +221,23 @@ public class AthenaInputPlugin implements InputPlugin {
183
221
  pageBuilder.close();
184
222
  resultSet.close();
185
223
  connection.close();
186
- } catch (Exception e) {
224
+ }
225
+ catch (Exception e) {
187
226
  e.printStackTrace();
188
- } finally {
227
+ }
228
+ finally {
189
229
  try {
190
- if (statement != null)
230
+ if (statement != null) {
191
231
  statement.close();
192
- } catch (Exception ex) {
193
-
232
+ }
194
233
  }
234
+ catch (Exception ex) { }
195
235
  try {
196
- if (connection != null)
236
+ if (connection != null) {
197
237
  connection.close();
198
- } catch (Exception ex) {
238
+ }
239
+ }
240
+ catch (Exception ex) {
199
241
  ex.printStackTrace();
200
242
  }
201
243
  }
@@ -204,11 +246,13 @@ public class AthenaInputPlugin implements InputPlugin {
204
246
  }
205
247
 
206
248
  @Override
207
- public ConfigDiff guess(ConfigSource config) {
249
+ public ConfigDiff guess(ConfigSource config)
250
+ {
208
251
  return Exec.newConfigDiff();
209
252
  }
210
253
 
211
- protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException {
254
+ protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException
255
+ {
212
256
  loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
213
257
  Properties properties = new Properties();
214
258
  properties.put("s3_staging_dir", task.getS3StagingDir());
@@ -227,23 +271,26 @@ public class AthenaInputPlugin implements InputPlugin {
227
271
  {
228
272
  if (driverPath.isPresent()) {
229
273
  addDriverJarToClasspath(driverPath.get());
230
- } else {
274
+ }
275
+ else {
231
276
  try {
232
277
  // Gradle test task will add JDBC driver to classpath
233
278
  Class.forName(className);
234
-
235
- } catch (ClassNotFoundException ex) {
279
+ }
280
+ catch (ClassNotFoundException ex) {
236
281
  File root = findPluginRoot();
237
282
  File driverLib = new File(root, "default_jdbc_driver");
238
283
  File[] files = driverLib.listFiles(new FileFilter() {
239
284
  @Override
240
- public boolean accept(File file) {
285
+ public boolean accept(File file)
286
+ {
241
287
  return file.isFile() && file.getName().endsWith(".jar");
242
288
  }
243
289
  });
244
290
  if (files == null || files.length == 0) {
245
291
  throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
246
- } else {
292
+ }
293
+ else {
247
294
  for (File file : files) {
248
295
  logger.info("JDBC Driver = " + file.getAbsolutePath());
249
296
  addDriverJarToClasspath(file.getAbsolutePath());
@@ -255,7 +302,8 @@ public class AthenaInputPlugin implements InputPlugin {
255
302
  // Load JDBC Driver
256
303
  try {
257
304
  Class.forName(className);
258
- } catch (ClassNotFoundException ex) {
305
+ }
306
+ catch (ClassNotFoundException ex) {
259
307
  throw new RuntimeException(ex);
260
308
  }
261
309
  }
@@ -289,7 +337,8 @@ public class AthenaInputPlugin implements InputPlugin {
289
337
  return folder;
290
338
  }
291
339
  }
292
- } catch (MalformedURLException | URISyntaxException e) {
340
+ }
341
+ catch (MalformedURLException | URISyntaxException e) {
293
342
  throw new RuntimeException(e);
294
343
  }
295
344
  }
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-athena
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - shinji19
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-17 00:00:00.000000000 Z
11
+ date: 2020-08-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ~>
16
+ - - "~>"
17
17
  - !ruby/object:Gem::Version
18
18
  version: '1.0'
19
19
  name: bundler
@@ -21,13 +21,13 @@ dependencies:
21
21
  type: :development
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - '>='
30
+ - - ">="
31
31
  - !ruby/object:Gem::Version
32
32
  version: '10.0'
33
33
  name: rake
@@ -35,7 +35,7 @@ dependencies:
35
35
  type: :development
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '10.0'
41
41
  description: Loads records from Athena.
@@ -45,23 +45,11 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
- - .gitignore
48
+ - ".gitignore"
49
49
  - Dockerfile
50
50
  - LICENSE
51
51
  - README.md
52
52
  - build.gradle
53
- - config/checkstyle/checkstyle.xml
54
- - config/checkstyle/default.xml
55
- - docker-compose.yml
56
- - gradle/wrapper/gradle-wrapper.jar
57
- - gradle/wrapper/gradle-wrapper.properties
58
- - gradlew
59
- - gradlew.bat
60
- - lib/embulk/input/athena.rb
61
- - src/main/java/org/embulk/input/athena/AthenaInputConnection.java
62
- - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
63
- - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
64
- - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
65
53
  - classpath/AthenaJDBC41-1.1.0.jar
66
54
  - classpath/aws-java-sdk-1.11.301.jar
67
55
  - classpath/aws-java-sdk-acm-1.11.301.jar
@@ -192,7 +180,7 @@ files:
192
180
  - classpath/aws-java-sdk-xray-1.11.301.jar
193
181
  - classpath/commons-codec-1.10.jar
194
182
  - classpath/commons-logging-1.2.jar
195
- - classpath/embulk-input-athena-0.1.2.jar
183
+ - classpath/embulk-input-athena-0.1.4.jar
196
184
  - classpath/embulk-input-jdbc-0.9.1.jar
197
185
  - classpath/httpclient-4.5.5.jar
198
186
  - classpath/httpcore-4.4.9.jar
@@ -207,6 +195,17 @@ files:
207
195
  - classpath/netty-handler-4.1.17.Final.jar
208
196
  - classpath/netty-resolver-4.1.17.Final.jar
209
197
  - classpath/netty-transport-4.1.17.Final.jar
198
+ - config/checkstyle/checkstyle.xml
199
+ - config/checkstyle/default.xml
200
+ - docker-compose.yml
201
+ - gradle/wrapper/gradle-wrapper.jar
202
+ - gradle/wrapper/gradle-wrapper.properties
203
+ - gradlew
204
+ - gradlew.bat
205
+ - lib/embulk/input/athena.rb
206
+ - src/main/java/org/embulk/input/athena/AthenaInputConnection.java
207
+ - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
208
+ - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
210
209
  homepage: https://github.com/shinji19/embulk-input-athena
211
210
  licenses:
212
211
  - MIT
@@ -217,17 +216,17 @@ require_paths:
217
216
  - lib
218
217
  required_ruby_version: !ruby/object:Gem::Requirement
219
218
  requirements:
220
- - - '>='
219
+ - - ">="
221
220
  - !ruby/object:Gem::Version
222
221
  version: '0'
223
222
  required_rubygems_version: !ruby/object:Gem::Requirement
224
223
  requirements:
225
- - - '>='
224
+ - - ">="
226
225
  - !ruby/object:Gem::Version
227
226
  version: '0'
228
227
  requirements: []
229
228
  rubyforge_project:
230
- rubygems_version: 2.1.9
229
+ rubygems_version: 2.6.8
231
230
  signing_key:
232
231
  specification_version: 4
233
232
  summary: Athena input plugin for Embulk
@@ -1,192 +0,0 @@
1
- package org.embulk.input.athena;
2
-
3
- import java.sql.Connection;
4
- import java.sql.DriverManager;
5
- import java.sql.ResultSet;
6
- import java.sql.SQLException;
7
- import java.sql.Statement;
8
- import java.util.List;
9
- import java.util.Properties;
10
-
11
- import com.google.common.base.Optional;
12
-
13
- import org.embulk.config.Config;
14
- import org.embulk.config.ConfigDefault;
15
- import org.embulk.config.ConfigDiff;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskReport;
19
- import org.embulk.config.TaskSource;
20
- import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
21
- import org.embulk.input.jdbc.JdbcInputConnection;
22
- import org.embulk.spi.Exec;
23
- import org.embulk.spi.InputPlugin;
24
- import org.embulk.spi.PageOutput;
25
- import org.embulk.spi.Schema;
26
- import org.embulk.spi.SchemaConfig;
27
-
28
- public class AthenaInputPlugin
29
- extends AbstractJdbcInputPlugin
30
- {
31
- public interface AthenaPluginTask
32
- extends AbstractJdbcInputPlugin.PluginTask
33
- {
34
- @Config("driver_path")
35
- @ConfigDefault("null")
36
- public Optional<String> getDriverPath();
37
-
38
- // athena_url (required string)
39
- @Config("athena_url")
40
- public String getAthenaUrl();
41
-
42
- // s3_staging_dir (required string)
43
- @Config("s3_staging_dir")
44
- public String getS3StagingDir();
45
-
46
- // access_key (required string)
47
- @Config("access_key")
48
- public String getAccessKey();
49
-
50
- // secret_key (required string)
51
- @Config("secret_key")
52
- public String getSecretKey();
53
-
54
- // configuration option 2 (optional string, null is not allowed)
55
- // @Config("option2")
56
- // @ConfigDefault("\"myvalue\"")
57
- // public String getOption2();
58
-
59
- // configuration option 3 (optional string, null is allowed)
60
- // @Config("option3")
61
- // @ConfigDefault("null")
62
- // public Optional<String> getOption3();
63
-
64
- // if you get schema from config
65
- // @Config("columns")
66
- // public SchemaConfig getColumns();
67
- }
68
-
69
- @Override
70
- protected Class<? extends PluginTask> getTaskClass() {
71
- return AthenaPluginTask.class;
72
- }
73
-
74
- @Override
75
- protected AthenaInputConnection newConnection(PluginTask pluginTask) throws SQLException {
76
- AthenaPluginTask task = (AthenaPluginTask) pluginTask;
77
- loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
78
- //Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
79
- Properties properties = new Properties();
80
- properties.put("s3_staging_dir", task.getS3StagingDir());
81
- properties.put("user", task.getAccessKey());
82
- properties.put("password", task.getSecretKey());
83
- properties.putAll(task.getOptions());
84
-
85
- Connection connection = DriverManager.getConnection(task.getAthenaUrl(), properties);
86
- try {
87
- AthenaInputConnection c = new AthenaInputConnection(connection);
88
- connection = null;
89
- return c;
90
- } finally {
91
- if (connection != null) {
92
- connection.close();
93
- }
94
- }
95
- }
96
-
97
- /*
98
- @Override
99
- public ConfigDiff transaction(ConfigSource config,
100
- InputPlugin.Control control)
101
- {
102
- PluginTask task = config.loadConfig(PluginTask.class);
103
-
104
- // Schema schema = task.getColumns().toSchema();
105
- Schema schema = Schema.builder().build();
106
- int taskCount = 1; // number of run() method calls
107
-
108
- return resume(task.dump(), schema, taskCount, control);
109
- }
110
-
111
- @Override
112
- public ConfigDiff resume(TaskSource taskSource,
113
- Schema schema, int taskCount,
114
- InputPlugin.Control control)
115
- {
116
- control.run(taskSource, schema, taskCount);
117
- return Exec.newConfigDiff();
118
- }
119
-
120
- @Override
121
- public void cleanup(TaskSource taskSource,
122
- Schema schema, int taskCount,
123
- List<TaskReport> successTaskReports)
124
- {
125
- }
126
-
127
- @Override
128
- public TaskReport run(TaskSource taskSource,
129
- Schema schema, int taskIndex,
130
- PageOutput output)
131
- {
132
- PluginTask task = taskSource.loadTask(PluginTask.class);
133
-
134
- // Write your code here :)
135
-
136
- Connection connection = null;
137
- Statement statement = null;
138
- try {
139
- connection = getAthenaConnection(task);
140
- statement = connection.createStatement();
141
- ResultSet resultSet = statement.executeQuery("select * from default.sample");
142
-
143
- while(resultSet.next()){
144
- String statusCode = resultSet.getString("created_at");
145
- System.out.println("st code" + statusCode);
146
- }
147
- resultSet.close();
148
- connection.close();
149
- } catch (Exception e){
150
- e.printStackTrace();
151
- } finally {
152
- try {
153
- if (statement != null)
154
- statement.close();
155
- } catch (Exception ex) {
156
-
157
- }
158
- try {
159
- if (connection != null)
160
- connection.close();
161
- } catch (Exception ex) {
162
- ex.printStackTrace();
163
- }
164
- }
165
-
166
- return Exec.newTaskReport();
167
- }
168
-
169
- @Override
170
- public ConfigDiff guess(ConfigSource config)
171
- {
172
- return Exec.newConfigDiff();
173
- }
174
- */
175
- /*
176
- protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException{
177
- Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
178
- Properties properties = new Properties();
179
- properties.put("s3_staging_dir", task.getS3StagingDir());
180
- properties.put("user", task.getAccessKey());
181
- properties.put("password", task.getSecretKey());
182
- if (task.getLogPath() != null){
183
- properties.put("log_path", task.getLogPath());
184
- }
185
- if (task.getLogLevel() != null){
186
- properties.put("log_level", task.getLogLevel());
187
- }
188
-
189
- return DriverManager.getConnection(task.getAthenaUrl(), properties);
190
- }
191
- */
192
- }