embulk-input-athena 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7deb74a8401b375ad2a48e2e0400d34334e0984f
4
- data.tar.gz: ab085cbbd9db5902183e979df7fd88a5111696db
3
+ metadata.gz: fcbb0e1c3535380f35815607c16d771e144766fa
4
+ data.tar.gz: 8d69e048e15ef7f534f20abaeff9e94fb864ed36
5
5
  SHA512:
6
- metadata.gz: 1c048ae148d4c2758986a1705776537646e423b32f26e7c0c07f4de6568403b43108f627e7c530961983a9492d974c6cb2e1d4048d4470c99c20fcdc6f25e592
7
- data.tar.gz: a2f42184c6ac26cc0e9f8d253de0ef7071c017aa45e6654524b5cd0f1c69b020c7ceed3a8685f1cfb868e68ec07bd8af29988c9b7cfb86411bc116105c9c2640
6
+ metadata.gz: abf0271e011bffc189ca6a2e44f71464bd14ed6129e1562381988c1b20440bbc6c14d6e81fd240a09eb3efdb0b9d8b50e85348e648483609be031bd489f17bf6
7
+ data.tar.gz: ece58b6ed17f7fd5c9da23c8528fb6c480e3b46dde0ff7064eb4e2222fe8e01f3f86044d000b744af9e95088324082376a14724736bed62aeb293346cbe13d88
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Athena input plugin for Embulk
2
2
 
3
+ [![Gem Version](https://badge.fury.io/rb/embulk-input-athena.svg)](https://badge.fury.io/rb/embulk-input-athena)
4
+ [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE)
5
+
3
6
  Athena input plugin for Embulk loads records from Athena(AWS).
4
7
 
5
8
  ## Overview
@@ -12,14 +15,15 @@ Athena input plugin for Embulk loads records from Athena(AWS).
12
15
  ## Configuration
13
16
 
14
17
  * **driver_path**: path to the jar file of the Athena JDBC driver. If not set, the bundled JDBC driver(AthenaJDBC41-1.1.0.jar) will be used. (string)
15
- * **database**: description (string, required)
16
- * **athena_url**: description (string, required)
18
+ * **database**: database name (string, required)
19
+ * **athena_url**: Athena url (string, required)
17
20
  * **s3_staging_dir**: The S3 location to which your query output is written, for example s3://query-results-bucket/folder/. (string, required)
18
- * **access_key**: description (string, required)
19
- * **secret_key**: description (string, required)
20
- * **query**: description (string, required)
21
- * **columns**: description (string, required)
22
- * **options**: description (string, default: {})
21
+ * **access_key**: AWS access key (string, required)
22
+ * **secret_key**: AWS secret key (string, required)
23
+ * **query**: SQL to run (string, required)
24
+ * **columns**: columns (string, required)
25
+ * **options**: extra JDBC properties (string, default: {})
26
+ * **null_to_zero**: if true, convert long, double and boolean value from null to zero (boolean, default: false)
23
27
 
24
28
  ## Example
25
29
 
@@ -36,6 +40,7 @@ in:
36
40
  columns:
37
41
  - {name: uid, type: string}
38
42
  - {name: created_at, type: timestamp}
43
+ null_to_zero: true
39
44
  ```
40
45
 
41
46
  ## Build
@@ -1,6 +1,6 @@
1
1
  plugins {
2
2
  id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
3
+ id "com.github.jruby-gradle.base" version "1.5.0"
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  // for task download
@@ -16,7 +16,7 @@ configurations {
16
16
  provided
17
17
  }
18
18
 
19
- version = "0.1.2"
19
+ version = "0.1.4"
20
20
 
21
21
  sourceCompatibility = 1.8
22
22
  targetCompatibility = 1.8
@@ -64,14 +64,16 @@ task checkstyle(type: Checkstyle) {
64
64
  }
65
65
 
66
66
  task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
67
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
68
- script "${project.name}.gemspec"
67
+ jrubyArgs "-S"
68
+ script "gem"
69
+ scriptArgs "build", "${project.name}.gemspec"
69
70
  doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
70
71
  }
71
72
 
72
73
  task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
73
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
74
- script "pkg/${project.name}-${project.version}.gem"
74
+ jrubyArgs "-S"
75
+ script "gem"
76
+ scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
75
77
  }
76
78
 
77
79
  task "package"(dependsOn: ["gemspec", "classpath"]) {
@@ -7,4 +7,10 @@ services:
7
7
  working_dir: /root/embulk-input-athena
8
8
  volumes:
9
9
  - ./:/root/embulk-input-athena
10
- command: tail -f /dev/null
10
+ tty: true
11
+ postgres:
12
+ image: postgres
13
+ ports:
14
+ - 5432:5432
15
+ environment:
16
+ - POSTGRES_PASSWORD=postgres
@@ -1,3 +1,4 @@
1
+ /*
1
2
  package org.embulk.input.athena;
2
3
 
3
4
  import java.util.List;
@@ -48,3 +49,4 @@ public class AthenaInputConnection
48
49
  return new SingleSelect(stmt);
49
50
  }
50
51
  }
52
+ */
@@ -1,5 +1,7 @@
1
1
  package org.embulk.input.athena;
2
2
 
3
+ import com.google.common.base.Optional;
4
+
3
5
  import java.io.File;
4
6
  import java.io.FileFilter;
5
7
  import java.net.MalformedURLException;
@@ -10,14 +12,11 @@ import java.nio.file.Paths;
10
12
  import java.sql.Connection;
11
13
  import java.sql.DriverManager;
12
14
  import java.sql.ResultSet;
13
- import java.sql.ResultSetMetaData;
14
15
  import java.sql.SQLException;
15
16
  import java.sql.Statement;
16
17
  import java.util.List;
17
18
  import java.util.Properties;
18
19
 
19
- import com.google.common.base.Optional;
20
-
21
20
  import org.embulk.config.Config;
22
21
  import org.embulk.config.ConfigDefault;
23
22
  import org.embulk.config.ConfigDiff;
@@ -41,15 +40,16 @@ import org.embulk.spi.SchemaConfig;
41
40
  import org.embulk.spi.time.Timestamp;
42
41
  import org.slf4j.Logger;
43
42
 
44
- public class AthenaInputPlugin implements InputPlugin {
45
-
43
+ public class AthenaInputPlugin implements InputPlugin
44
+ {
46
45
  protected final Logger logger = Exec.getLogger(getClass());
47
46
 
48
- public interface PluginTask extends Task {
47
+ public interface PluginTask extends Task
48
+ {
49
49
  @Config("driver_path")
50
50
  @ConfigDefault("null")
51
- public Optional <String> getDriverPath();
52
-
51
+ public Optional<String> getDriverPath();
52
+
53
53
  // database (required string)
54
54
  @Config("database")
55
55
  public String getDatabase();
@@ -82,12 +82,17 @@ public class AthenaInputPlugin implements InputPlugin {
82
82
  @ConfigDefault("{}")
83
83
  public ToStringMap getOptions();
84
84
 
85
+ @Config("null_to_zero")
86
+ @ConfigDefault("false")
87
+ public boolean getNullToZero();
88
+
85
89
  @ConfigInject
86
90
  BufferAllocator getBufferAllocator();
87
91
  }
88
92
 
89
93
  @Override
90
- public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control) {
94
+ public ConfigDiff transaction(ConfigSource config, InputPlugin.Control control)
95
+ {
91
96
  PluginTask task = config.loadConfig(PluginTask.class);
92
97
 
93
98
  Schema schema = task.getColumns().toSchema();
@@ -97,17 +102,20 @@ public class AthenaInputPlugin implements InputPlugin {
97
102
  }
98
103
 
99
104
  @Override
100
- public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control) {
105
+ public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, InputPlugin.Control control)
106
+ {
101
107
  control.run(taskSource, schema, taskCount);
102
108
  return Exec.newConfigDiff();
103
109
  }
104
110
 
105
111
  @Override
106
- public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
112
+ public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
113
+ {
107
114
  }
108
115
 
109
116
  @Override
110
- public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output) {
117
+ public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
118
+ {
111
119
  PluginTask task = taskSource.loadTask(PluginTask.class);
112
120
  BufferAllocator allocator = task.getBufferAllocator();
113
121
  PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
@@ -120,58 +128,88 @@ public class AthenaInputPlugin implements InputPlugin {
120
128
  connection = getAthenaConnection(task);
121
129
  statement = connection.createStatement();
122
130
  ResultSet resultSet = statement.executeQuery(task.getQuery());
131
+ boolean nullToZero = task.getNullToZero();
123
132
 
124
- ResultSetMetaData m = resultSet.getMetaData();
125
133
  while (resultSet.next()) {
126
- schema.visitColumns(new ColumnVisitor() {
134
+ schema.visitColumns(new ColumnVisitor()
135
+ {
127
136
  @Override
128
- public void timestampColumn(Column column) {
137
+ public void timestampColumn(Column column)
138
+ {
129
139
  try {
130
140
  java.sql.Timestamp t = resultSet.getTimestamp(column.getName());
131
141
  pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(t.getTime()));
132
- } catch (SQLException e) {
142
+ }
143
+ catch (SQLException e) {
133
144
  e.printStackTrace();
134
145
  }
135
146
  }
136
147
 
137
148
  @Override
138
- public void stringColumn(Column column) {
149
+ public void stringColumn(Column column)
150
+ {
139
151
  try {
140
152
  pageBuilder.setString(column, resultSet.getString(column.getName()));
141
- } catch (SQLException e) {
153
+ }
154
+ catch (SQLException e) {
142
155
  e.printStackTrace();
143
156
  }
144
157
  }
145
158
 
146
159
  @Override
147
- public void longColumn(Column column) {
160
+ public void longColumn(Column column)
161
+ {
148
162
  try {
149
- pageBuilder.setLong(column, resultSet.getLong(column.getName()));
150
- } catch (SQLException e) {
163
+ long ret = resultSet.getLong(column.getName());
164
+ if (resultSet.wasNull() && !nullToZero){
165
+ pageBuilder.setNull(column);
166
+ }
167
+ else {
168
+ pageBuilder.setLong(column, ret);
169
+ }
170
+ }
171
+ catch (SQLException e) {
151
172
  e.printStackTrace();
152
173
  }
153
174
  }
154
175
 
155
176
  @Override
156
- public void doubleColumn(Column column) {
177
+ public void doubleColumn(Column column)
178
+ {
157
179
  try {
158
- pageBuilder.setDouble(column, resultSet.getDouble(column.getName()));
159
- } catch (SQLException e) {
180
+ double ret = resultSet.getDouble(column.getName());
181
+ if (resultSet.wasNull() && !nullToZero){
182
+ pageBuilder.setNull(column);
183
+ }
184
+ else {
185
+ pageBuilder.setDouble(column, ret);
186
+ }
187
+ }
188
+ catch (SQLException e) {
160
189
  e.printStackTrace();
161
190
  }
162
191
  }
163
192
 
164
193
  @Override
165
- public void booleanColumn(Column column) {
194
+ public void booleanColumn(Column column)
195
+ {
166
196
  try {
167
- pageBuilder.setBoolean(column, resultSet.getBoolean(column.getName()));
168
- } catch (SQLException e) {
197
+ boolean ret = resultSet.getBoolean(column.getName());
198
+ if (resultSet.wasNull() && !nullToZero){
199
+ pageBuilder.setNull(column);
200
+ }
201
+ else {
202
+ pageBuilder.setBoolean(column, ret);
203
+ }
204
+ }
205
+ catch (SQLException e) {
169
206
  e.printStackTrace();
170
207
  }
171
208
  }
172
209
 
173
210
  @Override
174
- public void jsonColumn(Column column) {
211
+ public void jsonColumn(Column column)
212
+ {
175
213
  // TODO:
176
214
  }
177
215
  });
@@ -183,19 +221,23 @@ public class AthenaInputPlugin implements InputPlugin {
183
221
  pageBuilder.close();
184
222
  resultSet.close();
185
223
  connection.close();
186
- } catch (Exception e) {
224
+ }
225
+ catch (Exception e) {
187
226
  e.printStackTrace();
188
- } finally {
227
+ }
228
+ finally {
189
229
  try {
190
- if (statement != null)
230
+ if (statement != null) {
191
231
  statement.close();
192
- } catch (Exception ex) {
193
-
232
+ }
194
233
  }
234
+ catch (Exception ex) { }
195
235
  try {
196
- if (connection != null)
236
+ if (connection != null) {
197
237
  connection.close();
198
- } catch (Exception ex) {
238
+ }
239
+ }
240
+ catch (Exception ex) {
199
241
  ex.printStackTrace();
200
242
  }
201
243
  }
@@ -204,11 +246,13 @@ public class AthenaInputPlugin implements InputPlugin {
204
246
  }
205
247
 
206
248
  @Override
207
- public ConfigDiff guess(ConfigSource config) {
249
+ public ConfigDiff guess(ConfigSource config)
250
+ {
208
251
  return Exec.newConfigDiff();
209
252
  }
210
253
 
211
- protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException {
254
+ protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException
255
+ {
212
256
  loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
213
257
  Properties properties = new Properties();
214
258
  properties.put("s3_staging_dir", task.getS3StagingDir());
@@ -227,23 +271,26 @@ public class AthenaInputPlugin implements InputPlugin {
227
271
  {
228
272
  if (driverPath.isPresent()) {
229
273
  addDriverJarToClasspath(driverPath.get());
230
- } else {
274
+ }
275
+ else {
231
276
  try {
232
277
  // Gradle test task will add JDBC driver to classpath
233
278
  Class.forName(className);
234
-
235
- } catch (ClassNotFoundException ex) {
279
+ }
280
+ catch (ClassNotFoundException ex) {
236
281
  File root = findPluginRoot();
237
282
  File driverLib = new File(root, "default_jdbc_driver");
238
283
  File[] files = driverLib.listFiles(new FileFilter() {
239
284
  @Override
240
- public boolean accept(File file) {
285
+ public boolean accept(File file)
286
+ {
241
287
  return file.isFile() && file.getName().endsWith(".jar");
242
288
  }
243
289
  });
244
290
  if (files == null || files.length == 0) {
245
291
  throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
246
- } else {
292
+ }
293
+ else {
247
294
  for (File file : files) {
248
295
  logger.info("JDBC Driver = " + file.getAbsolutePath());
249
296
  addDriverJarToClasspath(file.getAbsolutePath());
@@ -255,7 +302,8 @@ public class AthenaInputPlugin implements InputPlugin {
255
302
  // Load JDBC Driver
256
303
  try {
257
304
  Class.forName(className);
258
- } catch (ClassNotFoundException ex) {
305
+ }
306
+ catch (ClassNotFoundException ex) {
259
307
  throw new RuntimeException(ex);
260
308
  }
261
309
  }
@@ -289,7 +337,8 @@ public class AthenaInputPlugin implements InputPlugin {
289
337
  return folder;
290
338
  }
291
339
  }
292
- } catch (MalformedURLException | URISyntaxException e) {
340
+ }
341
+ catch (MalformedURLException | URISyntaxException e) {
293
342
  throw new RuntimeException(e);
294
343
  }
295
344
  }
metadata CHANGED
@@ -1,19 +1,19 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-athena
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - shinji19
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-04-17 00:00:00.000000000 Z
11
+ date: 2020-08-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
- - - ~>
16
+ - - "~>"
17
17
  - !ruby/object:Gem::Version
18
18
  version: '1.0'
19
19
  name: bundler
@@ -21,13 +21,13 @@ dependencies:
21
21
  type: :development
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  requirement: !ruby/object:Gem::Requirement
29
29
  requirements:
30
- - - '>='
30
+ - - ">="
31
31
  - !ruby/object:Gem::Version
32
32
  version: '10.0'
33
33
  name: rake
@@ -35,7 +35,7 @@ dependencies:
35
35
  type: :development
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '>='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '10.0'
41
41
  description: Loads records from Athena.
@@ -45,23 +45,11 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
- - .gitignore
48
+ - ".gitignore"
49
49
  - Dockerfile
50
50
  - LICENSE
51
51
  - README.md
52
52
  - build.gradle
53
- - config/checkstyle/checkstyle.xml
54
- - config/checkstyle/default.xml
55
- - docker-compose.yml
56
- - gradle/wrapper/gradle-wrapper.jar
57
- - gradle/wrapper/gradle-wrapper.properties
58
- - gradlew
59
- - gradlew.bat
60
- - lib/embulk/input/athena.rb
61
- - src/main/java/org/embulk/input/athena/AthenaInputConnection.java
62
- - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
63
- - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
64
- - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
65
53
  - classpath/AthenaJDBC41-1.1.0.jar
66
54
  - classpath/aws-java-sdk-1.11.301.jar
67
55
  - classpath/aws-java-sdk-acm-1.11.301.jar
@@ -192,7 +180,7 @@ files:
192
180
  - classpath/aws-java-sdk-xray-1.11.301.jar
193
181
  - classpath/commons-codec-1.10.jar
194
182
  - classpath/commons-logging-1.2.jar
195
- - classpath/embulk-input-athena-0.1.2.jar
183
+ - classpath/embulk-input-athena-0.1.4.jar
196
184
  - classpath/embulk-input-jdbc-0.9.1.jar
197
185
  - classpath/httpclient-4.5.5.jar
198
186
  - classpath/httpcore-4.4.9.jar
@@ -207,6 +195,17 @@ files:
207
195
  - classpath/netty-handler-4.1.17.Final.jar
208
196
  - classpath/netty-resolver-4.1.17.Final.jar
209
197
  - classpath/netty-transport-4.1.17.Final.jar
198
+ - config/checkstyle/checkstyle.xml
199
+ - config/checkstyle/default.xml
200
+ - docker-compose.yml
201
+ - gradle/wrapper/gradle-wrapper.jar
202
+ - gradle/wrapper/gradle-wrapper.properties
203
+ - gradlew
204
+ - gradlew.bat
205
+ - lib/embulk/input/athena.rb
206
+ - src/main/java/org/embulk/input/athena/AthenaInputConnection.java
207
+ - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
208
+ - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
210
209
  homepage: https://github.com/shinji19/embulk-input-athena
211
210
  licenses:
212
211
  - MIT
@@ -217,17 +216,17 @@ require_paths:
217
216
  - lib
218
217
  required_ruby_version: !ruby/object:Gem::Requirement
219
218
  requirements:
220
- - - '>='
219
+ - - ">="
221
220
  - !ruby/object:Gem::Version
222
221
  version: '0'
223
222
  required_rubygems_version: !ruby/object:Gem::Requirement
224
223
  requirements:
225
- - - '>='
224
+ - - ">="
226
225
  - !ruby/object:Gem::Version
227
226
  version: '0'
228
227
  requirements: []
229
228
  rubyforge_project:
230
- rubygems_version: 2.1.9
229
+ rubygems_version: 2.6.8
231
230
  signing_key:
232
231
  specification_version: 4
233
232
  summary: Athena input plugin for Embulk
@@ -1,192 +0,0 @@
1
- package org.embulk.input.athena;
2
-
3
- import java.sql.Connection;
4
- import java.sql.DriverManager;
5
- import java.sql.ResultSet;
6
- import java.sql.SQLException;
7
- import java.sql.Statement;
8
- import java.util.List;
9
- import java.util.Properties;
10
-
11
- import com.google.common.base.Optional;
12
-
13
- import org.embulk.config.Config;
14
- import org.embulk.config.ConfigDefault;
15
- import org.embulk.config.ConfigDiff;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskReport;
19
- import org.embulk.config.TaskSource;
20
- import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
21
- import org.embulk.input.jdbc.JdbcInputConnection;
22
- import org.embulk.spi.Exec;
23
- import org.embulk.spi.InputPlugin;
24
- import org.embulk.spi.PageOutput;
25
- import org.embulk.spi.Schema;
26
- import org.embulk.spi.SchemaConfig;
27
-
28
- public class AthenaInputPlugin
29
- extends AbstractJdbcInputPlugin
30
- {
31
- public interface AthenaPluginTask
32
- extends AbstractJdbcInputPlugin.PluginTask
33
- {
34
- @Config("driver_path")
35
- @ConfigDefault("null")
36
- public Optional<String> getDriverPath();
37
-
38
- // athena_url (required string)
39
- @Config("athena_url")
40
- public String getAthenaUrl();
41
-
42
- // s3_staging_dir (required string)
43
- @Config("s3_staging_dir")
44
- public String getS3StagingDir();
45
-
46
- // access_key (required string)
47
- @Config("access_key")
48
- public String getAccessKey();
49
-
50
- // secret_key (required string)
51
- @Config("secret_key")
52
- public String getSecretKey();
53
-
54
- // configuration option 2 (optional string, null is not allowed)
55
- // @Config("option2")
56
- // @ConfigDefault("\"myvalue\"")
57
- // public String getOption2();
58
-
59
- // configuration option 3 (optional string, null is allowed)
60
- // @Config("option3")
61
- // @ConfigDefault("null")
62
- // public Optional<String> getOption3();
63
-
64
- // if you get schema from config
65
- // @Config("columns")
66
- // public SchemaConfig getColumns();
67
- }
68
-
69
- @Override
70
- protected Class<? extends PluginTask> getTaskClass() {
71
- return AthenaPluginTask.class;
72
- }
73
-
74
- @Override
75
- protected AthenaInputConnection newConnection(PluginTask pluginTask) throws SQLException {
76
- AthenaPluginTask task = (AthenaPluginTask) pluginTask;
77
- loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
78
- //Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
79
- Properties properties = new Properties();
80
- properties.put("s3_staging_dir", task.getS3StagingDir());
81
- properties.put("user", task.getAccessKey());
82
- properties.put("password", task.getSecretKey());
83
- properties.putAll(task.getOptions());
84
-
85
- Connection connection = DriverManager.getConnection(task.getAthenaUrl(), properties);
86
- try {
87
- AthenaInputConnection c = new AthenaInputConnection(connection);
88
- connection = null;
89
- return c;
90
- } finally {
91
- if (connection != null) {
92
- connection.close();
93
- }
94
- }
95
- }
96
-
97
- /*
98
- @Override
99
- public ConfigDiff transaction(ConfigSource config,
100
- InputPlugin.Control control)
101
- {
102
- PluginTask task = config.loadConfig(PluginTask.class);
103
-
104
- // Schema schema = task.getColumns().toSchema();
105
- Schema schema = Schema.builder().build();
106
- int taskCount = 1; // number of run() method calls
107
-
108
- return resume(task.dump(), schema, taskCount, control);
109
- }
110
-
111
- @Override
112
- public ConfigDiff resume(TaskSource taskSource,
113
- Schema schema, int taskCount,
114
- InputPlugin.Control control)
115
- {
116
- control.run(taskSource, schema, taskCount);
117
- return Exec.newConfigDiff();
118
- }
119
-
120
- @Override
121
- public void cleanup(TaskSource taskSource,
122
- Schema schema, int taskCount,
123
- List<TaskReport> successTaskReports)
124
- {
125
- }
126
-
127
- @Override
128
- public TaskReport run(TaskSource taskSource,
129
- Schema schema, int taskIndex,
130
- PageOutput output)
131
- {
132
- PluginTask task = taskSource.loadTask(PluginTask.class);
133
-
134
- // Write your code here :)
135
-
136
- Connection connection = null;
137
- Statement statement = null;
138
- try {
139
- connection = getAthenaConnection(task);
140
- statement = connection.createStatement();
141
- ResultSet resultSet = statement.executeQuery("select * from default.sample");
142
-
143
- while(resultSet.next()){
144
- String statusCode = resultSet.getString("created_at");
145
- System.out.println("st code" + statusCode);
146
- }
147
- resultSet.close();
148
- connection.close();
149
- } catch (Exception e){
150
- e.printStackTrace();
151
- } finally {
152
- try {
153
- if (statement != null)
154
- statement.close();
155
- } catch (Exception ex) {
156
-
157
- }
158
- try {
159
- if (connection != null)
160
- connection.close();
161
- } catch (Exception ex) {
162
- ex.printStackTrace();
163
- }
164
- }
165
-
166
- return Exec.newTaskReport();
167
- }
168
-
169
- @Override
170
- public ConfigDiff guess(ConfigSource config)
171
- {
172
- return Exec.newConfigDiff();
173
- }
174
- */
175
- /*
176
- protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException{
177
- Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
178
- Properties properties = new Properties();
179
- properties.put("s3_staging_dir", task.getS3StagingDir());
180
- properties.put("user", task.getAccessKey());
181
- properties.put("password", task.getSecretKey());
182
- if (task.getLogPath() != null){
183
- properties.put("log_path", task.getLogPath());
184
- }
185
- if (task.getLogLevel() != null){
186
- properties.put("log_level", task.getLogLevel());
187
- }
188
-
189
- return DriverManager.getConnection(task.getAthenaUrl(), properties);
190
- }
191
- */
192
- }