embulk-input-athena 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -2
- data/build.gradle +14 -9
- data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java +95 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7deb74a8401b375ad2a48e2e0400d34334e0984f
|
4
|
+
data.tar.gz: ab085cbbd9db5902183e979df7fd88a5111696db
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1c048ae148d4c2758986a1705776537646e423b32f26e7c0c07f4de6568403b43108f627e7c530961983a9492d974c6cb2e1d4048d4470c99c20fcdc6f25e592
|
7
|
+
data.tar.gz: a2f42184c6ac26cc0e9f8d253de0ef7071c017aa45e6654524b5cd0f1c69b020c7ceed3a8685f1cfb868e68ec07bd8af29988c9b7cfb86411bc116105c9c2640
|
data/README.md
CHANGED
@@ -11,9 +11,10 @@ Athena input plugin for Embulk loads records from Athena(AWS).
|
|
11
11
|
|
12
12
|
## Configuration
|
13
13
|
|
14
|
+
* **driver_path**: path to the jar file of the Athena JDBC driver. If not set, the bundled JDBC driver(AthenaJDBC41-1.1.0.jar) will be used. (string)
|
14
15
|
* **database**: description (string, required)
|
15
16
|
* **athena_url**: description (string, required)
|
16
|
-
* **s3_staging_dir**:
|
17
|
+
* **s3_staging_dir**: The S3 location to which your query output is written, for example s3://query-results-bucket/folder/. (string, required)
|
17
18
|
* **access_key**: description (string, required)
|
18
19
|
* **secret_key**: description (string, required)
|
19
20
|
* **query**: description (string, required)
|
@@ -39,7 +40,7 @@ in:
|
|
39
40
|
|
40
41
|
## Build
|
41
42
|
|
42
|
-
```
|
43
|
+
```bash
|
43
44
|
$ docker-compose up -d
|
44
45
|
$ docker-compose exec embulk bash
|
45
46
|
embulk>$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
data/build.gradle
CHANGED
@@ -3,20 +3,20 @@ plugins {
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
4
|
id "java"
|
5
5
|
id "checkstyle"
|
6
|
+
// for task download
|
7
|
+
id "de.undercouch.download" version "3.4.2"
|
6
8
|
}
|
7
9
|
import com.github.jrubygradle.JRubyExec
|
8
10
|
repositories {
|
9
11
|
mavenCentral()
|
10
12
|
jcenter()
|
11
|
-
// for athena jdbc
|
12
|
-
maven { url "https://maven.atlassian.com/repository/public" }
|
13
13
|
maven { url "https://dl.bintray.com/embulk-input-jdbc/maven" }
|
14
14
|
}
|
15
15
|
configurations {
|
16
16
|
provided
|
17
17
|
}
|
18
18
|
|
19
|
-
version = "0.1.
|
19
|
+
version = "0.1.2"
|
20
20
|
|
21
21
|
sourceCompatibility = 1.8
|
22
22
|
targetCompatibility = 1.8
|
@@ -24,19 +24,24 @@ targetCompatibility = 1.8
|
|
24
24
|
dependencies {
|
25
25
|
compile "org.embulk:embulk-core:0.8.39"
|
26
26
|
provided "org.embulk:embulk-core:0.8.39"
|
27
|
-
//
|
28
|
-
|
29
|
-
compile group: 'com.amazonaws.athena.jdbc', name: 'AthenaJDBC41', version: '1.0.1-atlassian-hosted'
|
30
|
-
//compile files ('build/AthenaJDBC41-1.1.0.jar')
|
27
|
+
// TODO: maven...
|
28
|
+
compile files ('build/AthenaJDBC41-1.1.0.jar')
|
31
29
|
compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.301'
|
30
|
+
// compile group: 'com.amazonaws', name: 'aws-java-sdk-athena', version: '1.11.301'
|
32
31
|
compile 'org.embulk.input.jdbc:embulk-input-jdbc:0.9.1'
|
33
32
|
testCompile "junit:junit:4.+"
|
34
33
|
}
|
35
34
|
|
36
|
-
task
|
35
|
+
task downloadFile(type: Download) {
|
36
|
+
src 'https://s3.amazonaws.com/athena-downloads/drivers/AthenaJDBC41-1.1.0.jar'
|
37
|
+
dest buildDir
|
38
|
+
onlyIfModified true
|
39
|
+
}
|
40
|
+
|
41
|
+
task classpath(type: Copy, dependsOn: ["downloadFile", "jar"]) {
|
37
42
|
doFirst { file("classpath").deleteDir() }
|
38
43
|
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
39
|
-
|
44
|
+
from ("build/AthenaJDBC41-1.1.0.jar'")
|
40
45
|
into "classpath"
|
41
46
|
}
|
42
47
|
clean { delete "classpath" }
|
@@ -1,5 +1,12 @@
|
|
1
1
|
package org.embulk.input.athena;
|
2
2
|
|
3
|
+
import java.io.File;
|
4
|
+
import java.io.FileFilter;
|
5
|
+
import java.net.MalformedURLException;
|
6
|
+
import java.net.URISyntaxException;
|
7
|
+
import java.net.URL;
|
8
|
+
import java.nio.file.Path;
|
9
|
+
import java.nio.file.Paths;
|
3
10
|
import java.sql.Connection;
|
4
11
|
import java.sql.DriverManager;
|
5
12
|
import java.sql.ResultSet;
|
@@ -9,15 +16,19 @@ import java.sql.Statement;
|
|
9
16
|
import java.util.List;
|
10
17
|
import java.util.Properties;
|
11
18
|
|
19
|
+
import com.google.common.base.Optional;
|
20
|
+
|
12
21
|
import org.embulk.config.Config;
|
13
22
|
import org.embulk.config.ConfigDefault;
|
14
23
|
import org.embulk.config.ConfigDiff;
|
24
|
+
import org.embulk.config.ConfigException;
|
15
25
|
import org.embulk.config.ConfigInject;
|
16
26
|
import org.embulk.config.ConfigSource;
|
17
27
|
import org.embulk.config.Task;
|
18
28
|
import org.embulk.config.TaskReport;
|
19
29
|
import org.embulk.config.TaskSource;
|
20
30
|
import org.embulk.input.jdbc.ToStringMap;
|
31
|
+
import org.embulk.plugin.PluginClassLoader;
|
21
32
|
import org.embulk.spi.BufferAllocator;
|
22
33
|
import org.embulk.spi.Column;
|
23
34
|
import org.embulk.spi.ColumnVisitor;
|
@@ -28,9 +39,17 @@ import org.embulk.spi.PageOutput;
|
|
28
39
|
import org.embulk.spi.Schema;
|
29
40
|
import org.embulk.spi.SchemaConfig;
|
30
41
|
import org.embulk.spi.time.Timestamp;
|
42
|
+
import org.slf4j.Logger;
|
31
43
|
|
32
44
|
public class AthenaInputPlugin implements InputPlugin {
|
45
|
+
|
46
|
+
protected final Logger logger = Exec.getLogger(getClass());
|
47
|
+
|
33
48
|
public interface PluginTask extends Task {
|
49
|
+
@Config("driver_path")
|
50
|
+
@ConfigDefault("null")
|
51
|
+
public Optional <String> getDriverPath();
|
52
|
+
|
34
53
|
// database (required string)
|
35
54
|
@Config("database")
|
36
55
|
public String getDatabase();
|
@@ -190,7 +209,7 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
190
209
|
}
|
191
210
|
|
192
211
|
protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException {
|
193
|
-
|
212
|
+
loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
|
194
213
|
Properties properties = new Properties();
|
195
214
|
properties.put("s3_staging_dir", task.getS3StagingDir());
|
196
215
|
properties.put("user", task.getAccessKey());
|
@@ -199,4 +218,79 @@ public class AthenaInputPlugin implements InputPlugin {
|
|
199
218
|
|
200
219
|
return DriverManager.getConnection(task.getAthenaUrl(), properties);
|
201
220
|
}
|
221
|
+
|
222
|
+
//
|
223
|
+
// copy from embulk-input-jdbc
|
224
|
+
//
|
225
|
+
|
226
|
+
protected void loadDriver(String className, Optional<String> driverPath)
|
227
|
+
{
|
228
|
+
if (driverPath.isPresent()) {
|
229
|
+
addDriverJarToClasspath(driverPath.get());
|
230
|
+
} else {
|
231
|
+
try {
|
232
|
+
// Gradle test task will add JDBC driver to classpath
|
233
|
+
Class.forName(className);
|
234
|
+
|
235
|
+
} catch (ClassNotFoundException ex) {
|
236
|
+
File root = findPluginRoot();
|
237
|
+
File driverLib = new File(root, "default_jdbc_driver");
|
238
|
+
File[] files = driverLib.listFiles(new FileFilter() {
|
239
|
+
@Override
|
240
|
+
public boolean accept(File file) {
|
241
|
+
return file.isFile() && file.getName().endsWith(".jar");
|
242
|
+
}
|
243
|
+
});
|
244
|
+
if (files == null || files.length == 0) {
|
245
|
+
throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
|
246
|
+
} else {
|
247
|
+
for (File file : files) {
|
248
|
+
logger.info("JDBC Driver = " + file.getAbsolutePath());
|
249
|
+
addDriverJarToClasspath(file.getAbsolutePath());
|
250
|
+
}
|
251
|
+
}
|
252
|
+
}
|
253
|
+
}
|
254
|
+
|
255
|
+
// Load JDBC Driver
|
256
|
+
try {
|
257
|
+
Class.forName(className);
|
258
|
+
} catch (ClassNotFoundException ex) {
|
259
|
+
throw new RuntimeException(ex);
|
260
|
+
}
|
261
|
+
}
|
262
|
+
|
263
|
+
protected void addDriverJarToClasspath(String glob)
|
264
|
+
{
|
265
|
+
// TODO match glob
|
266
|
+
PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
|
267
|
+
Path path = Paths.get(glob);
|
268
|
+
if (!path.toFile().exists()) {
|
269
|
+
throw new ConfigException("The specified driver jar doesn't exist: " + glob);
|
270
|
+
}
|
271
|
+
loader.addPath(Paths.get(glob));
|
272
|
+
}
|
273
|
+
|
274
|
+
protected File findPluginRoot()
|
275
|
+
{
|
276
|
+
try {
|
277
|
+
URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
|
278
|
+
if (url.toString().startsWith("jar:")) {
|
279
|
+
url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
|
280
|
+
}
|
281
|
+
|
282
|
+
File folder = new File(url.toURI()).getParentFile();
|
283
|
+
for (;; folder = folder.getParentFile()) {
|
284
|
+
if (folder == null) {
|
285
|
+
throw new RuntimeException("Cannot find 'embulk-input-xxx' folder.");
|
286
|
+
}
|
287
|
+
|
288
|
+
if (folder.getName().startsWith("embulk-input-")) {
|
289
|
+
return folder;
|
290
|
+
}
|
291
|
+
}
|
292
|
+
} catch (MalformedURLException | URISyntaxException e) {
|
293
|
+
throw new RuntimeException(e);
|
294
|
+
}
|
295
|
+
}
|
202
296
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-athena
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- shinji19
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-04-
|
11
|
+
date: 2018-04-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -62,7 +62,7 @@ files:
|
|
62
62
|
- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
|
63
63
|
- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
|
64
64
|
- src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
|
65
|
-
- classpath/AthenaJDBC41-1.0.
|
65
|
+
- classpath/AthenaJDBC41-1.1.0.jar
|
66
66
|
- classpath/aws-java-sdk-1.11.301.jar
|
67
67
|
- classpath/aws-java-sdk-acm-1.11.301.jar
|
68
68
|
- classpath/aws-java-sdk-alexaforbusiness-1.11.301.jar
|
@@ -192,7 +192,7 @@ files:
|
|
192
192
|
- classpath/aws-java-sdk-xray-1.11.301.jar
|
193
193
|
- classpath/commons-codec-1.10.jar
|
194
194
|
- classpath/commons-logging-1.2.jar
|
195
|
-
- classpath/embulk-input-athena-0.1.
|
195
|
+
- classpath/embulk-input-athena-0.1.2.jar
|
196
196
|
- classpath/embulk-input-jdbc-0.9.1.jar
|
197
197
|
- classpath/httpclient-4.5.5.jar
|
198
198
|
- classpath/httpcore-4.4.9.jar
|