embulk-input-athena 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad4b3382cf370da50743fdef54e85cbaf9ee3c6b
-  data.tar.gz: 8efd48a28b3dc003aff73c4253abccccdc465df9
+  metadata.gz: 7deb74a8401b375ad2a48e2e0400d34334e0984f
+  data.tar.gz: ab085cbbd9db5902183e979df7fd88a5111696db
 SHA512:
-  metadata.gz: b8dcea7434f50c191421d6bc79d91ab0c0725ead169e86fb40846906e243657cf722d01bba4f9437dfe8e1d40ca901da97583696f42ac8cce268a904038fd1a4
-  data.tar.gz: 69d636d9f53a252db12ff0106e1dfa9b3cfcb00f7c8de005f70137007f850272b54519154dd4db5cf6f3bd35b327dcfddf421389854e6256d1d4aad0358b54f8
+  metadata.gz: 1c048ae148d4c2758986a1705776537646e423b32f26e7c0c07f4de6568403b43108f627e7c530961983a9492d974c6cb2e1d4048d4470c99c20fcdc6f25e592
+  data.tar.gz: a2f42184c6ac26cc0e9f8d253de0ef7071c017aa45e6654524b5cd0f1c69b020c7ceed3a8685f1cfb868e68ec07bd8af29988c9b7cfb86411bc116105c9c2640

data/README.md

@@ -11,9 +11,10 @@ Athena input plugin for Embulk loads records from Athena(AWS).
 
 ## Configuration
 
+* **driver_path**: path to the jar file of the Athena JDBC driver. If not set, the bundled JDBC driver(AthenaJDBC41-1.1.0.jar) will be used. (string)
 * **database**: description (string, required)
 * **athena_url**: description (string, required)
-* **s3_staging_dir**: description (string, required)
+* **s3_staging_dir**: The S3 location to which your query output is written, for example s3://query-results-bucket/folder/. (string, required)
 * **access_key**: description (string, required)
 * **secret_key**: description (string, required)
 * **query**: description (string, required)
@@ -39,7 +40,7 @@ in:
 
 ## Build
 
-```
+```bash
 $ docker-compose up -d
 $ docker-compose exec embulk bash
 embulk>$ ./gradlew gem  # -t to watch change of files and rebuild continuously

data/build.gradle

@@ -3,20 +3,20 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    // for task download
+    id "de.undercouch.download" version "3.4.2"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
-    // for athena jdbc
-    maven { url "https://maven.atlassian.com/repository/public" }
     maven { url "https://dl.bintray.com/embulk-input-jdbc/maven" }
 }
 configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -24,19 +24,24 @@ targetCompatibility = 1.8
 dependencies {
     compile  "org.embulk:embulk-core:0.8.39"
     provided "org.embulk:embulk-core:0.8.39"
-    // https://mvnrepository.com/artifact/com.amazonaws.athena.jdbc/AthenaJDBC41
-    // TODO: update jdbc
-    compile group: 'com.amazonaws.athena.jdbc', name: 'AthenaJDBC41', version: '1.0.1-atlassian-hosted'
-    //compile files ('build/AthenaJDBC41-1.1.0.jar')
+    // TODO: maven...
+    compile files ('build/AthenaJDBC41-1.1.0.jar')
     compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.301'
+    // compile group: 'com.amazonaws', name: 'aws-java-sdk-athena', version: '1.11.301'
     compile 'org.embulk.input.jdbc:embulk-input-jdbc:0.9.1'
     testCompile "junit:junit:4.+"
 }
 
-task classpath(type: Copy, dependsOn: ["jar"]) {
+task downloadFile(type: Download) {
+    src 'https://s3.amazonaws.com/athena-downloads/drivers/AthenaJDBC41-1.1.0.jar'
+    dest buildDir
+    onlyIfModified true
+}
+
+task classpath(type: Copy, dependsOn: ["downloadFile", "jar"]) {
     doFirst { file("classpath").deleteDir() }
     from (configurations.runtime - configurations.provided + files(jar.archivePath))
-    // from ("build/AthenaJDBC41-1.1.0.jar'")
+    from ("build/AthenaJDBC41-1.1.0.jar'")
     into "classpath"
 }
 clean { delete "classpath" }

data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java

@@ -1,5 +1,12 @@
 package org.embulk.input.athena;
 
+import java.io.File;
+import java.io.FileFilter;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.ResultSet;
@@ -9,15 +16,19 @@ import java.sql.Statement;
 import java.util.List;
 import java.util.Properties;
 
+import com.google.common.base.Optional;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.input.jdbc.ToStringMap;
+import org.embulk.plugin.PluginClassLoader;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
@@ -28,9 +39,17 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
 
 public class AthenaInputPlugin implements InputPlugin {
+
+    protected final Logger logger = Exec.getLogger(getClass());
+
     public interface PluginTask extends Task {
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional <String> getDriverPath();
+        
         // database (required string)
         @Config("database")
         public String getDatabase();
@@ -190,7 +209,7 @@ public class AthenaInputPlugin implements InputPlugin {
     }
 
     protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException {
-        Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
+        loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
         Properties properties = new Properties();
         properties.put("s3_staging_dir", task.getS3StagingDir());
         properties.put("user", task.getAccessKey());
@@ -199,4 +218,79 @@ public class AthenaInputPlugin implements InputPlugin {
 
         return DriverManager.getConnection(task.getAthenaUrl(), properties);
     }
+
+    //
+    // copy from embulk-input-jdbc
+    //
+
+    protected void loadDriver(String className, Optional<String> driverPath)
+    {
+        if (driverPath.isPresent()) {
+            addDriverJarToClasspath(driverPath.get());
+        } else {
+            try {
+                // Gradle test task will add JDBC driver to classpath
+                Class.forName(className);
+
+            } catch (ClassNotFoundException ex) {
+                File root = findPluginRoot();
+                File driverLib = new File(root, "default_jdbc_driver");
+                File[] files = driverLib.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(File file) {
+                        return file.isFile() && file.getName().endsWith(".jar");
+                    }
+                });
+                if (files == null || files.length == 0) {
+                    throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
+                } else {
+                    for (File file : files) {
+                        logger.info("JDBC Driver = " + file.getAbsolutePath());
+                        addDriverJarToClasspath(file.getAbsolutePath());
+                    }
+                }
+            }
+        }
+
+        // Load JDBC Driver
+        try {
+            Class.forName(className);
+        } catch (ClassNotFoundException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    protected void addDriverJarToClasspath(String glob)
+    {
+        // TODO match glob
+        PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+             throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
+        loader.addPath(Paths.get(glob));
+    }
+
+    protected File findPluginRoot()
+    {
+        try {
+            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    throw new RuntimeException("Cannot find 'embulk-input-xxx' folder.");
+                }
+
+                if (folder.getName().startsWith("embulk-input-")) {
+                    return folder;
+                }
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-athena
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - shinji19
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-16 00:00:00.000000000 Z
+date: 2018-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ files:
 - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
 - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
 - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
-- classpath/AthenaJDBC41-1.0.1-atlassian-hosted.jar
+- classpath/AthenaJDBC41-1.1.0.jar
 - classpath/aws-java-sdk-1.11.301.jar
 - classpath/aws-java-sdk-acm-1.11.301.jar
 - classpath/aws-java-sdk-alexaforbusiness-1.11.301.jar
@@ -192,7 +192,7 @@ files:
 - classpath/aws-java-sdk-xray-1.11.301.jar
 - classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-athena-0.1.1.jar
+- classpath/embulk-input-athena-0.1.2.jar
 - classpath/embulk-input-jdbc-0.9.1.jar
 - classpath/httpclient-4.5.5.jar
 - classpath/httpcore-4.4.9.jar