embulk-filter-mysql_lookup 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 986a3e18c5daf7eeda74e5d4f1b32a047cdf0b817317ce43c66db91a932ea7e6
-  data.tar.gz: 7e6dea6395cdcd22e93d950a2c50b9f38eb955285680c2be2e10576208f5e49b
+  metadata.gz: 58c954c9c0b2d3f57d21464af114e2f115068901d8c5a6a1f52171337e58fdb7
+  data.tar.gz: 8de014de954b493160c205f3b5d5a5fb657011bf332b20717e3d354612495c73
 SHA512:
-  metadata.gz: 8e23daedc484493ebfb0c75a97ac731893a84db3b397587f626c2e4b2553230c86e9896c3e3e4ef6ba0aee8738a5a1ebb031af093023c71e9e5d8dd30854e82a
-  data.tar.gz: 3e688f47128c722364d080f4993a67ce52531654ed0898690374bf064f979607da10a788cde5012145a45469f031e6a97d928c17a6dd3ce029f97aa3c0d24788
+  metadata.gz: 23c6c2974484352b108c942e453b51d856a779e809bdf4e39407f16b3b33e72e2f1009cd2bf99fabd16c5cf891c1cf11dd7f0c1e6e5c94a0253ae06cd842e985
+  data.tar.gz: 6ae8efc216663bf2be139ee48f79535b3ad426c9aa781d189ba8d436592002f2522e5c8349aafa8a4d330cfc4feac06e6e30fae3b93e1915f53fa009120c7c6d

data/README.md

@@ -13,6 +13,10 @@ An Embulk filter plugin for Lookup Transformation with MySQL database
 ## Configuration
 ​
 - **mysql_lookup**: Required attributes for the LookUp Filter Plugin -
+    - **filters**:
+        - **type**: Name of lookup type (required)
+    - **driver_path**: path to the jar file of the MySQL JDBC driver. If not set, the bundled JDBC driver (MySQL Connector/J 8.0.19) will be used (string)
+    - **driver_class**: Here we can provide driver class name,if not set,the com.mysql.cj.jdbc.Driver class will be used
     - **host**: database host (example `localhost`) (required)
     - **port**: database port (example port for mssql `1433`) (required
     - **database**: database name (required)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/config1.yml

@@ -0,0 +1,47 @@
+exec:
+  max_threads: 2
+  min_output_tasks: 1
+in:
+  type: file
+  path_prefix: C:\Users\AnkitKumar\Desktop\embulk-filter-mysqllookup\calendar.csv
+  parser:
+    type: csv
+    columns:
+      - { name: dim_calendar_key, type: long }
+      - { name: year_number, type: long }
+      - { name: quarter_number, type: long }
+      - { name: attr_1, type: string }
+filters:
+  - type: mysql_lookup
+    host: localhost
+    port: 3306
+    driver_path: C:/Users/AnkitKumar/Desktop/embulk-plugins/mysql-connector-java-8.0.19.jar
+    driver_class: com.mysql.cj.jdbc.Driver
+    database: information
+    table_name: country
+    username: root
+    password: root
+    mapping_from:
+      - quarter_number
+      - attr_1
+    mapping_to:
+      - id
+      - country_code
+    new_columns:
+      - { name: counry_address, type: string }
+      - { name: country_code,type: double}
+out:
+  type: file
+  path_prefix: C:/Users/AnkitKumar/Desktop/embulk-files/sample/output.csv
+  file_ext: csv
+  formatter:
+      type: csv
+      delimiter: "\t"
+      newline: CRLF
+      newline_in_field: LF
+      charset: UTF-8
+      quote_policy: MINIMAL
+      quote: '"'
+      escape: "\\"
+      null_string: "\\N"
+      default_timezone: 'UTC'

data/example/{config.yml → config1.yml}

@@ -15,6 +15,8 @@ filters:
   - type: mysql_lookup
     host: localhost
     port: 3306
+    driver_path: C:/Users/AnkitKumar/Desktop/embulk-plugins/mysql-connector-java-8.0.19.jar
+    driver_class: com.mysql.cj.jdbc.Driver
     database: information
     table_name: country
     username: root

data/src/main/java/org/embulk/filter/mysql_lookup/DatabaseConnection.java

@@ -1,14 +1,35 @@
 package org.embulk.filter.mysql_lookup;
 
+
+import org.embulk.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
 import java.sql.DriverManager;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class DatabaseConnection {
     private static Connection connection=null;
 
     private DatabaseConnection(MysqlLookupFilterPlugin.PluginTask task) throws Exception {
         try{
-            Class.forName("com.mysql.cj.jdbc.Driver");
+            if(task.getDriverClass().isPresent()){
+                this.loadMySqlJdbcDriver(task.getDriverClass().get(),task.getDriverPath());
+            }else{
+                this.loadMySqlJdbcDriver("com.mysql.cj.jdbc.Driver",task.getDriverPath());
+            }
             String url = "jdbc:mysql://" + task.getHost() + ":"+task.getPort()+"/"+task.getDatabase();
             connection= DriverManager.getConnection(url, task.getUserName(), task.getPassword());
         }catch (Exception e){
@@ -35,4 +56,140 @@ public class DatabaseConnection {
 
         return connection;
     }
+
+    private Class<? extends java.sql.Driver> loadMySqlJdbcDriver(
+            final String className,
+            final Optional<String> driverPath)
+    {
+        synchronized (mysqlJdbcDriver) {
+            if (mysqlJdbcDriver.get() != null) {
+                return mysqlJdbcDriver.get();
+            }
+
+            try {
+                // If the class is found from the ClassLoader of the plugin, that is prioritized the highest.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mysqlJdbcDriver.compareAndSet(null, found);
+
+                if (driverPath.isPresent()) {
+                    logger.warn(
+                            "\"driver_path\" is set while the MySQL JDBC driver class \"{}\" is found from the PluginClassLoader."
+                                    + " \"driver_path\" is ignored.", className);
+                }
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                // Pass-through once.
+            }
+
+            if (driverPath.isPresent()) {
+                logger.info(
+                        "\"driver_path\" is set to load the MySQL JDBC driver class \"{}\". Adding it to classpath.", className);
+                this.addDriverJarToClasspath(driverPath.get());
+            }
+            else {
+                final File root = this.findPluginRoot();
+                final File driverLib = new File(root, "default_jdbc_driver");
+                final File[] files = driverLib.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(final File file)
+                    {
+                        return file.isFile() && file.getName().endsWith(".jar");
+                    }
+                });
+                if (files == null || files.length == 0) {
+                    throw new ConfigException(new ClassNotFoundException(
+                            "The MySQL JDBC driver for the class \"" + className + "\" is not found"
+                                    + " in \"default_jdbc_driver\" (" + root.getAbsolutePath() + ")."));
+                }
+                for (final File file : files) {
+                    logger.info(
+                            "The MySQL JDBC driver for the class \"{}\" is expected to be found"
+                                    + " in \"default_jdbc_driver\" at {}.", className, file.getAbsolutePath());
+                    this.addDriverJarToClasspath(file.getAbsolutePath());
+                }
+            }
+
+            try {
+                // Retrying to find the class from the ClassLoader of the plugin.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mysqlJdbcDriver.compareAndSet(null, found);
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                throw new ConfigException("The MySQL JDBC driver for the class \"" + className + "\" is not found.", ex);
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Class<? extends java.sql.Driver> loadJdbcDriverClassForName(final String className) throws ClassNotFoundException
+    {
+        return (Class<? extends java.sql.Driver>) Class.forName(className);
+    }
+
+    private static final AtomicReference<Class<? extends java.sql.Driver>> mysqlJdbcDriver = new AtomicReference<>();
+
+    private static final Logger logger = LoggerFactory.getLogger(DatabaseConnection.class);
+
+    protected void addDriverJarToClasspath(String glob)
+    {
+        // TODO match glob
+        final ClassLoader loader = getClass().getClassLoader();
+        if (!(loader instanceof URLClassLoader)) {
+            throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
+        }
+        if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
+            throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
+        }
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+            throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
+        final Method addPathMethod;
+        try {
+            addPathMethod = loader.getClass().getMethod("addPath", Path.class);
+        } catch (final NoSuchMethodException ex) {
+            throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
+        }
+        try {
+            addPathMethod.invoke(loader, Paths.get(glob));
+        } catch (final IllegalAccessException ex) {
+            throw new RuntimeException(ex);
+        } catch (final InvocationTargetException ex) {
+            final Throwable targetException = ex.getTargetException();
+            if (targetException instanceof MalformedURLException) {
+                throw new IllegalArgumentException(targetException);
+            } else if (targetException instanceof RuntimeException) {
+                throw (RuntimeException) targetException;
+            } else {
+                throw new RuntimeException(targetException);
+            }
+        }
+    }
+
+    protected File findPluginRoot()
+    {
+        try {
+            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    throw new RuntimeException("Cannot find 'embulk-filter-xxx' folder.");
+                }
+
+                if (folder.getName().startsWith("embulk-input-")) {
+                    return folder;
+                }
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+
 }

data/src/main/java/org/embulk/filter/mysql_lookup/MysqlLookupFilterPlugin.java

@@ -1,21 +1,13 @@
 package org.embulk.filter.mysql_lookup;
 import com.google.common.collect.ImmutableList;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
+import org.embulk.config.*;
 import org.embulk.spi.*;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.type.Types;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
+
+import java.sql.*;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class MysqlLookupFilterPlugin
         implements FilterPlugin {
@@ -27,6 +19,15 @@ public class MysqlLookupFilterPlugin
         @Config("port")
         public String getPort();
 
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
+        @Config("driver_class")
+        @ConfigDefault("null")
+        public Optional<String> getDriverClass();
+
+
         @Config("database")
         public String getDatabase();
 
@@ -96,8 +97,8 @@ public class MysqlLookupFilterPlugin
     private Map<String, List<String>> getKeyValueMap(PluginTask task) throws SQLException {
         Map<String, List<String>> map = new HashMap<>();
         Connection con = DatabaseConnection.getConnection(task);
-        try {
 
+        try {
             List<String> targetColumns = task.getMappingTo();
             List<String> newColumns = new ArrayList<>();
 
@@ -124,6 +125,8 @@ public class MysqlLookupFilterPlugin
 
             while (rs.next()) {
 
+                //console.log("");
+
                 //for key
                 String key = "";
                 String comp = "";
@@ -160,7 +163,6 @@ public class MysqlLookupFilterPlugin
         return map;
     }
 
-
     public static class MyOutput implements PageOutput {
         private PageReader reader;
         private PageBuilder builder;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-mysql_lookup
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - InfoObjects Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-26 00:00:00.000000000 Z
+date: 2022-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -49,13 +49,15 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-filter-mysql_lookup-0.1.4.jar
+- classpath/embulk-filter-mysql_lookup-0.1.5.jar
 - classpath/mysql-connector-java-8.0.19.jar
 - classpath/protobuf-java-3.6.1.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- config1.yml
 - embulk-filter-mysql_lookup-0.1.3.gem
-- example/config.yml
+- embulk-filter-mysql_lookup-0.1.4.gem
+- example/config1.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew