embulk-filter-postgres_lookup 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6741ec605cc98b03bd51993021bf8a0d9dcecc828ce90f0af88617c7d2ffd80
-  data.tar.gz: b8e5cabee3f41285f0f112e5b22fadd600cf0945482c1069648e02ca3eb6ba51
+  metadata.gz: 3d280c1d74ef7e126116a7effa655881ae726ab47139154aacd73ede5ca917c8
+  data.tar.gz: e1453e6e3fc0592116c69a5a3396ab9be9b97824e80ac61e9c6e94600a35b898
 SHA512:
-  metadata.gz: 7a7ab485b7c312547d89f3fb92b4fdd9ef1d6e1c631fe4a2f31f65373f8974bf3312e2e033630014ed5ccaf8563348b18c1075cc16b8891c321f72ee58d7b986
-  data.tar.gz: 206c090fc9e79c89c287cda83a284c02b65edcabb8a75c4c032252b2638a386d78932e26a7daa5c549906a3de488b371615b4704b028dbb3adb294f641f519f5
+  metadata.gz: 113881844a08f75a395da98c66ecc08e9c659e4300389546298afb5300d48be8826b6ee990c34761148b2f0933848682ba9172ec736afca672ac28377d7322d2
+  data.tar.gz: 444e9064fa2b54e7343b956bae28e642f7a21e64642e2956e2f40d7af7e6b2dd2889129deebd388959b6133c2600a7c0c3223592996b3db3274edf33019e20c2

data/README.md

@@ -1,9 +1,16 @@
+
+<p align="center">
+  <a href="https://www.infoobjects.com/" target="blank"><img src="screenshots/logo.png" width="150" alt="InfoObjects Logo" /></a>
+</p>
+<p align="center">Infoobjects is a consulting company that helps enterprises transform how and where they run applications and infrastructure.
+From strategy, to implementation, to ongoing managed services, Infoobjects creates tailored cloud solutions for enterprises at all stages of the cloud journey.</p>
+
 # Postgres lookup filter plugin for Embulk
+[![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
+
 ​
 An Embulk filter plugin for Lookup Transformation with Postgres database
 ​
-## Configuration
-​
 - **postgres_lookup**: Required attributes for the LookUp Filter Plugin -
     - **host**: database host (example `localhost`) (required)
     - **port**: database port (example port for postgres `1433`) (required)
@@ -131,4 +138,7 @@ Release gem:
 ​
 ```
 $ ./gradlew gemPush
-```
+```
+## Licensing
+
+InfoObjects [license](LICENSE) (MIT License)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -23,8 +23,8 @@ dependencies {
     provided "org.embulk:embulk-core:0.9.8"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     testCompile "junit:junit:4.+"
-    compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.19'
-    compile group: 'postgresql', name: 'postgresql', version: '9.0-801.jdbc4'
+//    compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.19'
+//    compile group: 'postgresql', name: 'postgresql', version: '9.0-801.jdbc4'
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/example/config.yml

@@ -3,7 +3,7 @@ exec:
   min_output_tasks: 1
 in:
   type: file
-  path_prefix: C:\Users\Abhishek Gupta\Desktop\new one\embulk-filter-postgress_lookup\calendar.csv
+  path_prefix: C:\Users\Abhishek Gupta\Desktop\github\embulk-filter-postgres_lookup\calendar.csv
   parser:
     type: csv
     columns:
@@ -13,28 +13,13 @@ in:
       - { name: attr_1, type: string }
 filters:
   - type: postgress_lookup
-    lookup_destination: mysql
+    driver_path: C:/Users/Abhishek Gupta/Desktop/postgresql-9.0-801.jdbc4.jar
+    driver_class: org.postgresql.Driver
+    schema_name: testSchema
     host: localhost
     port: 5432
     database: Abhishek
-    tablename: country
-    username: postgres
-    password: root
-    mapping_from:
-      - quarter_number
-      - attr_1
-    mapping_to:
-      - id
-      - country_address
-    new_columns:
-      - { name: country_name, type: string }
-      - { name: country_address, type: string }
-  - type: postgress_lookup
-    lookup_destination: mysql
-    host: localhost
-    port: 5432
-    database: Abhishek
-    tablename: country
+    table: country
     username: postgres
     password: root
     mapping_from:

data/src/main/java/org/embulk/filter/postgress_lookup/PostGresConnection.java

@@ -1,7 +1,24 @@
 package org.embulk.filter.postgress_lookup;
 
+import org.embulk.config.ConfigException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
+import java.sql.Driver;
 import java.sql.DriverManager;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class PostGresConnection {
 
@@ -9,7 +26,12 @@ public class PostGresConnection {
 
     private PostGresConnection(PostgressLookupFilterPlugin.PluginTask task) throws Exception {
         try{
-            Class.forName("org.postgresql.Driver");
+           // Class.forName("org.postgresql.Driver");
+            if(task.getDriverClass().isPresent()){
+                this.loadMySqlJdbcDriver(task.getDriverClass().get(),task.getDriverPath());
+            }else{
+                this.loadMySqlJdbcDriver("org.postgresql.Driver",task.getDriverPath());
+            }
             String url = "jdbc:postgresql://" + task.getHost() + ":"+task.getPort()+"/"+task.getDatabase();
             connection= DriverManager.getConnection(url, task.getUserName(), task.getPassword());
         }catch (Exception e){
@@ -36,4 +58,137 @@ public class PostGresConnection {
 
         return connection;
     }
+    private Class<? extends java.sql.Driver> loadMySqlJdbcDriver(
+            final String className,
+            final Optional<String> driverPath)
+    {
+        synchronized (mysqlJdbcDriver) {
+            if (mysqlJdbcDriver.get() != null) {
+                return mysqlJdbcDriver.get();
+            }
+
+            try {
+                // If the class is found from the ClassLoader of the plugin, that is prioritized the highest.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mysqlJdbcDriver.compareAndSet(null, found);
+
+                if (driverPath.isPresent()) {
+                    logger.warn(
+                            "\"driver_path\" is set while the MySQL JDBC driver class \"{}\" is found from the PluginClassLoader."
+                                    + " \"driver_path\" is ignored.", className);
+                }
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                // Pass-through once.
+            }
+
+            if (driverPath.isPresent()) {
+                logger.info(
+                        "\"driver_path\" is set to load the MySQL JDBC driver class \"{}\". Adding it to classpath.", className);
+                this.addDriverJarToClasspath(driverPath.get());
+            }
+            else {
+                final File root = this.findPluginRoot();
+                final File driverLib = new File(root, "default_jdbc_driver");
+                final File[] files = driverLib.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(final File file)
+                    {
+                        return file.isFile() && file.getName().endsWith(".jar");
+                    }
+                });
+                if (files == null || files.length == 0) {
+                    throw new ConfigException(new ClassNotFoundException(
+                            "The MySQL JDBC driver for the class \"" + className + "\" is not found"
+                                    + " in \"default_jdbc_driver\" (" + root.getAbsolutePath() + ")."));
+                }
+                for (final File file : files) {
+                    logger.info(
+                            "The MySQL JDBC driver for the class \"{}\" is expected to be found"
+                                    + " in \"default_jdbc_driver\" at {}.", className, file.getAbsolutePath());
+                    this.addDriverJarToClasspath(file.getAbsolutePath());
+                }
+            }
+
+            try {
+                // Retrying to find the class from the ClassLoader of the plugin.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mysqlJdbcDriver.compareAndSet(null, found);
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                throw new ConfigException("The MySQL JDBC driver for the class \"" + className + "\" is not found.", ex);
+            }
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Class<? extends java.sql.Driver> loadJdbcDriverClassForName(final String className) throws ClassNotFoundException
+    {
+        return (Class<? extends java.sql.Driver>) Class.forName(className);
+    }
+
+    private static final AtomicReference<Class<? extends Driver>> mysqlJdbcDriver = new AtomicReference<>();
+
+    private static final Logger logger = LoggerFactory.getLogger(PostGresConnection.class);
+
+    protected void addDriverJarToClasspath(String glob)
+    {
+        // TODO match glob
+        final ClassLoader loader = getClass().getClassLoader();
+        if (!(loader instanceof URLClassLoader)) {
+            throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
+        }
+        if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
+            throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
+        }
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+            throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
+        final Method addPathMethod;
+        try {
+            addPathMethod = loader.getClass().getMethod("addPath", Path.class);
+        } catch (final NoSuchMethodException ex) {
+            throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
+        }
+        try {
+            addPathMethod.invoke(loader, Paths.get(glob));
+        } catch (final IllegalAccessException ex) {
+            throw new RuntimeException(ex);
+        } catch (final InvocationTargetException ex) {
+            final Throwable targetException = ex.getTargetException();
+            if (targetException instanceof MalformedURLException) {
+                throw new IllegalArgumentException(targetException);
+            } else if (targetException instanceof RuntimeException) {
+                throw (RuntimeException) targetException;
+            } else {
+                throw new RuntimeException(targetException);
+            }
+        }
+    }
+
+    protected File findPluginRoot()
+    {
+        try {
+            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    throw new RuntimeException("Cannot find 'embulk-filter-xxx' folder.");
+                }
+
+                if (folder.getName().startsWith("embulk-input-")) {
+                    return folder;
+                }
+            }
+        } catch (URISyntaxException | MalformedURLException e) {
+            throw new RuntimeException(e);
+        }
+    }
 }

data/src/main/java/org/embulk/filter/postgress_lookup/PostgressLookupFilterPlugin.java

@@ -2,23 +2,14 @@ package org.embulk.filter.postgress_lookup;
 
 
 import com.google.common.collect.ImmutableList;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
+import org.embulk.config.*;
 import org.embulk.spi.*;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.type.Types;
 
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
+import java.sql.*;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 public class PostgressLookupFilterPlugin
         implements FilterPlugin {
@@ -34,7 +25,7 @@ public class PostgressLookupFilterPlugin
         @Config("database")
         public String getDatabase();
 
-        @Config("tablename")
+        @Config("table")
         public String getTableName();
 
         @Config("username")
@@ -51,6 +42,19 @@ public class PostgressLookupFilterPlugin
 
         @Config("new_columns")
         public SchemaConfig getNewColumns();
+
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
+        @Config("driver_class")
+        @ConfigDefault("null")
+        public Optional<String> getDriverClass();
+
+        @Config("schema_name")
+        @ConfigDefault("null")
+        public Optional<String> getSchemaName();
+
     }
 
 
@@ -100,6 +104,22 @@ public class PostgressLookupFilterPlugin
     private Map<String, List<String>> getKeyValueMap(PluginTask task) throws SQLException {
         Map<String, List<String>> map = new HashMap<>();
         Connection con = PostGresConnection.getConnection(task);
+        DatabaseMetaData databaseMetaData =con.getMetaData();
+        String identifierQuoteString=databaseMetaData.getIdentifierQuoteString();
+        //String schemaName=null;
+
+        if (task.getSchemaName().isPresent()) {
+            String sql = "SET search_path TO " + identifierQuoteString + task.getSchemaName().get() + identifierQuoteString;
+            System.out.println(sql);
+
+            Statement stmt = con.createStatement();
+            try {
+                stmt.executeUpdate(sql);
+            } finally {
+                stmt.close();
+            }
+        }
+        con.setAutoCommit(false);
         try {
 
             List<String> targetColumns = task.getMappingTo();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-postgres_lookup
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Infoobjects Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-16 00:00:00.000000000 Z
+date: 2022-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,23 +46,20 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- LICENSE.txt
+- LICENSE
 - README.md
 - build.gradle
 - calendar.csv
-- classpath/embulk-filter-postgres_lookup-0.1.0.jar
-- classpath/mysql-connector-java-8.0.19.jar
-- classpath/postgresql-9.0-801.jdbc4.jar
-- classpath/protobuf-java-3.6.1.jar
+- classpath/embulk-filter-postgres_lookup-0.1.2.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
-- embulk-filter-postgress_lookup-0.1.0.gem
 - example/config.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/postgress_lookup.rb
+- screenshots/logo.png
 - src/main/java/org/embulk/filter/postgress_lookup/PostGresConnection.java
 - src/main/java/org/embulk/filter/postgress_lookup/PostgressLookupFilterPlugin.java
 - src/test/java/org/embulk/filter/postgress_lookup/TestPostgressLookupFilterPlugin.java