embulk-filter-mssql_lookup 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 824dbb4c0c5e607fc9a69c607b4fd9e741155c0b3933ea2e587584549f0bd3bc
-  data.tar.gz: fcd7322b3786a0c9e933487226416b4226f095c01b1c9873385083dbfcd565e1
+  metadata.gz: 40422447490354ff161a3de66b9f38aa09276bacbcbe4c0951e728c0e7f16c97
+  data.tar.gz: 010cec4f77a7c7d9e7565ee80c9f8d90985632b9fb90dad43bf00b9aff27a69b
 SHA512:
-  metadata.gz: 88ce5dcca3d990d6ffbb8590a10cde4db0a4d4ae8e746d0511184d662ac0fdb40210c16999ddf25dd076551f566ac3554d54365325304530ebbd3d0500ade97c
-  data.tar.gz: c9adb088037d5eb57c9aa3cf63a44c5c22269b01f251db2f1275c7febd6f538a93f3252c26ef1b7fee8df9e8f3fa7f9e6b19192e2fa9b264b114262b249cdb77
+  metadata.gz: 0577f89f211199c9ace7105bb0f8997733a5c3f95c9398ed481a3b9a03254bd7d2367cc334e0541bf816e285be7ea139e33a32af362e4e6cfc0b658574e1ce10
+  data.tar.gz: aa72a4aa80975d05c6c070fd5be67962ae005f5b525946c904a3afc2a53d71443839606397dfad6d6612ea0e6ba454958b8b5bbf966eb046aede6b19450d1e81

data/README.md

@@ -1,14 +1,26 @@
+<p align="center">
+  <a href="https://www.infoobjects.com/" target="blank"><img src="screenshots/logo.png" width="150" alt="InfoObjects Logo" /></a>
+</p>
+<p align="center">Infoobjects is a consulting company that helps enterprises transform how and where they run applications and infrastructure.
+From strategy, to implementation, to ongoing managed services, Infoobjects creates tailored cloud solutions for enterprises at all stages of the cloud journey.</p>
+
 # MS-SQL lookup filter plugin for Embulk
+[![license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 
 An Embulk filter plugin for Lookup Transformation with MS-SQL database
 
 ## Configuration
 
 - **mssql_lookup**: Required attributes for the LookUp Filter Plugin -
+    - **filters**:
+        - **type**: Name of lookup type (required)
+    - **driver_path**: /home/infoobjects/Downloads/sqljdbc_1s1.2.0.0_enu/sqljdbc_11.2/enu/mssql-jdbc-11.2.0.jre8.jar
+    - **driver_class**: com.microsoft.sqlserver.jdbc.SQLServerDriver
     - **host**: database host (example `localhost`) (required)
     - **port**: database port (example port for mssql `1433`) (required
     - **database**: database name (required)
-    - **tablename**: table name of your database (required)
+    - **schemaName**: name of schema in database in which table is located
+    - **table_name**: table name of your database (required)
     - **username**: username for your database (required)
     - **password**: password for database (required)
     - **mapping_from**: (Name of columns to be matched with table 2 columns) (required)
@@ -80,11 +92,14 @@ Output File generated :-
 ```
 
 ```yaml
- - type: mssql_lookup
+   type: mssql_lookup
+   driver_path: /home/infoobjects/Downloads/sqljdbc_1s1.2.0.0_enu/sqljdbc_11.2/enu/mssql-jdbc-11.2.0.jre8.jar
+   driver_class: com.microsoft.sqlserver.jdbc.SQLServerDriver
    host: localhost
    port: 1433
    database: test
-   tablename: country_details
+   schemaName: testSchema1
+   table_name: country_details
    username: root
    password: root
    mapping_from:
@@ -125,4 +140,8 @@ Release gem:
 
 ```
 $ ./gradlew gemPush
-```
+```
+
+## Licensing
+
+InfoObjects [license](LICENSE) (MIT License)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -80,7 +80,7 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Infoobjects Inc."]
   spec.summary       = %[Mssql Lookup filter plugin for Embulk]
   spec.description   = %[Mssql Lookup]
-  spec.email         = ["ankit.navral@infoobjects.com"]
+  spec.email         = ["utkarsh@infoobjects.com"]
   spec.licenses      = ["MIT"]
   # TODO set this: 
   spec.homepage      = "https://github.com/ankit31223/embulk-filter-mysql_lookup"

data/example/config.yml

@@ -1,59 +1,32 @@
 exec:
-  max_threads: 2
+  max_threads: 1
   min_output_tasks: 1
 in:
   type: file
-  path_prefix: /home/infoobjects/Downloads/sample/calendarFloat.csv
+  path_prefix: /home/infoobjects/Downloads/sample/calendarFloat1 (copy).csv
   parser:
     type: csv
     columns:
-      - { name: dim_calendar_key, type: long }
-      - { name: year_number, type: long }
+      - { name: Country_Name, type: string }
       - { name: quarter_number, type: long }
-      - { name: attr_1, type: string }
+
 filters:
   - type: mssql_lookup
+    driver_path: /home/infoobjects/Downloads/sqljdbc_1s1.2.0.0_enu/sqljdbc_11.2/enu/mssql-jdbc-11.2.0.jre8.jar
+    driver_class: com.microsoft.sqlserver.jdbc.SQLServerDriver
     host: infoobjects-ThinkPad-E490
     database: test
-    tablename: countryNew
+    schemaName: testSchema1
+    table_name: cnew
     username: SA
     password: Mssqlroot95
     mapping_from:
       - quarter_number
-      - attr_1
     mapping_to:
-      - id
-      - country_code
+      - qtr_no
     new_columns:
-      - { name: country_name, type: string }
-  - type: mssql_lookup
-    host: infoobjects-ThinkPad-E490
-    database: test
-    tablename: countryNew
-    username: SA
-    password: Mssqlroot95
-    mapping_from:
-      - quarter_number
-      - attr_1
-    mapping_to:
-      - id
-      - country_code
-    new_columns:
-      - { name: country_address, type: string }
-  - type: mssql_lookup
-    host: infoobjects-ThinkPad-E490
-    database: test
-    tablename: countryNew
-    username: SA
-    password: Mssqlroot95
-    mapping_from:
-      - quarter_number
-      - attr_1
-    mapping_to:
-      - id
-      - country_code
-    new_columns:
-      - { name: country_code,type: double}
+      - { name: qtr_name, type: string }
+
 out:
   type: file
   path_prefix: /home/infoobjects/sqlOutput/output.csv

data/src/main/java/org/embulk/filter/mssql_lookup/DatabaseConnection.java

@@ -1,50 +1,191 @@
 package org.embulk.filter.mssql_lookup;
 
+import org.embulk.config.ConfigException;
+import org.jruby.RubyProcess;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.Connection;
-import java.sql.DatabaseMetaData;
 import java.sql.DriverManager;
-import java.sql.SQLException;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicReference;
 
 public class DatabaseConnection {
     private static Connection connection = null;
     private DatabaseConnection(MssqlLookupFilterPlugin.PluginTask task) throws Exception{
         try {
-            Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-            String dbURL = "jdbc:sqlserver://" +task.getHost() + ";databaseName=" +task.getDatabase();
-            connection = DriverManager.getConnection(dbURL, task.getUserName(), task.getPassword());
-            if (connection != null){
-                DatabaseMetaData dm = null;
+            if(task.getDriverClass().isPresent()){
+                this.loadMsSqlJdbcDriver(task.getDriverClass().get(), task.getDriverPath());
+            }else{
+                this.loadMsSqlJdbcDriver("com.microsoft.sqlserver.jdbc.SQLServerDriver",task.getDriverPath());
+            }
+            String url = "jdbc:sqlserver://" +task.getHost() + ";databaseName=" +task.getDatabase();
+            connection= DriverManager.getConnection(url, task.getUserName(), task.getPassword());
+        }catch (Exception e){
+            e.printStackTrace();
+            throw new Exception(e);
+        }
+    }
+    public static Connection getConnection(MssqlLookupFilterPlugin.PluginTask task){
+        try {
+            if(connection==null || connection.isClosed()){
                 try {
-                    dm = connection.getMetaData();
-                } catch (SQLException e) {
-                    throw new RuntimeException(e);
+                    new DatabaseConnection(task);
+                    return connection;
+                } catch (Exception e) {
+                    e.printStackTrace();
+                    throw new RuntimeException();
                 }
-                try {
-                    System.out.println("Driver name: " + dm.getDriverName());
-                    System.out.println("Driver version: " + dm.getDriverVersion());
-                    System.out.println("Product name: " + dm.getDatabaseProductName());
-                    System.out.println("Product version: " + dm.getDatabaseProductVersion());
-                } catch (SQLException e) {
-                    throw new RuntimeException(e);
+            }
+        }catch (Exception e){
+            throw new RuntimeException(e);
+        }
+
+        return connection;
+    }
+
+    private Class<? extends java.sql.Driver> loadMsSqlJdbcDriver(
+            final String className,
+              final Optional<String>driverPath)
+    {
+        synchronized (mssqlJdbcDriver) {
+            if (mssqlJdbcDriver.get() != null) {
+                return mssqlJdbcDriver.get();
+            }
+
+            try {
+                // If the class is found from the ClassLoader of the plugin, that is prioritized the highest.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mssqlJdbcDriver.compareAndSet(null, found);
+
+                if (driverPath.isPresent()) {
+                    logger.warn(
+                            "\"driver_path\" is set while the MsSQL JDBC driver class \"{}\" is found from the PluginClassLoader."
+                                    + " \"driver_path\" is ignored.", className);
                 }
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                // Pass-through once.
+            }
 
+            if (driverPath.isPresent()) {
+                logger.info(
+                        "\"driver_path\" is set to load the MsSQL JDBC driver class \"{}\". Adding it to classpath.", className);
+                this.addDriverJarToClasspath(driverPath.get());
+            }
+            else {
+                final File root = this.findPluginRoot();
+                final File driverLib = new File(root, "default_jdbc_driver");
+                final File[] files = driverLib.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(final File file)
+                    {
+                        return file.isFile() && file.getName().endsWith(".jar");
+                    }
+                });
+                if (files == null || files.length == 0) {
+                    throw new ConfigException(new ClassNotFoundException(
+                            "The MsSQL JDBC driver for the class \"" + className + "\" is not found"
+                                    + " in \"default_jdbc_driver\" (" + root.getAbsolutePath() + ")."));
+                }
+                for (final File file : files) {
+                    logger.info(
+                            "The MsSQL JDBC driver for the class \"{}\" is expected to be found"
+                                    + " in \"default_jdbc_driver\" at {}.", className, file.getAbsolutePath());
+                    this.addDriverJarToClasspath(file.getAbsolutePath());
+                }
             }
 
-        }catch (Exception e){
-            e.printStackTrace();
-            throw new Exception(e);
+            try {
+                // Retrying to find the class from the ClassLoader of the plugin.
+                final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
+                mssqlJdbcDriver.compareAndSet(null, found);
+                return found;
+            }
+            catch (final ClassNotFoundException ex) {
+                throw new ConfigException("The MsSQL JDBC driver for the class \"" + className + "\" is not found.", ex);
+            }
         }
     }
-    public static Connection getConnection(MssqlLookupFilterPlugin.PluginTask task) throws SQLException {
-        if(connection==null || connection.isClosed()){
-            try {
-                new DatabaseConnection(task);
-                return connection;
-            } catch (Exception e) {
-                e.printStackTrace();
-                throw new RuntimeException();
+
+    @SuppressWarnings("unchecked")
+    private static Class<? extends java.sql.Driver> loadJdbcDriverClassForName(final String className) throws ClassNotFoundException
+    {
+        return (Class<? extends java.sql.Driver>) Class.forName(className);
+    }
+
+    private static final AtomicReference<Class<? extends java.sql.Driver>> mssqlJdbcDriver = new AtomicReference<>();
+
+    private static final Logger logger = LoggerFactory.getLogger(DatabaseConnection.class);
+
+    protected void addDriverJarToClasspath(String glob)
+    {
+        // TODO match glob
+        final ClassLoader loader = getClass().getClassLoader();
+        if (!(loader instanceof URLClassLoader)) {
+            throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
+        }
+        if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
+            throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
+        }
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+            throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
+        final Method addPathMethod;
+        try {
+            addPathMethod = loader.getClass().getMethod("addPath", Path.class);
+        } catch (final NoSuchMethodException ex) {
+            throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
+        }
+        try {
+            addPathMethod.invoke(loader, Paths.get(glob));
+        } catch (final IllegalAccessException ex) {
+            throw new RuntimeException(ex);
+        } catch (final InvocationTargetException ex) {
+            final Throwable targetException = ex.getTargetException();
+            if (targetException instanceof MalformedURLException) {
+                throw new IllegalArgumentException(targetException);
+            } else if (targetException instanceof RuntimeException) {
+                throw (RuntimeException) targetException;
+            } else {
+                throw new RuntimeException(targetException);
             }
         }
-        return connection;
     }
-}
+
+    protected File findPluginRoot()
+    {
+        try {
+            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    throw new RuntimeException("Cannot find 'embulk-filter-xxx' folder.");
+                }
+
+                if (folder.getName().startsWith("embulk-input-")) {
+                    return folder;
+                }
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+}

data/src/main/java/org/embulk/filter/mssql_lookup/MssqlLookupFilterPlugin.java

@@ -1,14 +1,8 @@
 package org.embulk.filter.mssql_lookup;
 
 import com.google.common.base.Optional;
-
 import com.google.common.collect.ImmutableList;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
+import org.embulk.config.*;
 import org.embulk.spi.*;
 import org.embulk.spi.type.Types;
 
@@ -34,7 +28,7 @@ public class MssqlLookupFilterPlugin
         @Config("database")
         public String getDatabase();
 
-        @Config("tablename")
+        @Config("table_name")
         public String getTableName();
 
         @Config("username")
@@ -52,6 +46,18 @@ public class MssqlLookupFilterPlugin
         @Config("new_columns")
         public SchemaConfig getNewColumns();
 
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public java.util.Optional<String> getDriverPath();
+
+        @Config("driver_class")
+        @ConfigDefault("null")
+        public Optional<String> getDriverClass();
+
+        @Config("schemaName")
+        @ConfigDefault("null")
+        public Optional<String> getSchemaName();
+
     }
 
     @Override
@@ -101,7 +107,6 @@ public class MssqlLookupFilterPlugin
         Map<String, List<String>> map = new HashMap<>();
         Connection con = DatabaseConnection.getConnection(task);
         try {
-
             List<String> targetColumns = task.getMappingTo();
             List<String> newColumns = new ArrayList<>();
 
@@ -121,8 +126,12 @@ public class MssqlLookupFilterPlugin
                     columnNeedsToBeFetched += newColumns.get(i);
                 }
             }
-            query += columnNeedsToBeFetched + " from " + task.getTableName();
+            if (task.getSchemaName().isPresent()) {
+                query += columnNeedsToBeFetched + " from " + task.getSchemaName().get() + "." + task.getTableName();
+            }else {
+                query += columnNeedsToBeFetched + " from " + task.getTableName();
 
+            }
             Statement stmt = con.createStatement();
             ResultSet rs = stmt.executeQuery(query);
 
@@ -233,10 +242,8 @@ public class MssqlLookupFilterPlugin
                 }
                 builder.addRecord();
             }
-
         }
 
-
         @Override
         public void finish() {
             builder.finish();
@@ -344,14 +351,12 @@ public class MssqlLookupFilterPlugin
                         }else{
                             java.sql.Timestamp timestamp = java.sql.Timestamp.valueOf(matchedData);
                             Instant instant = timestamp.toInstant();
-                            //  Timestamp spiTimeStamp = Timestamp.ofInstant(instant);
                             org.embulk.spi.time.Timestamp spiTimeStamp = org.embulk.spi.time.Timestamp.ofInstant(instant);
                             builder.setTimestamp(colNum, spiTimeStamp);
                         }
                     } else {
                         builder.setNull(colNum);
                     }
-
                 }
             }catch (Exception e){
                 e.printStackTrace();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-mssql_lookup
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.2
 platform: ruby
 authors:
 - Infoobjects Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-16 00:00:00.000000000 Z
+date: 2022-11-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -40,16 +40,16 @@ dependencies:
         version: '12.0'
 description: Mssql Lookup
 email:
-- ankit.navral@infoobjects.com
+- utkarsh@infoobjects.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- LICENSE.txt
+- LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-filter-mssql_lookup-0.1.0.jar
+- classpath/embulk-filter-mssql_lookup-0.1.2.jar
 - classpath/sqljdbc4-4.0.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
@@ -59,6 +59,7 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/mssql_lookup.rb
+- screenshots/logo.png
 - src/main/java/org/embulk/filter/mssql_lookup/DatabaseConnection.java
 - src/main/java/org/embulk/filter/mssql_lookup/MssqlLookupFilterPlugin.java
 - src/test/java/org/embulk/filter/mssql_lookup/TestMssqlLookupFilterPlugin.java