embulk-filter-postgres_lookup 0.1.5 → 0.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +1 -1
- data/calendar.csv +10 -10
- data/classpath/{embulk-filter-postgres_lookup-0.1.5.jar → embulk-filter-postgres_lookup-0.1.6.jar} +0 -0
- data/example/config.yml +47 -47
- data/gradlew +0 -0
- data/gradlew.bat +84 -84
- data/src/main/java/org/embulk/filter/postgress_lookup/PostGresConnection.java +194 -194
- data/src/main/java/org/embulk/filter/postgress_lookup/PostgressLookupFilterPlugin.java +1 -1
- metadata +9 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 20b93f14dc975405be1a678e0646c142aec8b8e097478593d55b81de2f90010a
|
4
|
+
data.tar.gz: 84ed7ed57d8d408630faf86380daf24d4e5b0a0636fc61c9cd3560b3486132af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b4d07337a0f56e160de416a79d836ad2804aedc3867a9393662b064dab628fd367c1d33d59b221654bb46ba68afdd1586e0880d9bee7f9406b357b63006bed3c
|
7
|
+
data.tar.gz: d6d20a5856938c1a1b94a91408a6c9afe6def0408d18e799b5f1732ba63c2c9fbe1e1020c026710de3d7fa19b32460c3ce927b89e8849bbb3d0d2db3a54d2674
|
data/build.gradle
CHANGED
data/calendar.csv
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
dim_calendar_key,year_number,quarter_number,attr_1
|
2
|
-
-1,0,0
|
3
|
-
19900101,1990,1,FirstIndia
|
4
|
-
19900102,1990,2,SecondUSA
|
5
|
-
19900103,1990,,1000.56
|
6
|
-
19900104,1990,4,40_China
|
7
|
-
19900105,1990,5,50_Ukraine
|
8
|
-
19900106,1990,2,10_USA
|
9
|
-
19900107,1990,3,30_UK
|
10
|
-
19900109,1990,6,
|
1
|
+
dim_calendar_key,year_number,quarter_number,attr_1
|
2
|
+
-1,0,0
|
3
|
+
19900101,1990,1,FirstIndia
|
4
|
+
19900102,1990,2,SecondUSA
|
5
|
+
19900103,1990,,1000.56
|
6
|
+
19900104,1990,4,40_China
|
7
|
+
19900105,1990,5,50_Ukraine
|
8
|
+
19900106,1990,2,10_USA
|
9
|
+
19900107,1990,3,30_UK
|
10
|
+
19900109,1990,6,
|
data/classpath/{embulk-filter-postgres_lookup-0.1.5.jar → embulk-filter-postgres_lookup-0.1.6.jar}
RENAMED
Binary file
|
data/example/config.yml
CHANGED
@@ -1,48 +1,48 @@
|
|
1
|
-
exec:
|
2
|
-
max_threads: 2
|
3
|
-
min_output_tasks: 1
|
4
|
-
in:
|
5
|
-
type: file
|
6
|
-
path_prefix: C:\Users\Abhishek Gupta\Desktop\github\embulk-filter-postgres_lookup\calendar.csv
|
7
|
-
parser:
|
8
|
-
type: csv
|
9
|
-
columns:
|
10
|
-
- { name: dim_calendar_key, type: long }
|
11
|
-
- { name: year_number, type: long }
|
12
|
-
- { name: quarter_number, type: long }
|
13
|
-
- { name: attr_1, type: string }
|
14
|
-
filters:
|
15
|
-
- type: postgress_lookup
|
16
|
-
driver_path: C:/Users/Abhishek Gupta/Desktop/postgresql-9.0-801.jdbc4.jar
|
17
|
-
driver_class: org.postgresql.Driver
|
18
|
-
schema_name: testSchema
|
19
|
-
host: localhost
|
20
|
-
port: 5432
|
21
|
-
database: Abhishek
|
22
|
-
table: country
|
23
|
-
username: postgres
|
24
|
-
password: root
|
25
|
-
mapping_from:
|
26
|
-
- quarter_number
|
27
|
-
- attr_1
|
28
|
-
mapping_to:
|
29
|
-
- id
|
30
|
-
- country_address
|
31
|
-
new_columns:
|
32
|
-
- { name: country_name, type: string }
|
33
|
-
- { name: country_address, type: string }
|
34
|
-
out:
|
35
|
-
type: file
|
36
|
-
path_prefix: C:\Users\Abhishek Gupta\Desktop\B\output.csv
|
37
|
-
file_ext: csv
|
38
|
-
formatter:
|
39
|
-
type: csv
|
40
|
-
delimiter: "\t"
|
41
|
-
newline: CRLF
|
42
|
-
newline_in_field: LF
|
43
|
-
charset: UTF-8
|
44
|
-
quote_policy: MINIMAL
|
45
|
-
quote: '"'
|
46
|
-
escape: "\\"
|
47
|
-
null_string: "\\N"
|
1
|
+
exec:
|
2
|
+
max_threads: 2
|
3
|
+
min_output_tasks: 1
|
4
|
+
in:
|
5
|
+
type: file
|
6
|
+
path_prefix: C:\Users\Abhishek Gupta\Desktop\github\embulk-filter-postgres_lookup\calendar.csv
|
7
|
+
parser:
|
8
|
+
type: csv
|
9
|
+
columns:
|
10
|
+
- { name: dim_calendar_key, type: long }
|
11
|
+
- { name: year_number, type: long }
|
12
|
+
- { name: quarter_number, type: long }
|
13
|
+
- { name: attr_1, type: string }
|
14
|
+
filters:
|
15
|
+
- type: postgress_lookup
|
16
|
+
driver_path: C:/Users/Abhishek Gupta/Desktop/postgresql-9.0-801.jdbc4.jar
|
17
|
+
driver_class: org.postgresql.Driver
|
18
|
+
schema_name: testSchema
|
19
|
+
host: localhost
|
20
|
+
port: 5432
|
21
|
+
database: Abhishek
|
22
|
+
table: country
|
23
|
+
username: postgres
|
24
|
+
password: root
|
25
|
+
mapping_from:
|
26
|
+
- quarter_number
|
27
|
+
- attr_1
|
28
|
+
mapping_to:
|
29
|
+
- id
|
30
|
+
- country_address
|
31
|
+
new_columns:
|
32
|
+
- { name: country_name, type: string }
|
33
|
+
- { name: country_address, type: string }
|
34
|
+
out:
|
35
|
+
type: file
|
36
|
+
path_prefix: C:\Users\Abhishek Gupta\Desktop\B\output.csv
|
37
|
+
file_ext: csv
|
38
|
+
formatter:
|
39
|
+
type: csv
|
40
|
+
delimiter: "\t"
|
41
|
+
newline: CRLF
|
42
|
+
newline_in_field: LF
|
43
|
+
charset: UTF-8
|
44
|
+
quote_policy: MINIMAL
|
45
|
+
quote: '"'
|
46
|
+
escape: "\\"
|
47
|
+
null_string: "\\N"
|
48
48
|
default_timezone: 'UTC'
|
data/gradlew
CHANGED
File without changes
|
data/gradlew.bat
CHANGED
@@ -1,84 +1,84 @@
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
2
|
-
@rem ##########################################################################
|
3
|
-
@rem
|
4
|
-
@rem Gradle startup script for Windows
|
5
|
-
@rem
|
6
|
-
@rem ##########################################################################
|
7
|
-
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
10
|
-
|
11
|
-
set DIRNAME=%~dp0
|
12
|
-
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
-
set APP_BASE_NAME=%~n0
|
14
|
-
set APP_HOME=%DIRNAME%
|
15
|
-
|
16
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
-
set DEFAULT_JVM_OPTS=
|
18
|
-
|
19
|
-
@rem Find java.exe
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
-
|
22
|
-
set JAVA_EXE=java.exe
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
25
|
-
|
26
|
-
echo.
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
-
echo.
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
-
echo location of your Java installation.
|
31
|
-
|
32
|
-
goto fail
|
33
|
-
|
34
|
-
:findJavaFromJavaHome
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
-
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
39
|
-
|
40
|
-
echo.
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
-
echo.
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
-
echo location of your Java installation.
|
45
|
-
|
46
|
-
goto fail
|
47
|
-
|
48
|
-
:init
|
49
|
-
@rem Get command-line arguments, handling Windows variants
|
50
|
-
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
|
53
|
-
:win9xME_args
|
54
|
-
@rem Slurp the command line arguments.
|
55
|
-
set CMD_LINE_ARGS=
|
56
|
-
set _SKIP=2
|
57
|
-
|
58
|
-
:win9xME_args_slurp
|
59
|
-
if "x%~1" == "x" goto execute
|
60
|
-
|
61
|
-
set CMD_LINE_ARGS=%*
|
62
|
-
|
63
|
-
:execute
|
64
|
-
@rem Setup the command line
|
65
|
-
|
66
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
-
|
68
|
-
@rem Execute Gradle
|
69
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
-
|
71
|
-
:end
|
72
|
-
@rem End local scope for the variables with windows NT shell
|
73
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
-
|
75
|
-
:fail
|
76
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
-
rem the _cmd.exe /c_ return code!
|
78
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
-
exit /b 1
|
80
|
-
|
81
|
-
:mainEnd
|
82
|
-
if "%OS%"=="Windows_NT" endlocal
|
83
|
-
|
84
|
-
:omega
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS=
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
|
53
|
+
:win9xME_args
|
54
|
+
@rem Slurp the command line arguments.
|
55
|
+
set CMD_LINE_ARGS=
|
56
|
+
set _SKIP=2
|
57
|
+
|
58
|
+
:win9xME_args_slurp
|
59
|
+
if "x%~1" == "x" goto execute
|
60
|
+
|
61
|
+
set CMD_LINE_ARGS=%*
|
62
|
+
|
63
|
+
:execute
|
64
|
+
@rem Setup the command line
|
65
|
+
|
66
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
67
|
+
|
68
|
+
@rem Execute Gradle
|
69
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
70
|
+
|
71
|
+
:end
|
72
|
+
@rem End local scope for the variables with windows NT shell
|
73
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
74
|
+
|
75
|
+
:fail
|
76
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
77
|
+
rem the _cmd.exe /c_ return code!
|
78
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
79
|
+
exit /b 1
|
80
|
+
|
81
|
+
:mainEnd
|
82
|
+
if "%OS%"=="Windows_NT" endlocal
|
83
|
+
|
84
|
+
:omega
|
@@ -1,194 +1,194 @@
|
|
1
|
-
package org.embulk.filter.postgress_lookup;
|
2
|
-
|
3
|
-
import org.embulk.config.ConfigException;
|
4
|
-
import org.slf4j.Logger;
|
5
|
-
import org.slf4j.LoggerFactory;
|
6
|
-
|
7
|
-
import java.io.File;
|
8
|
-
import java.io.FileFilter;
|
9
|
-
import java.lang.reflect.InvocationTargetException;
|
10
|
-
import java.lang.reflect.Method;
|
11
|
-
import java.net.MalformedURLException;
|
12
|
-
import java.net.URISyntaxException;
|
13
|
-
import java.net.URL;
|
14
|
-
import java.net.URLClassLoader;
|
15
|
-
import java.nio.file.Path;
|
16
|
-
import java.nio.file.Paths;
|
17
|
-
import java.sql.Connection;
|
18
|
-
import java.sql.Driver;
|
19
|
-
import java.sql.DriverManager;
|
20
|
-
import java.util.Optional;
|
21
|
-
import java.util.concurrent.atomic.AtomicReference;
|
22
|
-
|
23
|
-
public class PostGresConnection {
|
24
|
-
|
25
|
-
private static Connection connection=null;
|
26
|
-
|
27
|
-
private PostGresConnection(PostgressLookupFilterPlugin.PluginTask task) throws Exception {
|
28
|
-
try{
|
29
|
-
// Class.forName("org.postgresql.Driver");
|
30
|
-
if(task.getDriverClass().isPresent()){
|
31
|
-
this.loadMySqlJdbcDriver(task.getDriverClass().get(),task.getDriverPath());
|
32
|
-
}else{
|
33
|
-
this.loadMySqlJdbcDriver("org.postgresql.Driver",task.getDriverPath());
|
34
|
-
}
|
35
|
-
String url = "jdbc:postgresql://" + task.getHost() + ":"+task.getPort()+"/"+task.getDatabase();
|
36
|
-
connection= DriverManager.getConnection(url, task.getUserName(), task.getPassword());
|
37
|
-
}catch (Exception e){
|
38
|
-
e.printStackTrace();
|
39
|
-
throw new Exception(e);
|
40
|
-
}
|
41
|
-
|
42
|
-
}
|
43
|
-
|
44
|
-
public static Connection getConnection(PostgressLookupFilterPlugin.PluginTask task){
|
45
|
-
try {
|
46
|
-
if(connection==null || connection.isClosed()){
|
47
|
-
try {
|
48
|
-
new PostGresConnection(task);
|
49
|
-
return connection;
|
50
|
-
} catch (Exception e) {
|
51
|
-
e.printStackTrace();
|
52
|
-
throw new RuntimeException();
|
53
|
-
}
|
54
|
-
}
|
55
|
-
}catch (Exception e){
|
56
|
-
throw new RuntimeException(e);
|
57
|
-
}
|
58
|
-
|
59
|
-
return connection;
|
60
|
-
}
|
61
|
-
private Class<? extends java.sql.Driver> loadMySqlJdbcDriver(
|
62
|
-
final String className,
|
63
|
-
final Optional<String> driverPath)
|
64
|
-
{
|
65
|
-
synchronized (mysqlJdbcDriver) {
|
66
|
-
if (mysqlJdbcDriver.get() != null) {
|
67
|
-
return mysqlJdbcDriver.get();
|
68
|
-
}
|
69
|
-
|
70
|
-
try {
|
71
|
-
// If the class is found from the ClassLoader of the plugin, that is prioritized the highest.
|
72
|
-
final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
|
73
|
-
mysqlJdbcDriver.compareAndSet(null, found);
|
74
|
-
|
75
|
-
if (driverPath.isPresent()) {
|
76
|
-
logger.warn(
|
77
|
-
"\"driver_path\" is set while the MySQL JDBC driver class \"{}\" is found from the PluginClassLoader."
|
78
|
-
+ " \"driver_path\" is ignored.", className);
|
79
|
-
}
|
80
|
-
return found;
|
81
|
-
}
|
82
|
-
catch (final ClassNotFoundException ex) {
|
83
|
-
// Pass-through once.
|
84
|
-
}
|
85
|
-
|
86
|
-
if (driverPath.isPresent()) {
|
87
|
-
logger.info(
|
88
|
-
"\"driver_path\" is set to load the MySQL JDBC driver class \"{}\". Adding it to classpath.", className);
|
89
|
-
this.addDriverJarToClasspath(driverPath.get());
|
90
|
-
}
|
91
|
-
else {
|
92
|
-
final File root = this.findPluginRoot();
|
93
|
-
final File driverLib = new File(root, "default_jdbc_driver");
|
94
|
-
final File[] files = driverLib.listFiles(new FileFilter() {
|
95
|
-
@Override
|
96
|
-
public boolean accept(final File file)
|
97
|
-
{
|
98
|
-
return file.isFile() && file.getName().endsWith(".jar");
|
99
|
-
}
|
100
|
-
});
|
101
|
-
if (files == null || files.length == 0) {
|
102
|
-
throw new ConfigException(new ClassNotFoundException(
|
103
|
-
"The MySQL JDBC driver for the class \"" + className + "\" is not found"
|
104
|
-
+ " in \"default_jdbc_driver\" (" + root.getAbsolutePath() + ")."));
|
105
|
-
}
|
106
|
-
for (final File file : files) {
|
107
|
-
logger.info(
|
108
|
-
"The MySQL JDBC driver for the class \"{}\" is expected to be found"
|
109
|
-
+ " in \"default_jdbc_driver\" at {}.", className, file.getAbsolutePath());
|
110
|
-
this.addDriverJarToClasspath(file.getAbsolutePath());
|
111
|
-
}
|
112
|
-
}
|
113
|
-
|
114
|
-
try {
|
115
|
-
// Retrying to find the class from the ClassLoader of the plugin.
|
116
|
-
final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
|
117
|
-
mysqlJdbcDriver.compareAndSet(null, found);
|
118
|
-
return found;
|
119
|
-
}
|
120
|
-
catch (final ClassNotFoundException ex) {
|
121
|
-
throw new ConfigException("The MySQL JDBC driver for the class \"" + className + "\" is not found.", ex);
|
122
|
-
}
|
123
|
-
}
|
124
|
-
}
|
125
|
-
|
126
|
-
@SuppressWarnings("unchecked")
|
127
|
-
private static Class<? extends java.sql.Driver> loadJdbcDriverClassForName(final String className) throws ClassNotFoundException
|
128
|
-
{
|
129
|
-
return (Class<? extends java.sql.Driver>) Class.forName(className);
|
130
|
-
}
|
131
|
-
|
132
|
-
private static final AtomicReference<Class<? extends Driver>> mysqlJdbcDriver = new AtomicReference<>();
|
133
|
-
|
134
|
-
private static final Logger logger = LoggerFactory.getLogger(PostGresConnection.class);
|
135
|
-
|
136
|
-
protected void addDriverJarToClasspath(String glob)
|
137
|
-
{
|
138
|
-
// TODO match glob
|
139
|
-
final ClassLoader loader = getClass().getClassLoader();
|
140
|
-
if (!(loader instanceof URLClassLoader)) {
|
141
|
-
throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
|
142
|
-
}
|
143
|
-
if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
|
144
|
-
throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
|
145
|
-
}
|
146
|
-
Path path = Paths.get(glob);
|
147
|
-
if (!path.toFile().exists()) {
|
148
|
-
throw new ConfigException("The specified driver jar doesn't exist: " + glob);
|
149
|
-
}
|
150
|
-
final Method addPathMethod;
|
151
|
-
try {
|
152
|
-
addPathMethod = loader.getClass().getMethod("addPath", Path.class);
|
153
|
-
} catch (final NoSuchMethodException ex) {
|
154
|
-
throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
|
155
|
-
}
|
156
|
-
try {
|
157
|
-
addPathMethod.invoke(loader, Paths.get(glob));
|
158
|
-
} catch (final IllegalAccessException ex) {
|
159
|
-
throw new RuntimeException(ex);
|
160
|
-
} catch (final InvocationTargetException ex) {
|
161
|
-
final Throwable targetException = ex.getTargetException();
|
162
|
-
if (targetException instanceof MalformedURLException) {
|
163
|
-
throw new IllegalArgumentException(targetException);
|
164
|
-
} else if (targetException instanceof RuntimeException) {
|
165
|
-
throw (RuntimeException) targetException;
|
166
|
-
} else {
|
167
|
-
throw new RuntimeException(targetException);
|
168
|
-
}
|
169
|
-
}
|
170
|
-
}
|
171
|
-
|
172
|
-
protected File findPluginRoot()
|
173
|
-
{
|
174
|
-
try {
|
175
|
-
URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
|
176
|
-
if (url.toString().startsWith("jar:")) {
|
177
|
-
url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
|
178
|
-
}
|
179
|
-
|
180
|
-
File folder = new File(url.toURI()).getParentFile();
|
181
|
-
for (;; folder = folder.getParentFile()) {
|
182
|
-
if (folder == null) {
|
183
|
-
throw new RuntimeException("Cannot find 'embulk-filter-xxx' folder.");
|
184
|
-
}
|
185
|
-
|
186
|
-
if (folder.getName().startsWith("embulk-input-")) {
|
187
|
-
return folder;
|
188
|
-
}
|
189
|
-
}
|
190
|
-
} catch (URISyntaxException | MalformedURLException e) {
|
191
|
-
throw new RuntimeException(e);
|
192
|
-
}
|
193
|
-
}
|
194
|
-
}
|
1
|
+
package org.embulk.filter.postgress_lookup;
|
2
|
+
|
3
|
+
import org.embulk.config.ConfigException;
|
4
|
+
import org.slf4j.Logger;
|
5
|
+
import org.slf4j.LoggerFactory;
|
6
|
+
|
7
|
+
import java.io.File;
|
8
|
+
import java.io.FileFilter;
|
9
|
+
import java.lang.reflect.InvocationTargetException;
|
10
|
+
import java.lang.reflect.Method;
|
11
|
+
import java.net.MalformedURLException;
|
12
|
+
import java.net.URISyntaxException;
|
13
|
+
import java.net.URL;
|
14
|
+
import java.net.URLClassLoader;
|
15
|
+
import java.nio.file.Path;
|
16
|
+
import java.nio.file.Paths;
|
17
|
+
import java.sql.Connection;
|
18
|
+
import java.sql.Driver;
|
19
|
+
import java.sql.DriverManager;
|
20
|
+
import java.util.Optional;
|
21
|
+
import java.util.concurrent.atomic.AtomicReference;
|
22
|
+
|
23
|
+
public class PostGresConnection {
|
24
|
+
|
25
|
+
private static Connection connection=null;
|
26
|
+
|
27
|
+
private PostGresConnection(PostgressLookupFilterPlugin.PluginTask task) throws Exception {
|
28
|
+
try{
|
29
|
+
// Class.forName("org.postgresql.Driver");
|
30
|
+
if(task.getDriverClass().isPresent()){
|
31
|
+
this.loadMySqlJdbcDriver(task.getDriverClass().get(),task.getDriverPath());
|
32
|
+
}else{
|
33
|
+
this.loadMySqlJdbcDriver("org.postgresql.Driver",task.getDriverPath());
|
34
|
+
}
|
35
|
+
String url = "jdbc:postgresql://" + task.getHost() + ":"+task.getPort()+"/"+task.getDatabase();
|
36
|
+
connection= DriverManager.getConnection(url, task.getUserName(), task.getPassword());
|
37
|
+
}catch (Exception e){
|
38
|
+
e.printStackTrace();
|
39
|
+
throw new Exception(e);
|
40
|
+
}
|
41
|
+
|
42
|
+
}
|
43
|
+
|
44
|
+
public static Connection getConnection(PostgressLookupFilterPlugin.PluginTask task){
|
45
|
+
try {
|
46
|
+
if(connection==null || connection.isClosed()){
|
47
|
+
try {
|
48
|
+
new PostGresConnection(task);
|
49
|
+
return connection;
|
50
|
+
} catch (Exception e) {
|
51
|
+
e.printStackTrace();
|
52
|
+
throw new RuntimeException();
|
53
|
+
}
|
54
|
+
}
|
55
|
+
}catch (Exception e){
|
56
|
+
throw new RuntimeException(e);
|
57
|
+
}
|
58
|
+
|
59
|
+
return connection;
|
60
|
+
}
|
61
|
+
private Class<? extends java.sql.Driver> loadMySqlJdbcDriver(
|
62
|
+
final String className,
|
63
|
+
final Optional<String> driverPath)
|
64
|
+
{
|
65
|
+
synchronized (mysqlJdbcDriver) {
|
66
|
+
if (mysqlJdbcDriver.get() != null) {
|
67
|
+
return mysqlJdbcDriver.get();
|
68
|
+
}
|
69
|
+
|
70
|
+
try {
|
71
|
+
// If the class is found from the ClassLoader of the plugin, that is prioritized the highest.
|
72
|
+
final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
|
73
|
+
mysqlJdbcDriver.compareAndSet(null, found);
|
74
|
+
|
75
|
+
if (driverPath.isPresent()) {
|
76
|
+
logger.warn(
|
77
|
+
"\"driver_path\" is set while the MySQL JDBC driver class \"{}\" is found from the PluginClassLoader."
|
78
|
+
+ " \"driver_path\" is ignored.", className);
|
79
|
+
}
|
80
|
+
return found;
|
81
|
+
}
|
82
|
+
catch (final ClassNotFoundException ex) {
|
83
|
+
// Pass-through once.
|
84
|
+
}
|
85
|
+
|
86
|
+
if (driverPath.isPresent()) {
|
87
|
+
logger.info(
|
88
|
+
"\"driver_path\" is set to load the MySQL JDBC driver class \"{}\". Adding it to classpath.", className);
|
89
|
+
this.addDriverJarToClasspath(driverPath.get());
|
90
|
+
}
|
91
|
+
else {
|
92
|
+
final File root = this.findPluginRoot();
|
93
|
+
final File driverLib = new File(root, "default_jdbc_driver");
|
94
|
+
final File[] files = driverLib.listFiles(new FileFilter() {
|
95
|
+
@Override
|
96
|
+
public boolean accept(final File file)
|
97
|
+
{
|
98
|
+
return file.isFile() && file.getName().endsWith(".jar");
|
99
|
+
}
|
100
|
+
});
|
101
|
+
if (files == null || files.length == 0) {
|
102
|
+
throw new ConfigException(new ClassNotFoundException(
|
103
|
+
"The MySQL JDBC driver for the class \"" + className + "\" is not found"
|
104
|
+
+ " in \"default_jdbc_driver\" (" + root.getAbsolutePath() + ")."));
|
105
|
+
}
|
106
|
+
for (final File file : files) {
|
107
|
+
logger.info(
|
108
|
+
"The MySQL JDBC driver for the class \"{}\" is expected to be found"
|
109
|
+
+ " in \"default_jdbc_driver\" at {}.", className, file.getAbsolutePath());
|
110
|
+
this.addDriverJarToClasspath(file.getAbsolutePath());
|
111
|
+
}
|
112
|
+
}
|
113
|
+
|
114
|
+
try {
|
115
|
+
// Retrying to find the class from the ClassLoader of the plugin.
|
116
|
+
final Class<? extends java.sql.Driver> found = loadJdbcDriverClassForName(className);
|
117
|
+
mysqlJdbcDriver.compareAndSet(null, found);
|
118
|
+
return found;
|
119
|
+
}
|
120
|
+
catch (final ClassNotFoundException ex) {
|
121
|
+
throw new ConfigException("The MySQL JDBC driver for the class \"" + className + "\" is not found.", ex);
|
122
|
+
}
|
123
|
+
}
|
124
|
+
}
|
125
|
+
|
126
|
+
@SuppressWarnings("unchecked")
|
127
|
+
private static Class<? extends java.sql.Driver> loadJdbcDriverClassForName(final String className) throws ClassNotFoundException
|
128
|
+
{
|
129
|
+
return (Class<? extends java.sql.Driver>) Class.forName(className);
|
130
|
+
}
|
131
|
+
|
132
|
+
private static final AtomicReference<Class<? extends Driver>> mysqlJdbcDriver = new AtomicReference<>();
|
133
|
+
|
134
|
+
private static final Logger logger = LoggerFactory.getLogger(PostGresConnection.class);
|
135
|
+
|
136
|
+
protected void addDriverJarToClasspath(String glob)
|
137
|
+
{
|
138
|
+
// TODO match glob
|
139
|
+
final ClassLoader loader = getClass().getClassLoader();
|
140
|
+
if (!(loader instanceof URLClassLoader)) {
|
141
|
+
throw new RuntimeException("Plugin is not loaded by URLClassLoader unexpectedly.");
|
142
|
+
}
|
143
|
+
if (!"org.embulk.plugin.PluginClassLoader".equals(loader.getClass().getName())) {
|
144
|
+
throw new RuntimeException("Plugin is not loaded by PluginClassLoader unexpectedly.");
|
145
|
+
}
|
146
|
+
Path path = Paths.get(glob);
|
147
|
+
if (!path.toFile().exists()) {
|
148
|
+
throw new ConfigException("The specified driver jar doesn't exist: " + glob);
|
149
|
+
}
|
150
|
+
final Method addPathMethod;
|
151
|
+
try {
|
152
|
+
addPathMethod = loader.getClass().getMethod("addPath", Path.class);
|
153
|
+
} catch (final NoSuchMethodException ex) {
|
154
|
+
throw new RuntimeException("Plugin is not loaded a ClassLoader which has addPath(Path), unexpectedly.");
|
155
|
+
}
|
156
|
+
try {
|
157
|
+
addPathMethod.invoke(loader, Paths.get(glob));
|
158
|
+
} catch (final IllegalAccessException ex) {
|
159
|
+
throw new RuntimeException(ex);
|
160
|
+
} catch (final InvocationTargetException ex) {
|
161
|
+
final Throwable targetException = ex.getTargetException();
|
162
|
+
if (targetException instanceof MalformedURLException) {
|
163
|
+
throw new IllegalArgumentException(targetException);
|
164
|
+
} else if (targetException instanceof RuntimeException) {
|
165
|
+
throw (RuntimeException) targetException;
|
166
|
+
} else {
|
167
|
+
throw new RuntimeException(targetException);
|
168
|
+
}
|
169
|
+
}
|
170
|
+
}
|
171
|
+
|
172
|
+
protected File findPluginRoot()
|
173
|
+
{
|
174
|
+
try {
|
175
|
+
URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
|
176
|
+
if (url.toString().startsWith("jar:")) {
|
177
|
+
url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
|
178
|
+
}
|
179
|
+
|
180
|
+
File folder = new File(url.toURI()).getParentFile();
|
181
|
+
for (;; folder = folder.getParentFile()) {
|
182
|
+
if (folder == null) {
|
183
|
+
throw new RuntimeException("Cannot find 'embulk-filter-xxx' folder.");
|
184
|
+
}
|
185
|
+
|
186
|
+
if (folder.getName().startsWith("embulk-input-")) {
|
187
|
+
return folder;
|
188
|
+
}
|
189
|
+
}
|
190
|
+
} catch (URISyntaxException | MalformedURLException e) {
|
191
|
+
throw new RuntimeException(e);
|
192
|
+
}
|
193
|
+
}
|
194
|
+
}
|
@@ -218,7 +218,7 @@ public class PostgressLookupFilterPlugin
|
|
218
218
|
|
219
219
|
for (Column column : inputSchema.getColumns()) {
|
220
220
|
if (reader.isNull(column)) {
|
221
|
-
if (column.getName().equalsIgnoreCase(inputColumns.get(keyMap.get("Key")))) {
|
221
|
+
if ((keyMap.get("Key") < inputColumns.size()) && column.getName().equalsIgnoreCase(inputColumns.get(keyMap.get("Key")))) {
|
222
222
|
searchingKeyData.add("");
|
223
223
|
int key = keyMap.get("Key");
|
224
224
|
keyMap.put("Key", ++key);
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-postgres_lookup
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Infoobjects Inc.
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-02-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -50,7 +50,7 @@ files:
|
|
50
50
|
- README.md
|
51
51
|
- build.gradle
|
52
52
|
- calendar.csv
|
53
|
-
- classpath/embulk-filter-postgres_lookup-0.1.
|
53
|
+
- classpath/embulk-filter-postgres_lookup-0.1.6.jar
|
54
54
|
- classpath/mysql-connector-java-8.0.19.jar
|
55
55
|
- classpath/postgresql-9.0-801.jdbc4.jar
|
56
56
|
- classpath/protobuf-java-3.6.1.jar
|
@@ -66,11 +66,11 @@ files:
|
|
66
66
|
- src/main/java/org/embulk/filter/postgress_lookup/PostGresConnection.java
|
67
67
|
- src/main/java/org/embulk/filter/postgress_lookup/PostgressLookupFilterPlugin.java
|
68
68
|
- src/test/java/org/embulk/filter/postgress_lookup/TestPostgressLookupFilterPlugin.java
|
69
|
-
homepage:
|
69
|
+
homepage:
|
70
70
|
licenses:
|
71
71
|
- MIT
|
72
72
|
metadata: {}
|
73
|
-
post_install_message:
|
73
|
+
post_install_message:
|
74
74
|
rdoc_options: []
|
75
75
|
require_paths:
|
76
76
|
- lib
|
@@ -85,8 +85,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
85
85
|
- !ruby/object:Gem::Version
|
86
86
|
version: '0'
|
87
87
|
requirements: []
|
88
|
-
|
89
|
-
|
88
|
+
rubyforge_project:
|
89
|
+
rubygems_version: 2.7.6
|
90
|
+
signing_key:
|
90
91
|
specification_version: 4
|
91
92
|
summary: Postgress Lookup filter plugin for Embulk
|
92
93
|
test_files: []
|