embulk-output-postgres-udf 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8c33d61b3ca4584ca6de3ce9ccb1edb0cf155fa1
-  data.tar.gz: d9b8c3834e1bbe1f8112d27f1d322e85651eaafd
+  metadata.gz: 37c4908db6315bd68cc4dd25ba11342740de1a23
+  data.tar.gz: ce34918cfbcb005d90100fac17bc71852b0910df
 SHA512:
-  metadata.gz: 7622ffeb0048c75f3a3417c09e2ad9b9de5981b7f85bf624c6f2df01d50b9f3be15980b50f860642743912b652b02800375ba3a0e0c0a44c34bb630849661151
-  data.tar.gz: 8e113b89669dbc41ac37fa607c54769d22bcb2cd4d39ac7f1b945a416c4ee5d1020c8055be9e9b4b645254fd5588f701a82583802ae1662b3acfe46b353d1268
+  metadata.gz: cfc2a0da351d5d809717d278bfcd47412caf4659ac647e0a25c982795a2ccf2bce8c238c5bc4e743918d4f0be31e0815a0944687499a1372517c141ef0ed9562
+  data.tar.gz: 63b18e4c1848b01a3bc60c6da9095cc669d94d997d97a2232fd239f60c212cafe82a9061ef2b55b21b9f118192b0335538f87c74dad2aa3d5dfdb6f2096c8856

data/.travis.yml

@@ -0,0 +1,17 @@
+language: java
+addons:
+  postgresql: 9.4
+jdk:
+  - oraclejdk7
+  - oraclejdk8
+env:
+  - version=0.7.4
+before_install:
+  - curl --create-dirs -o .embulk/bin/embulk -L "https://bintray.com/artifact/download/embulk/maven/embulk-${version}.jar"
+  - chmod +x .embulk/bin/embulk
+  - export PATH=".embulk/bin:$PATH"
+before_script:
+  - ./gradlew clean gem
+  - psql -c 'create database travis_ci_test;' -U postgres
+script:
+  - ls ci/*.yml | xargs -I% embulk run -I lib %

data/build.gradle

@@ -12,15 +12,21 @@ configurations {
     provided
 }
 
-compileJava {
-    sourceCompatibility = '1.7'
+version = "0.2.0"
+
+ext {
+    javaVersion = '1.7'
 }
 
-version = "0.1.2"
+compileJava {
+    sourceCompatibility = javaVersion
+    targetCompatibility = javaVersion
+    options.compilerArgs += ['-source', javaVersion, '-target', javaVersion]
+}
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.5.4"
-    provided "org.embulk:embulk-core:0.5.4"
+    compile  "org.embulk:embulk-core:0.7.4"
+    provided "org.embulk:embulk-core:0.7.4"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile 'org.postgresql:postgresql:9.4-1200-jdbc41'
     testCompile "junit:junit:4.+"

data/ci/config.yml

@@ -0,0 +1,37 @@
+in:
+  type: file
+  path_prefix: ./ci/csv/sample_
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    escape: ''
+    skip_header_lines: 1
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+out:
+  type: postgres_udf
+  host: localhost
+  user: postgres
+  database: travis_ci_test
+  function: |
+    begin
+      begin
+        create table if not exists sample (
+          id integer,
+          account integer,
+          time timestamp,
+          purchase timestamp,
+          comment text
+        );
+      exception
+        when unique_violation then -- do nothing
+      end;
+      insert into sample values(id, account, time, purchase, comment);
+    end;

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Feb 04 13:46:12 PST 2015
+#Tue Aug 11 00:26:20 PDT 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/gradlew.bat

@@ -1,90 +1,90 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/output/PostgresUDFConnector.java

@@ -1,8 +1,13 @@
 package org.embulk.output;
 
 import com.google.common.base.Throwables;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
 
-import java.sql.*;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.SQLException;
 import java.util.Properties;
 
 /**
@@ -10,6 +15,8 @@ import java.util.Properties;
  */
 public class PostgresUDFConnector {
     private static final Driver driver = new org.postgresql.Driver();
+    private final Logger log = Exec.getLogger(PostgresUDFOutputPlugin.class);
+
 
     private final String url;
     private final Properties properties;
@@ -35,25 +42,47 @@ public class PostgresUDFConnector {
     }
 
     private Connection createConnection() throws SQLException {
-        SQLException firstException = null;
-        int count = 0;
-        while(count < 10) {
-            count++;
+        try {
+            return RetryExecutor.retryExecutor()
+                    .withRetryLimit(12)
+                    .withInitialRetryWait(1000)
+                    .withMaxRetryWait(30 * 60 * 1000)
+                    .runInterruptible(new RetryExecutor.Retryable<Connection>() {
+                        @Override
+                        public Connection call() throws Exception {
+                            return driver.connect(url, properties);
+                        }
 
-            try {
-                return driver.connect(url, properties);
-            }
-            catch (SQLRecoverableException | SQLTimeoutException e) {
-                if (firstException == null) {
-                    firstException = e;
-                }
-                try {
-                    Thread.sleep(300);
-                } catch (InterruptedException e2) {
-                    throw Throwables.propagate(e2);
-                }
-            }
+                        @Override
+                        public boolean isRetryableException(Exception exception) {
+                            /*
+                            if (exception instanceof SQLRecoverableException) return true;
+                            if (exception instanceof SQLTimeoutException) return true;
+                            if (exception instanceof ConnectException) return true;
+                            */
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException {
+                            String message = String.format("connection failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            } else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException {
+                        }
+                    });
+        } catch (InterruptedException e) {
+            throw new RuntimeException(e);
+        } catch (RetryExecutor.RetryGiveupException e) {
+            Throwables.propagateIfInstanceOf(e.getCause(), SQLException.class);
+            throw Throwables.propagate(e.getCause());
         }
-        throw firstException;
     }
 }

data/src/main/java/org/embulk/output/PostgresUDFOutputPlugin.java

@@ -76,7 +76,7 @@ public class PostgresUDFOutputPlugin implements OutputPlugin {
     }
 
     @Override
-    public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<CommitReport> successCommitReports) {
+    public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
         PluginTask task = taskSource.loadTask(PluginTask.class);
         try {
             ConnectionWrapper con = getConnector(task).connect(true);
@@ -197,14 +197,14 @@ public class PostgresUDFOutputPlugin implements OutputPlugin {
         }
 
         @Override
-        public CommitReport commit() {
+        public TaskReport commit() {
             try {
                 connection.commit();
             }
             catch (Exception e) {
                 throw Throwables.propagate(e);
             }
-            return Exec.newCommitReport();
+            return Exec.newTaskReport();
         }
 
     }

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-postgres-udf
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - kakusuke
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-03 00:00:00.000000000 Z
+date: 2015-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Dumps records to PostgreSQL via user-defined function.
 email:
 - bananafishmonger+github@gmail.com
@@ -46,9 +46,12 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
+- ci/config.yml
+- ci/csv/sample_01.csv.gz
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -58,7 +61,7 @@ files:
 - src/main/java/org/embulk/output/PostgresUDFConnector.java
 - src/main/java/org/embulk/output/PostgresUDFOutputPlugin.java
 - src/test/java/org/embulk/output/TestPostgresUDFOutputPlugin.java
-- classpath/embulk-output-postgres-udf-0.1.2.jar
+- classpath/embulk-output-postgres-udf-0.2.0.jar
 - classpath/jna-4.1.0.jar
 - classpath/jna-platform-4.1.0.jar
 - classpath/postgresql-9.4-1200-jdbc41.jar