embulk-output-jdbc 0.7.12 → 0.7.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae6b599749ebe50a9b832c6ba8d0784a6737cb7c
-  data.tar.gz: 43c7475f21f28132563ba0a6592e7fbedb4790e2
+  metadata.gz: 411a4c67839886d900bdf7bb5a47216f365103f4
+  data.tar.gz: 53967bc25669370087fa99085b5975785a093630
 SHA512:
-  metadata.gz: 5bb01073bf551b713858e89f0f7364c428457c51107c25f357d962573ca5556f3623d0d4462b0d44fb75506c936853996c81684c639c7073b0e9921cb54be33b
-  data.tar.gz: 468e740668ce4e9f03edfc001703fcb21017f5380f9fe00f3de71c1de53fe35ce4ee580517c5f8a07cd45ee6fd165633b6994a4999bf2a5db039a09101094ae3
+  metadata.gz: 461685691438c570b0ffebd33954cfb24f086b39cac220ecc7483aa0ed678af5f75240019f5f184f0dd5119180c1147e2eba43e3a6e1fcdb39b1331644502093
+  data.tar.gz: 4ed522ff2ae37f35e95c91122e3a146a24e095f9c74f821ded1ee6110c1affbeafc68cfb79c485c37b6750ba7118d15a2d6f63fd69b68db1b0c302819ba3a893

data/README.md

@@ -17,6 +17,8 @@ Generic JDBC output plugin for Embulk loads records to a database using a JDBC d
 - **password**: database login password (string, optional)
 - **schema**: destination schema name (string, default: use default schema)
 - **table**: destination table name (string, required)
+- **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **options**: extra JDBC properties (hash, default: {})
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))

data/src/main/java/org/embulk/output/JdbcOutputPlugin.java

@@ -67,7 +67,7 @@ public class JdbcOutputPlugin
         GenericPluginTask t = (GenericPluginTask) task;
 
         if (t.getDriverPath().isPresent()) {
-            loadDriverJar(t.getDriverPath().get());
+            addDriverJarToClasspath(t.getDriverPath().get());
         }
 
         Properties props = new Properties();

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -1,13 +1,18 @@
 package org.embulk.output.jdbc;
 
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Locale;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
+import java.io.File;
+import java.io.FileFilter;
 import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.charset.Charset;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.sql.Types;
 import java.sql.ResultSet;
@@ -47,7 +52,6 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.output.jdbc.setter.ColumnSetter;
 import org.embulk.output.jdbc.setter.ColumnSetterFactory;
 import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
@@ -59,8 +63,6 @@ import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
 public abstract class AbstractJdbcOutputPlugin
         implements OutputPlugin
 {
-    private final static Set<String> loadedJarGlobs = new HashSet<String>();
-
     protected final Logger logger = Exec.getLogger(getClass());
 
     public interface PluginTask
@@ -89,6 +91,14 @@ public abstract class AbstractJdbcOutputPlugin
         @ConfigDefault("{}")
         public Map<String, JdbcColumnOption> getColumnOptions();
 
+        @Config("create_table_constraint")
+        @ConfigDefault("null")
+        public Optional<String> getCreateTableConstraint();
+
+        @Config("create_table_option")
+        @ConfigDefault("null")
+        public Optional<String> getCreateTableOption();
+
         @Config("default_timezone")
         @ConfigDefault("\"UTC\"")
         public DateTimeZone getDefaultTimeZone();
@@ -217,16 +227,52 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
-    protected void loadDriverJar(String glob)
+    protected void addDriverJarToClasspath(String glob)
     {
-        synchronized (loadedJarGlobs) {
-            if (!loadedJarGlobs.contains(glob)) {
-                // TODO match glob
-                PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
-                loader.addPath(Paths.get(glob));
-                loadedJarGlobs.add(glob);
+        // TODO match glob
+        PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+             throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
+        loader.addPath(Paths.get(glob));
+    }
+
+    protected void loadDriver(String className, Optional<String> driverPath)
+    {
+        if (driverPath.isPresent()) {
+            addDriverJarToClasspath(driverPath.get());
+        } else {
+            try {
+                // Gradle test task will add JDBC driver to classpath
+                Class.forName(className);
+
+            } catch (ClassNotFoundException ex) {
+                File root = findPluginRoot(getClass());
+                File driverLib = new File(root, "default_jdbc_driver");
+                File[] files = driverLib.listFiles(new FileFilter() {
+                    @Override
+                    public boolean accept(File file) {
+                        return file.isFile() && file.getName().endsWith(".jar");
+                    }
+                });
+                if (files == null || files.length == 0) {
+                    throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
+                } else {
+                    for (File file : files) {
+                        logger.info("JDBC Driver = " + file.getAbsolutePath());
+                        addDriverJarToClasspath(file.getAbsolutePath());
+                    }
+                }
             }
         }
+
+        // Load JDBC Driver
+        try {
+            Class.forName(className);
+        } catch (ClassNotFoundException ex) {
+            throw new RuntimeException(ex);
+        }
     }
 
     // for subclasses to add @Config
@@ -372,13 +418,6 @@ public abstract class AbstractJdbcOutputPlugin
         return commit(task, schema, taskCount);
     }
 
-    protected String getTransactionUniqueName()
-    {
-        // TODO use uuid?
-        Timestamp t = Exec.session().getTransactionTime();
-        return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
-    }
-
     private PluginTask begin(final PluginTask task,
             final Schema schema, final int taskCount)
     {
@@ -525,7 +564,7 @@ public abstract class AbstractJdbcOutputPlugin
         } else {
             // also create the target table if not exists
             // CREATE TABLE IF NOT EXISTS xyz
-            con.createTableIfNotExists(task.getActualTable(), newTableSchema);
+            con.createTableIfNotExists(task.getActualTable(), newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
             targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getActualTable()).get();
             task.setNewTableSchema(Optional.<JdbcSchema>absent());
         }
@@ -598,17 +637,17 @@ public abstract class AbstractJdbcOutputPlugin
                         String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 3,
                                 task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
                         for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
-                            String tableName = namePrefix + String.format("%03d", taskIndex);
+                            String tableName = namePrefix + String.format("%03d", taskIndex % 1000);
                             table = buildIntermediateTableId(con, task, tableName);
                             // if table already exists, SQLException will be thrown
-                            con.createTable(table, newTableSchema);
+                            con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
                             intermTables.add(table);
                         }
                     } else {
                         String tableName = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 0,
                                 task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
                         table = buildIntermediateTableId(con, task, tableName);
-                        con.createTable(table, newTableSchema);
+                        con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
                         intermTables.add(table);
                     }
                     return intermTables.build();
@@ -667,8 +706,8 @@ public abstract class AbstractJdbcOutputPlugin
     {
         Charset tableNameCharset = con.getTableNameCharset();
         String tableName = baseTableName;
-        String suffix = "_bl_tmp";
-        String uniqueSuffix = getTransactionUniqueName() + suffix;
+        String suffix = "_embulk";
+        String uniqueSuffix = String.format("%016x", System.currentTimeMillis()) + suffix;
 
         // way to count length of table name varies by DBMSs (bytes or characters),
         // so truncate swap table name by one character.
@@ -766,7 +805,8 @@ public abstract class AbstractJdbcOutputPlugin
         case INSERT:
             // aggregate insert into target
             if (task.getNewTableSchema().isPresent()) {
-                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
+                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
+                        task.getCreateTableConstraint(), task.getCreateTableOption());
             }
             con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), false, task.getBeforeLoad(), task.getAfterLoad());
             break;
@@ -774,7 +814,8 @@ public abstract class AbstractJdbcOutputPlugin
         case TRUNCATE_INSERT:
             // truncate & aggregate insert into target
             if (task.getNewTableSchema().isPresent()) {
-                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
+                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
+                        task.getCreateTableConstraint(), task.getCreateTableOption());
             }
             con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), true, task.getBeforeLoad(), task.getAfterLoad());
             break;
@@ -782,7 +823,8 @@ public abstract class AbstractJdbcOutputPlugin
         case MERGE:
             // aggregate merge into target
             if (task.getNewTableSchema().isPresent()) {
-                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
+                con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
+                        task.getCreateTableConstraint(), task.getCreateTableOption());
             }
             con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(),
                     new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()), task.getBeforeLoad(), task.getAfterLoad());
@@ -987,6 +1029,29 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
+    public static File findPluginRoot(Class<?> cls)
+    {
+        try {
+            URL url = cls.getResource("/" + cls.getName().replace('.', '/') + ".class");
+            if (url.toString().startsWith("jar:")) {
+                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
+            }
+
+            File folder = new File(url.toURI()).getParentFile();
+            for (;; folder = folder.getParentFile()) {
+                if (folder == null) {
+                    throw new RuntimeException("Cannot find 'embulk-output-xxx' folder.");
+                }
+
+                if (folder.getName().startsWith("embulk-output-")) {
+                    return folder;
+                }
+            }
+        } catch (MalformedURLException | URISyntaxException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public class PluginPageOutput
             implements TransactionalPageOutput
     {

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -85,6 +85,11 @@ public class JdbcOutputConnection
         return tableExists(new TableIdentifier(null, schemaName, tableName));
     }
 
+    protected boolean supportsTableIfExistsClause()
+    {
+        return true;
+    }
+
     public void dropTableIfExists(TableIdentifier table) throws SQLException
     {
         Statement stmt = connection.createStatement();
@@ -100,8 +105,14 @@ public class JdbcOutputConnection
 
     protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
     {
-        String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
-        executeUpdate(stmt, sql);
+        if (supportsTableIfExistsClause()) {
+            String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
+            executeUpdate(stmt, sql);
+        } else {
+            if (tableExists(table)) {
+                dropTable(stmt, table);
+            }
+        }
     }
 
     public void dropTable(TableIdentifier table) throws SQLException
@@ -123,35 +134,48 @@ public class JdbcOutputConnection
         executeUpdate(stmt, sql);
     }
 
-    public void createTableIfNotExists(TableIdentifier targetTable, JdbcSchema schema) throws SQLException
+    public void createTableIfNotExists(TableIdentifier table, JdbcSchema schema,
+            Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
     {
-        Statement stmt = connection.createStatement();
-        try {
-            String sql = buildCreateTableIfNotExistsSql(targetTable, schema);
-            executeUpdate(stmt, sql);
-            commitIfNecessary(connection);
-        } catch (SQLException ex) {
-            throw safeRollback(connection, ex);
-        } finally {
-            stmt.close();
+        if (supportsTableIfExistsClause()) {
+            Statement stmt = connection.createStatement();
+            try {
+                String sql = buildCreateTableIfNotExistsSql(table, schema, tableConstraint, tableOption);
+                executeUpdate(stmt, sql);
+                commitIfNecessary(connection);
+            } catch (SQLException ex) {
+                throw safeRollback(connection, ex);
+            } finally {
+                stmt.close();
+            }
+        } else {
+            if (!tableExists(table)) {
+                createTable(table, schema, tableConstraint, tableOption);
+            }
         }
     }
 
-    protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema)
+    protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema,
+            Optional<String> tableConstraint, Optional<String> tableOption)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("CREATE TABLE IF NOT EXISTS ");
         quoteTableIdentifier(sb, table);
-        sb.append(buildCreateTableSchemaSql(schema));
+        sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
+        if (tableOption.isPresent()) {
+            sb.append(" ");
+            sb.append(tableOption.get());
+        }
         return sb.toString();
     }
 
-    public void createTable(TableIdentifier table, JdbcSchema schema) throws SQLException
+    public void createTable(TableIdentifier table, JdbcSchema schema,
+            Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
     {
         Statement stmt = connection.createStatement();
         try {
-            String sql = buildCreateTableSql(table, schema);
+            String sql = buildCreateTableSql(table, schema, tableConstraint, tableOption);
             executeUpdate(stmt, sql);
             commitIfNecessary(connection);
         } catch (SQLException ex) {
@@ -161,28 +185,37 @@ public class JdbcOutputConnection
         }
     }
 
-    protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema)
+    protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema,
+            Optional<String> tableConstraint, Optional<String> tableOption)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("CREATE TABLE ");
         quoteTableIdentifier(sb, table);
-        sb.append(buildCreateTableSchemaSql(schema));
+        sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
+        if (tableOption.isPresent()) {
+            sb.append(" ");
+            sb.append(tableOption.get());
+        }
         return sb.toString();
     }
 
-    protected String buildCreateTableSchemaSql(JdbcSchema schema)
+    protected String buildCreateTableSchemaSql(JdbcSchema schema, Optional<String> tableConstraint)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append(" (");
-        for (int i=0; i < schema.getCount(); i++) {
+        for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, schema.getColumnName(i));
             sb.append(" ");
             String typeName = getCreateTableTypeName(schema.getColumn(i));
             sb.append(typeName);
         }
+        if (tableConstraint.isPresent()) {
+            sb.append(", ");
+            sb.append(tableConstraint.get());
+        }
         sb.append(")");
 
         return sb.toString();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.7.12
+  version: 0.7.13
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-24 00:00:00.000000000 Z
+date: 2017-12-08 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,7 +19,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.12.jar
+- classpath/embulk-output-jdbc-0.7.13.jar
 - lib/embulk/output/jdbc.rb
 - src/main/java/org/embulk/output/JdbcOutputPlugin.java
 - src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java