embulk-output-jdbc 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b00ca29d7e5aac5ac1369fe312807937c0953336
-  data.tar.gz: 7fd2236f250fd4d24e825a3d974326e60f163b76
+  metadata.gz: ed0f3d732ac1fa2ee1d660480be8587ac137d3f0
+  data.tar.gz: c534795f00578dc8a30e580f9b34a8252e5cc5d4
 SHA512:
-  metadata.gz: 3f09b8a3a96a5df1fb9bb7a00619399fccaca549be880f7729ed155f146f5f6a06ac394cea4187b059b7a938115b9986e18e3ccefef5cccb6a32e1d23c9ea64d
-  data.tar.gz: 61d57fc29b2816f9809b1dbacc807b55569ec252e4ad55f442b392777738b0732fdf853da1918c7f0f14f8c0e8fd6953676ae35e312ddc3771f47232bbf243b9
+  metadata.gz: 57157af5fd5be3ceeab92d8dbf36c9f031e2a1ed5bb8f2ae78bf0d91629efdcaf0523de04bbeed799633f40596c4159d8ef589c267b0693d6acd591973b6ee1c
+  data.tar.gz: cbfa1e7646587f442a233f946bde40fe3525414713727333bbe98720408d71067b7e2e60e88358d9d86ac367ed58d1744f4f876e2e7357f5ae3f1009ac5c3786

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -9,7 +9,6 @@ import java.util.concurrent.ExecutionException;
 import java.io.IOException;
 import java.nio.file.Paths;
 import java.sql.Types;
-import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
@@ -32,7 +31,6 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.PageOutput;
 import org.embulk.spi.PluginClassLoader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
@@ -128,7 +126,12 @@ public abstract class AbstractJdbcOutputPlugin
 
         public boolean isInplace()
         {
-            return this == INSERT_DIRECT || this == REPLACE_INPLACE;
+            return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
+        }
+
+        public boolean isMerge()
+        {
+            return this == MERGE;
         }
 
         public boolean usesMultipleLoadTables()
@@ -156,8 +159,11 @@ public abstract class AbstractJdbcOutputPlugin
         case "replace":
             task.setMode(Mode.REPLACE_INPLACE);
             break;
+        case "merge":
+            task.setMode(Mode.MERGE);
+            break;
         default:
-            throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace", task.getModeConfig()));
+            throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
         }
 
         //switch(task.getModeConfig()) {
@@ -383,35 +389,35 @@ public abstract class AbstractJdbcOutputPlugin
                 {
                     columns.add(new JdbcColumn(
                             columnName, "BOOLEAN",
-                            Types.BOOLEAN, 1, 0));
+                            Types.BOOLEAN, 1, 0, false));
                 }
 
                 public void longColumn(Column column)
                 {
                     columns.add(new JdbcColumn(
                             columnName, "BIGINT",
-                            Types.BIGINT, 22, 0));
+                            Types.BIGINT, 22, 0, false));
                 }
 
                 public void doubleColumn(Column column)
                 {
                     columns.add(new JdbcColumn(
                             columnName, "DOUBLE PRECISION",
-                            Types.FLOAT, 24, 0));
+                            Types.FLOAT, 24, 0, false));
                 }
 
                 public void stringColumn(Column column)
                 {
                     columns.add(new JdbcColumn(
                                 columnName, "CLOB",
-                                Types.CLOB, 4000, 0));  // TODO size type param
+                                Types.CLOB, 4000, 0, false));  // TODO size type param
                 }
 
                 public void timestampColumn(Column column)
                 {
                     columns.add(new JdbcColumn(
                                 columnName, "TIMESTAMP",
-                                Types.TIMESTAMP, 26, 0));  // size type param is from postgresql.
+                                Types.TIMESTAMP, 26, 0, false));  // size type param is from postgresql.
                 }
             });
         }
@@ -423,15 +429,27 @@ public abstract class AbstractJdbcOutputPlugin
     {
         DatabaseMetaData dbm = connection.getMetaData();
         String escape = dbm.getSearchStringEscape();
-
-        ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
         String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
+
+        ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
+        ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
+        try {
+            while(rs.next()) {
+                primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
+            }
+        } finally {
+            rs.close();
+        }
+        ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
+
         String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
-        ResultSet rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
+        ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
+        rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
         try {
             while(rs.next()) {
                 String columnName = rs.getString("COLUMN_NAME");
                 String typeName = rs.getString("TYPE_NAME");
+                boolean isPrimaryKey = primaryKeys.contains(columnName);
                 typeName = typeName.toUpperCase(Locale.ENGLISH);
                 int sqlType = rs.getInt("DATA_TYPE");
                 int colSize = rs.getInt("COLUMN_SIZE");
@@ -443,7 +461,7 @@ public abstract class AbstractJdbcOutputPlugin
                 //rs.getString("COLUMN_DEF") // or null  // TODO
                 columns.add(new JdbcColumn(
                             columnName, typeName,
-                            sqlType, colSize, decDigit));
+                            sqlType, colSize, decDigit, isPrimaryKey));
             }
         } finally {
             rs.close();
@@ -522,12 +540,11 @@ public abstract class AbstractJdbcOutputPlugin
                         // insert_direct
                         loadTable = task.getTable();
                     }
-
                     b.prepare(loadTable, insertSchema);
                 }
             });
 
-            PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters.build(),
+            PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(),
                     task.getBatchSize());
             batch = null;
             return output;
@@ -552,13 +569,20 @@ public abstract class AbstractJdbcOutputPlugin
     	return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
     }
 
+    protected PluginPageOutput newPluginPageOutput(PageReader reader,
+                                                   BatchInsert batch, List<ColumnSetter> columnSetters,
+                                                   int batchSize)
+    {
+        return new PluginPageOutput(reader, batch, columnSetters, batchSize);
+    }
+
     public static class PluginPageOutput
             implements TransactionalPageOutput
     {
+        protected final List<Column> columns;
+        protected final List<ColumnSetter> columnSetters;
         private final PageReader pageReader;
         private final BatchInsert batch;
-        private final List<Column> columns;
-        private final List<ColumnSetter> columnSetters;
         private final int batchSize;
         private final int foraceBatchFlushSize;
 
@@ -583,9 +607,7 @@ public abstract class AbstractJdbcOutputPlugin
                     if (batch.getBatchWeight() > foraceBatchFlushSize) {
                         batch.flush();
                     }
-                    for (int i=0; i < columnSetters.size(); i++) {
-                        columns.get(i).visit(columnSetters.get(i));
-                    }
+                    handleColumnsSetters();
                     batch.add();
                 }
                 if (batch.getBatchWeight() > batchSize) {
@@ -626,6 +648,15 @@ public abstract class AbstractJdbcOutputPlugin
         {
             return Exec.newCommitReport();
         }
+
+        protected void handleColumnsSetters()
+        {
+            int size = columnSetters.size();
+            for (int i=0; i < size; i++) {
+                columns.get(i).visit(columnSetters.get(i));
+            }
+        }
+
     }
 
     public static interface IdempotentSqlRunnable

data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java

@@ -11,6 +11,7 @@ public class JdbcColumn
     private int sqlType;
     private int sizeTypeParameter;
     private int scaleTypeParameter;
+    private boolean isPrimaryKey;
 
     @JsonCreator
     public JdbcColumn(
@@ -18,19 +19,21 @@ public class JdbcColumn
             @JsonProperty("typeName") String typeName,
             @JsonProperty("sqlType") int sqlType,
             @JsonProperty("sizeTypeParameter") int sizeTypeParameter,
-            @JsonProperty("scaleTypeParameter") int scaleTypeParameter)
+            @JsonProperty("scaleTypeParameter") int scaleTypeParameter,
+            @JsonProperty("primaryKey") boolean isPrimaryKey)
     {
         this.name = name;
         this.typeName = typeName;
         this.sqlType = sqlType;
         this.sizeTypeParameter = sizeTypeParameter;
         this.scaleTypeParameter = scaleTypeParameter;
+        this.isPrimaryKey = isPrimaryKey;
     }
 
     @JsonIgnore
     public static JdbcColumn skipColumn()
     {
-        return new JdbcColumn(null, null, 0, 0, 0);
+        return new JdbcColumn(null, null, 0, 0, 0, false);
     }
 
     @JsonIgnore
@@ -39,6 +42,12 @@ public class JdbcColumn
         return name == null;
     }
 
+    @JsonProperty("primaryKey")
+    public boolean isPrimaryKey()
+    {
+        return isPrimaryKey;
+    }
+
     @JsonProperty("name")
     public String getName()
     {

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -1,6 +1,5 @@
 package org.embulk.output.jdbc;
 
-import java.util.List;
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
 import java.sql.PreparedStatement;
@@ -66,7 +65,7 @@ public class JdbcOutputConnection
             return rs.next();
         }
     }
-    
+
     public void dropTableIfExists(String tableName) throws SQLException
     {
         Statement stmt = connection.createStatement();
@@ -152,7 +151,7 @@ public class JdbcOutputConnection
         sb.append(buildColumnsOfCreateTableSql(schema));
         return sb.toString();
     }
-    
+
     private String buildColumnsOfCreateTableSql(JdbcSchema schema)
     {
         StringBuilder sb = new StringBuilder();
@@ -306,6 +305,13 @@ public class JdbcOutputConnection
         return connection.prepareStatement(insertSql);
     }
 
+    public PreparedStatement prepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
+    {
+        String upsertSql = buildPrepareUpsertSql(toTable, toTableSchema);
+        logger.info("Prepared SQL: {}", upsertSql);
+        return connection.prepareStatement(upsertSql);
+    }
+
     protected String buildPrepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
@@ -328,6 +334,11 @@ public class JdbcOutputConnection
         return sb.toString();
     }
 
+    protected String buildPrepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
+    {
+        throw new UnsupportedOperationException("not implemented yet");
+    }
+
     // TODO
     //protected void gatherInsertTables(List<String> fromTables, JdbcSchema fromTableSchema, String toTable,
     //        boolean truncateDestinationFirst) throws SQLException

data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java

@@ -32,13 +32,19 @@ public class StandardBatchInsert
     public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
     {
         this.connection = connector.connect(true);
-        this.batch = connection.prepareInsertSql(loadTable, insertSchema);
+        this.batch = newPreparedStatement(connection, loadTable, insertSchema);
         this.index = 1;  // PreparedStatement index begings from 1
         this.batchRows = 0;
         this.totalRows = 0;
         batch.clearBatch();
     }
 
+    protected PreparedStatement newPreparedStatement(JdbcOutputConnection connection,
+                                                     String loadTable, JdbcSchema insertSchema) throws SQLException
+    {
+        return connection.prepareInsertSql(loadTable, insertSchema);
+    }
+
     public int getBatchWeight()
     {
         return batchWeight;

data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java

@@ -24,6 +24,11 @@ public abstract class ColumnSetter
         this.column = column;
     }
 
+    public JdbcColumn getColumn()
+    {
+        return column;
+    }
+
     public int getSqlType()
     {
         return column.getSqlType();

data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java

@@ -2,7 +2,6 @@ package org.embulk.output.jdbc.setter;
 
 import org.embulk.spi.PageReader;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.BatchInsert;
 
 public class SkipColumnSetter
@@ -32,4 +31,8 @@ public class SkipColumnSetter
     protected void timestampValue(Timestamp v)
     {
     }
+
+    protected void nullValue()
+    {
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-07 00:00:00.000000000 Z
+date: 2015-04-23 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -39,7 +39,7 @@ files:
 - src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java
 - src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java
 - src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
-- classpath/embulk-output-jdbc-0.2.2.jar
+- classpath/embulk-output-jdbc-0.2.3.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0