embulk-output-sqlserver 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76d8fb06a79d4382fab1f378b165ab10518fe9ef
-  data.tar.gz: cc766e5eeff96a7a1e74e00893a01beb58a13e83
+  metadata.gz: 107530637391bd44da5f7291004ccef1e1ba0a3b
+  data.tar.gz: a89e2a72c151e984261519e31d879d5449db9060
 SHA512:
-  metadata.gz: e39ecf1f59e3b3c555733821925e0a098be114b1d15844c6bcb1f8349de93168cb46bbfc999f427f5231f061d89bfae509e875d9d480e366224e639385dccd54
-  data.tar.gz: 45419b194e670602d291ff1e1ee5ecedb0c7a6d9b236340e056ce82a37b1cdc7838dd01cc03ebafa41470f10b5cef673ea25f85a902b4a625ffaab7501a00968
+  metadata.gz: b6a2ce12e01cbe02743d0ca42de24bf1105ab3ff40ab333cf56f198877ca967577792af1f8b3d6ee0fdaae02894788b7aef66a6aabed86db0c99883459c7b6f5
+  data.tar.gz: 96aeaf1f4c005375a4ce991f6dc9eef502bc896aa3ddb69ed2c492c34809e1b5468726a71adbc7d793a8895988e442db87ff9d3e88fefb8eb9ad6d0a6d7ddfd8

data/README.md

@@ -5,8 +5,8 @@ SQL Server output plugins for Embulk loads records to SQL Server.
 ## Overview
 
 * **Plugin type**: output
-* **Load all or nothing**: depnds on the mode. see bellow.
-* **Resume supported**: depnds on the mode. see bellow.
+* **Load all or nothing**: depnds on the mode. see below.
+* **Resume supported**: depnds on the mode. see below.
 
 ## Configuration
 
@@ -25,7 +25,8 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **url**: URL of the JDBC connection (string, optional)
 - **table**: destination table name (string, required)
 - **options**: extra connection properties (hash, default: {})
-- **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See bellow. (string, required)
+- **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See below. (string, required)
+- **insert_method**: see below
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -53,6 +54,15 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
   * Transactional: No. If fails, the target table could be dropped (because SQL Server can't rollback DDL).
   * Resumable: No.
 
+### Insert methods
+
+insert_method supports three options.
+
+"normal" means normal insert (default). It requires SQL Server JDBC driver.
+
+"native" means bulk insert using native client. It is faster than "normal".
+It requires both SQL Server JDBC driver and SQL Server Native Client (11.0).
+
 ### Example
 
 ```yaml
@@ -81,6 +91,7 @@ out:
   database: my_database
   table: my_table
   mode: insert_direct
+  insert_method: native
   column_options:
     my_col_1: {type: 'TEXT'}
     my_col_3: {type: 'INT NOT NULL'}

data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java

@@ -11,6 +11,8 @@ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.StandardBatchInsert;
 import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.sqlserver.InsertMethod;
+import org.embulk.output.sqlserver.NativeBatchInsert;
 import org.embulk.output.sqlserver.SQLServerOutputConnector;
 import org.embulk.output.sqlserver.setter.SQLServerColumnSetterFactory;
 import org.joda.time.DateTimeZone;
@@ -60,6 +62,9 @@ public class SQLServerOutputPlugin
         @ConfigDefault("\"\"")
         public Optional<String> getPassword();
 
+        @Config("insert_method")
+        @ConfigDefault("\"normal\"")
+        public InsertMethod getInsertMethod();
     }
 
     @Override
@@ -88,6 +93,10 @@ public class SQLServerOutputPlugin
 
         String url;
         if (sqlServerTask.getUrl().isPresent()) {
+            if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
+                throw new IllegalArgumentException("Cannot set 'url' when 'insert_method' is 'native'.");
+            }
+
             if (sqlServerTask.getHost().isPresent()
                     || sqlServerTask.getInstance().isPresent()
                     || sqlServerTask.getDatabase().isPresent()
@@ -104,8 +113,8 @@ public class SQLServerOutputPlugin
             }
             StringBuilder urlBuilder = new StringBuilder();
             if (sqlServerTask.getInstance().isPresent()) {
-                urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s:%d",
-                        sqlServerTask.getHost().get(), sqlServerTask.getInstance().get(), sqlServerTask.getPort()));
+                urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s",
+                        sqlServerTask.getHost().get(), sqlServerTask.getInstance().get()));
             } else {
                 urlBuilder.append(String.format("jdbc:sqlserver://%s:%d",
                         sqlServerTask.getHost().get(), sqlServerTask.getPort()));
@@ -130,9 +139,13 @@ public class SQLServerOutputPlugin
         Properties props = new Properties();
         props.putAll(sqlServerTask.getOptions());
 
-        props.setProperty("user", sqlServerTask.getUser().get());
+        if (sqlServerTask.getUser().isPresent()) {
+            props.setProperty("user", sqlServerTask.getUser().get());
+        }
         logger.info("Connecting to {} options {}", url, props);
-        props.setProperty("password", sqlServerTask.getPassword().get());
+        if (sqlServerTask.getPassword().isPresent()) {
+            props.setProperty("password", sqlServerTask.getPassword().get());
+        }
 
         return new SQLServerOutputConnector(url, props, null);
     }
@@ -140,6 +153,11 @@ public class SQLServerOutputPlugin
     @Override
     protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
     {
+        SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
+        if (sqlServerTask.getInsertMethod() == InsertMethod.NATIVE) {
+            return new NativeBatchInsert(sqlServerTask.getHost().get(), sqlServerTask.getPort(), sqlServerTask.getInstance(),
+                    sqlServerTask.getDatabase().get(), sqlServerTask.getUser(), sqlServerTask.getPassword());
+        }
         return new StandardBatchInsert(getConnector(task, true), mergeKeys);
     }
 

data/src/main/java/org/embulk/output/sqlserver/InsertMethod.java

@@ -0,0 +1,32 @@
+package org.embulk.output.sqlserver;
+
+import java.util.Locale;
+
+import org.embulk.config.ConfigException;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public enum InsertMethod
+{
+    NORMAL,
+    NATIVE;
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        return name().toLowerCase(Locale.ENGLISH);
+    }
+
+    @JsonCreator
+    public static InsertMethod fromString(String value)
+    {
+        for (InsertMethod insertMethod : InsertMethod.values()) {
+            if (insertMethod.toString().equals(value)) {
+                return insertMethod;
+            }
+        }
+        throw new ConfigException(String.format("Unknown insert_method '%s'.", value));
+    }
+}

data/src/main/java/org/embulk/output/sqlserver/NativeBatchInsert.java

@@ -0,0 +1,247 @@
+package org.embulk.output.sqlserver;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.SQLException;
+import java.sql.Types;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcColumn;
+import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.StandardBatchInsert;
+import org.embulk.output.jdbc.TimestampFormat;
+import org.embulk.output.sqlserver.nativeclient.NativeClientWrapper;
+import org.embulk.spi.Exec;
+import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+import com.google.common.base.Optional;
+
+public class NativeBatchInsert implements BatchInsert
+{
+    private final Logger logger = Exec.getLogger(StandardBatchInsert.class);
+
+    private NativeClientWrapper client = new NativeClientWrapper();
+
+    private final String server;
+    private final int port;
+    private final Optional<String> instance;
+    private final String database;
+    private final Optional<String> user;
+    private final Optional<String> password;
+
+    private int batchWeight;
+    private int batchRows;
+    private long totalRows;
+
+    private int columnCount;
+    private int lastColumnIndex;
+
+    private DateFormat[] formats;
+
+
+    public NativeBatchInsert(String server, int port, Optional<String> instance,
+            String database, Optional<String> user, Optional<String> password)
+    {
+        this.server = server;
+        this.port = port;
+        this.instance = instance;
+        this.database = database;
+        this.user = user;
+        this.password = password;
+
+        lastColumnIndex = 0;
+    }
+
+
+    @Override
+    public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
+    {
+        columnCount = insertSchema.getCount();
+        client.open(server, port, instance, database, user, password, loadTable);
+
+        formats = new DateFormat[insertSchema.getCount()];
+        for (int i = 0; i < insertSchema.getCount(); i++) {
+            JdbcColumn column = insertSchema.getColumn(i);
+            switch (column.getSqlType()) {
+                case Types.DATE:
+                    formats[i] = new SimpleDateFormat("yyyy-MM-dd");
+                    break;
+
+                case Types.TIME:
+                    formats[i] = new TimestampFormat("HH:mm:ss", column.getScaleTypeParameter());
+                    break;
+
+                case Types.TIMESTAMP:
+                    if (column.getSimpleTypeName().equals("SMALLDATETIME")) {
+                        formats[i] = new SmallDateTimeFormat("yyyy-MM-dd HH:mm:ss");
+                    } else {
+                        formats[i] = new TimestampFormat("yyyy-MM-dd HH:mm:ss", column.getScaleTypeParameter());
+                    }
+                    break;
+
+                default:
+                    break;
+            }
+        }
+    }
+
+    @Override
+    public int getBatchWeight()
+    {
+        return batchWeight;
+    }
+
+    @Override
+    public void add() throws IOException, SQLException
+    {
+        client.sendRow();
+
+        batchRows++;
+        batchWeight += 32;  // add weight as overhead of each rows
+    }
+
+    private int nextColumnIndex()
+    {
+        int nextColumnIndex = lastColumnIndex + 1;
+        if (nextColumnIndex == columnCount) {
+            lastColumnIndex = 0;
+        } else {
+            lastColumnIndex++;
+        }
+        return nextColumnIndex;
+    }
+
+    @Override
+    public void setNull(int sqlType) throws IOException, SQLException
+    {
+        batchWeight += client.bindNull(nextColumnIndex());
+    }
+
+    @Override
+    public void setBoolean(boolean v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setByte(byte v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setShort(short v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setInt(int v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setLong(long v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setFloat(float v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setDouble(double v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setBigDecimal(BigDecimal v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v.toPlainString());
+    }
+
+    @Override
+    public void setString(String v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setNString(String v) throws IOException, SQLException
+    {
+        batchWeight += client.bindValue(nextColumnIndex(), v);
+    }
+
+    @Override
+    public void setBytes(byte[] v) throws IOException, SQLException
+    {
+        throw new SQLException("Unsupported");
+    }
+
+    @Override
+    public void setSqlDate(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        setSqlTimestamp(v, cal);
+    }
+
+    @Override
+    public void setSqlTime(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        setSqlTimestamp(v, cal);
+    }
+
+    @Override
+    public void setSqlTimestamp(Timestamp v, Calendar cal) throws IOException, SQLException
+    {
+        int columnIndex = nextColumnIndex();
+        DateFormat format = formats[columnIndex - 1];
+        format.setCalendar(cal);
+
+        java.sql.Timestamp timestamp = new java.sql.Timestamp(v.toEpochMilli());
+        timestamp.setNanos(v.getNano());
+
+        batchWeight += client.bindValue(columnIndex, format.format(timestamp));
+    }
+
+    @Override
+    public void flush() throws IOException, SQLException
+    {
+        logger.info(String.format("Loading %,d rows", batchRows));
+        long startTime = System.currentTimeMillis();
+
+        client.commit(false);
+
+        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+        totalRows += batchRows;
+
+        logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
+
+        batchRows = 0;
+        batchWeight = 0;
+    }
+
+    @Override
+    public void finish() throws IOException, SQLException
+    {
+        if (getBatchWeight() != 0) {
+            flush();
+        }
+        client.commit(true);
+    }
+
+    @Override
+    public void close() throws IOException, SQLException
+    {
+        client.close();
+    }
+
+}

data/src/main/java/org/embulk/output/sqlserver/SmallDateTimeFormat.java

@@ -0,0 +1,31 @@
+package org.embulk.output.sqlserver;
+
+import java.text.FieldPosition;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+
+public class SmallDateTimeFormat extends SimpleDateFormat
+{
+    public SmallDateTimeFormat(String pattern)
+    {
+        super(pattern);
+    }
+
+    @Override
+    public StringBuffer format(Date date, StringBuffer toAppendTo, FieldPosition pos)
+    {
+        long time = date.getTime();
+
+        // round seconds
+        long underMinutes = time % 60000;
+        if (underMinutes < 30000) {
+            time -= underMinutes;
+        } else {
+            time += 60000 - underMinutes;
+        }
+
+        return super.format(new Date(time), toAppendTo, pos);
+    }
+
+}

data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClient.java

@@ -0,0 +1,33 @@
+package org.embulk.output.sqlserver.nativeclient;
+
+import jnr.ffi.Pointer;
+
+public interface NativeClient
+{
+    static int SQL_NULL_DATA = -1;
+    static int SQLCHARACTER = 0x2F;
+    static int SQLINT1 = 0x30;
+    static int SQLBIT = 0x32;
+    static int SQLINT2 = 0x34;
+    static int SQLINT4 = 0x38;
+    static int SQLFLT8 = 0x3E;
+    static int SQLFLT4 = 0x3B;
+    static int SQLINT8 = 0x7F;
+
+    static short FAIL = 0;
+    static short SUCCEED = 1;
+    static int DB_IN = 1;
+
+    short bcp_initW(Pointer hdbc, Pointer szTable, Pointer szDataFile, Pointer szErrorFile, int eDirection);
+
+    short bcp_bind(Pointer hdbc,
+            Pointer pData, int cbIndicator, int cbData,
+            Pointer pTerm, int cbTerm,
+            int eDataType, int idxServerCol);
+
+    short bcp_sendrow(Pointer hdbc);
+
+    int bcp_batch(Pointer hdbc);
+
+    int bcp_done(Pointer hdbc);
+}