embulk-output-sqlserver 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8eb048f302970ac3e0c9361d1eb2e489f9be73f
-  data.tar.gz: 9f23d5b8eaeb9b7d372b2f8ec193d9008fa768e2
+  metadata.gz: 64f02cdd0fc14b4644e8dfc1dbdce80a286b80ee
+  data.tar.gz: f286424dbcbb308fc98ff2996ecaab4c32f3bdf5
 SHA512:
-  metadata.gz: 6f815371db4df43dd637bf8691fd92f3c23ccf8a07c9f737e94fa34eaabe44a1c053d3e0b78ddcf694a19cf743f9427d0ebdb3636cb5bd676792dee3bc2d722e
-  data.tar.gz: 3fd2806f6b4ea2025a3ee7393407cb8425c74e2758f71440735f816e2bb983f3b7a428303b8faf95d69ec5603b7e2021831d2b0398764353c8f3c32a929c75a6
+  metadata.gz: e40b2bf9a2ca5e076cb3f2054bebb39fded85da86196a9627fac8870adc0ff64e8ec90f5d23d1685a9319f0a535b51c02ef4b13784bbd7747c4e93894fb82590
+  data.tar.gz: 5379f7fc7836540a50544d7e1c905346bc1fa2d84a737dd3ef7e1ff5fe2779e886514dd64ccee92645571b9868b84eb4a77b45ffcc9e90c5db6e4352ce906a07

data/README.md

@@ -26,12 +26,12 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **temp_schema**: schema name for intermediate tables. by default, intermediate tables will be created in the same schema as destination table. (string, optional)
 - **url**: URL of the JDBC connection (string, optional)
 - **table**: destination table name (string, required)
-- **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
-- **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **create_table_constraint**: table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **create_table_option**: table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **options**: extra connection properties (hash, default: {})
-- **retry_limit** max retry count for database operations (integer, default: 12)
-- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
-- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
+- **retry_limit**: max retry count for database operations (integer, default: 12)
+- **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))
+- **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert" , "replace" or "merge". See below. (string, required)
 - **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
 - **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = T.foo + S.foo` (`T` means target table and `S` means source table). (string array, default: always overwrites with new values)

data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClient.java

@@ -25,6 +25,8 @@ public interface NativeClient
             Pointer pTerm, int cbTerm,
             int eDataType, int idxServerCol);
 
+    short bcp_collen(Pointer hdbc, int cbData, int idxServerCol);
+
     short bcp_sendrow(Pointer hdbc);
 
     int bcp_batch(Pointer hdbc);

data/src/main/java/org/embulk/output/sqlserver/nativeclient/NativeClientWrapper.java

@@ -21,6 +21,8 @@ import com.google.common.base.Optional;
 
 public class NativeClientWrapper
 {
+    private static int MAX_COLUMN_SIZE_FOR_BCP_BIND = 7999;
+
     private static ODBC odbc;
     private static NativeClient client;
 
@@ -206,11 +208,19 @@ public class NativeClientWrapper
                 odbcHandle,
                 pointer,
                 0,
-                size,
+                Math.min(size, MAX_COLUMN_SIZE_FOR_BCP_BIND),
                 null,
                 0,
                 NativeClient.SQLCHARACTER,
                 columnIndex));
+
+        if (size > MAX_COLUMN_SIZE_FOR_BCP_BIND) {
+            checkBCPResult("bcp_collen", client.bcp_collen(
+                    odbcHandle,
+                    size,
+                    columnIndex));
+        }
+
         return (int)pointer.size();
     }
 

data/src/test/java/org/embulk/output/sqlserver/NativeTest.java

@@ -8,10 +8,15 @@ import static org.junit.Assert.assertThat;
 import java.io.File;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.nio.charset.Charset;
 import java.nio.file.FileSystems;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.List;
 
-import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.output.SQLServerOutputPlugin;
 import org.embulk.spi.OutputPlugin;
@@ -21,6 +26,7 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.io.Resources;
 
 public class NativeTest
@@ -249,6 +255,59 @@ public class NativeTest
         //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
     }
 
+    @Test
+    public void testHuge() throws Exception
+    {
+        Path in1 = toPath("test_huge.csv");
+
+        // create input data dynamically because it is huge.
+
+        ImmutableList.Builder<String[]> recordsBuilder = ImmutableList.builder();
+        recordsBuilder.add(new String[]{"a", "X", "あ"});
+        recordsBuilder.add(new String[]{"b", createString(9000), createString(10000) + "い"});
+        recordsBuilder.add(new String[]{"c", createString(20000), createString(30000) + "う"});
+        List<String[]> records = recordsBuilder.build();
+
+        ImmutableList.Builder<String> linesBuilder = ImmutableList.builder();
+        linesBuilder.add("ITEM1:long,ITEM2:string,ITEM3:string,ITEM4:string");
+        for (int i = 0; i < records.size(); i++) {
+            String[] record = records.get(i);
+            linesBuilder.add((i + 1) + "," + record[0] + "," + record[1] + "," + record[2]);
+        }
+
+        Charset charset = Charset.forName("UTF8");
+        Files.write(in1, linesBuilder.build(), charset);
+
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_huge.yml")), in1);
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+
+        try (Connection conn = SQLServerTests.connect()) {
+            try (Statement statement = conn.createStatement()) {
+                try (ResultSet rs = statement.executeQuery("SELECT * FROM TEST_HUGE ORDER BY ITEM1")) {
+                    int recordIndex = 0;
+                    while (rs.next()) {
+                        assertThat(rs.getInt(1), is(recordIndex + 1));
+                        assertThat(rs.getString(2), is(records.get(recordIndex)[0]));
+                        assertThat(rs.getString(3), is(records.get(recordIndex)[1]));
+                        assertThat(rs.getString(4), is(records.get(recordIndex)[2]));
+                        recordIndex++;
+                    }
+
+                    assertThat(recordIndex, is(records.size()));
+                }
+            }
+        }
+    }
+
+    private String createString(int size)
+    {
+        char[] chars = new char[size];
+        for (int i = 0; i < size; i++) {
+            chars[i] = (char)('A' + i % 26);
+        }
+        return new String(chars);
+    }
+
     private Path toPath(String fileName) throws URISyntaxException
     {
         URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);

data/src/test/java/org/embulk/output/sqlserver/SQLServerTests.java

@@ -6,6 +6,9 @@ import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 import java.util.Collections;
 import java.util.List;
 
@@ -24,6 +27,21 @@ public class SQLServerTests
         return EmbulkTests.config("EMBULK_OUTPUT_SQLSERVER_TEST_CONFIG");
     }
 
+    public static Connection connect() throws SQLException
+    {
+        ConfigSource config = baseConfig();
+
+        String user = config.get(String.class, "user");
+        String password = config.get(String.class, "password");
+        String host = config.get(String.class, "host");
+        Integer port = config.get(Integer.class, "port");
+        String database = config.get(String.class, "database");
+
+        String url = String.format("jdbc:jtds:sqlserver://%s:%d/%s", host, port, database);
+
+        return DriverManager.getConnection(url, user, password);
+    }
+
     public static void execute(String sql, String... options)
     {
         ConfigSource config = baseConfig();

data/src/test/resources/org/embulk/output/sqlserver/test/expect/native/setup.sql

@@ -158,3 +158,12 @@ CREATE TABLE TEST_TIME (
     VALUE               TIME,
     PRIMARY KEY (ID)
 );
+
+DROP TABLE TEST_HUGE;
+CREATE TABLE TEST_HUGE (
+    ITEM1 INT,
+    ITEM2 VARCHAR(4),
+    ITEM3 TEXT,
+    ITEM4 NTEXT,
+    PRIMARY KEY(ITEM1)
+);

data/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.csv

@@ -0,0 +1 @@
+dummy

data/src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.yml

@@ -0,0 +1,3 @@
+table: TEST_HUGE
+mode: insert_direct
+insert_method: native

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-26 00:00:00.000000000 Z
+date: 2018-11-16 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.0.jar
-- classpath/embulk-output-sqlserver-0.8.0.jar
+- classpath/embulk-output-jdbc-0.8.1.jar
+- classpath/embulk-output-sqlserver-0.8.1.jar
 - classpath/jtds-1.3.1.jar
 - lib/embulk/output/sqlserver.rb
 - src/main/java/org/embulk/output/SQLServerOutputPlugin.java
@@ -79,6 +79,8 @@ files:
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null.csv
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_float_null_expected.csv
+- src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.csv
+- src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_huge.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_int_null.yml
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_integer_null.csv
 - src/test/resources/org/embulk/output/sqlserver/test/expect/native/test_integer_null_expected.csv