embulk-output-sqlserver 0.7.10 → 0.7.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42a9d71ba89b5088b54354c9bcb99868a7406023
-  data.tar.gz: d841e4d1c9238a7d2db62ee6ed09d2918903c583
+  metadata.gz: d8d8e2c0bbd2a1255d5825697f0c6f2e893d9753
+  data.tar.gz: 14ee14826f9dea6d9d5549e7bdaa41bcb01d8943
 SHA512:
-  metadata.gz: 4b054292699bdec82190edfc751b6546b2645baf4c92d2bfb12eeeccf2d8798e4d436e1a20193b2d834663987f8207bd363610729daff458d614b227c2092350
-  data.tar.gz: 63c96e25c33380edcb209a746ddc0956950290d80b8f71290805f053d2cf193e3544beb81872b8bed0a34d774ee3b3ed9475d63c7f7a774a0d90e5cf7d02fa6e
+  metadata.gz: 913d8c0c5b02f0ef7488c3063b0e022b606774c6c0856d2a78670e2119a5b7708eb88efa1f1e47e4cbe9014ecf448eb0a867bc528d6ef76e67c4f493658a437c
+  data.tar.gz: df036fa9b4a164a18238f56273ba578d962bc1f43d97037e9dc740aa0517fedaf624bef35443631e5df7b8afeb728712aa5d90473e38ac0ed73ec52f546261e0

data/README.md

@@ -147,3 +147,19 @@ out:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+You need to put 'sqljdbc41.jar' into 'embulk-output-sqlserver/driver' directory and create 'sqlserver.yml' as follows.
+```
+type: sqlserver
+host: localhost
+port: 1433
+database: database
+user: user
+password: pass
+```
+
+```
+$ EMBULK_OUTPUT_SQLSERVER_TEST_CONFIG=sqlserver.yml ./gradlew :embulk-output-sqlserver:check --info
+```

data/build.gradle

@@ -2,7 +2,7 @@ dependencies {
     compile project(':embulk-output-jdbc')
     compile 'net.sourceforge.jtds:jtds:1.3.1'
 
-    testCompile 'org.embulk:embulk-standards:0.8.8'
+    testCompile 'org.embulk:embulk-standards:0.8.22'
     testCompile project(':embulk-output-jdbc').sourceSets.test.output
     testCompile files('driver/sqljdbc41.jar')
 }

data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java

@@ -28,6 +28,9 @@ import static java.util.Locale.ENGLISH;
 public class SQLServerOutputPlugin
         extends AbstractJdbcOutputPlugin
 {
+    // for test
+    public static boolean preferMicrosoftDriver = true;
+
     private static int DEFAULT_PORT = 1433;
 
     public interface SQLServerPluginTask
@@ -128,20 +131,26 @@ public class SQLServerOutputPlugin
             loadDriverJar(sqlServerTask.getDriverPath().get());
             try {
                 Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-            } catch(Exception e) {
+            } catch (Exception e) {
                 throw new ConfigException("Driver set at field 'driver_path' doesn't include Microsoft SQLServerDriver", e);
             }
         } else {
-            // prefer Microsoft SQLServerDriver if it is in classpath
-            try {
-                Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
-            } catch(Exception e) {
+            boolean useMicrosoftDriver = false;
+            if (preferMicrosoftDriver) {
+                // prefer Microsoft SQLServerDriver if it is in classpath
+                try {
+                    Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
+                    useMicrosoftDriver = true;
+                } catch (Exception e) {
+                }
+            }
+
+            if (!useMicrosoftDriver) {
                 logger.info("Using jTDS Driver");
                 try {
                     Class.forName("net.sourceforge.jtds.jdbc.Driver");
-                } catch(Exception e2) {
-                    throw new ConfigException("'driver_path' doesn't set and can't found jTDS driver", e2);
-
+                } catch (Exception e) {
+                    throw new ConfigException("'driver_path' doesn't set and can't find jTDS driver", e);
                 }
                 useJtdsDriver = true;
             }

data/src/test/java/org/embulk/output/sqlserver/BasicTest.java

@@ -0,0 +1,200 @@
+package org.embulk.output.sqlserver;
+
+import static org.embulk.output.sqlserver.SQLServerTests.execute;
+import static org.embulk.output.sqlserver.SQLServerTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.SQLServerOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class BasicTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/sqlserver/test/expect/basic/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "sqlserver", SQLServerOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = SQLServerTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsert() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertCreate() throws Exception
+    {
+        execute("DROP TABLE TEST1");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_create_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirect() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectCreate() throws Exception
+    {
+        execute("DROP TABLE TEST1");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_create_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsert() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_truncate_insert_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplace() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_create_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplaceCreate() throws Exception
+    {
+        execute("DROP TABLE TEST1");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_create_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplaceLongName() throws Exception
+    {
+        String tableName = "TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９";
+        assertThat(tableName.length(), is(127));
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace_longname.yml")), in1);
+        assertThat(selectRecords(embulk, tableName), is(readResource("test_insert_create_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMerge() throws Exception
+    {
+        Path in1 = toPath("test_merge.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST_MERGE1"), is(readResource("test_merge_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeWithKeys() throws Exception
+    {
+        Path in1 = toPath("test_merge.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_keys.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST_MERGE2"), is(readResource("test_merge_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeWithRule() throws Exception
+    {
+        Path in1 = toPath("test_merge.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_rule.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST_MERGE1"), is(readResource("test_merge_rule_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testNativeInsertDirect() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_native_insert_direct.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testStringToTimestamp() throws Exception
+    {
+        Path in1 = toPath("test_string_timestamp.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_string_timestamp.yml")), in1);
+        assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_string_timestamp_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testJtds() throws Exception
+    {
+        SQLServerOutputPlugin.preferMicrosoftDriver = false;
+        try {
+            Path in1 = toPath("test1.csv");
+            TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert.yml")), in1);
+            assertThat(selectRecords(embulk, "TEST1"), is(readResource("test_insert_expected.csv")));
+            //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+        } finally {
+            SQLServerOutputPlugin.preferMicrosoftDriver = true;
+        }
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/java/org/embulk/output/sqlserver/SQLServerTests.java

@@ -0,0 +1,80 @@
+package org.embulk.output.sqlserver;
+
+import static java.util.Locale.ENGLISH;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+
+public class SQLServerTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_OUTPUT_SQLSERVER_TEST_CONFIG");
+    }
+
+    public static void execute(String sql, String... options)
+    {
+        ConfigSource config = baseConfig();
+
+        ImmutableList.Builder<String> args = ImmutableList.builder();
+        args.add("sqlcmd")
+                .add("-U")
+                .add(config.get(String.class, "user"))
+                .add("-P")
+                .add(config.get(String.class, "password"))
+                .add("-H")
+                .add(config.get(String.class, "host"))
+                .add("-d")
+                .add(config.get(String.class, "database"))
+                .add("-Q")
+                .add(sql);
+        for (String option : options) {
+            args.add(option);
+        }
+
+        ProcessBuilder pb = new ProcessBuilder(args.build());
+        pb.redirectErrorStream(true);
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                    "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+
+    public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        Path temp = embulk.createTempFile("txt");
+        Files.delete(temp);
+
+        // should not use UTF8 because of BOM
+        execute("SET NOCOUNT ON; SELECT * FROM " + tableName, "-h", "-1", "-s", ",", "-W", "-f", "932", "-o", temp.toString());
+
+        List<String> lines = Files.readAllLines(temp, Charset.forName("MS932"));
+        Collections.sort(lines);
+        StringBuilder sb = new StringBuilder();
+        for (String line : lines) {
+            sb.append(line);
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+}

data/src/test/resources/org/embulk/output/sqlserver/test/expect/basic/setup.sql

@@ -0,0 +1,82 @@
+DROP TABLE TEST1
+CREATE TABLE TEST1 (
+    ID                  CHAR(4),
+    TINYINT_ITEM        TINYINT,
+    SMALLINT_ITEM       SMALLINT,
+    INT_ITEM            INT,
+    BIGINT_ITEM         BIGINT,
+    BIT_ITEM            BIT,
+    DECIMAL_ITEM        DECIMAL(12,2),
+    NUMERIC_ITEM        NUMERIC(5,3),
+    SMALLMONEY_ITEM     SMALLMONEY,
+    MONEY_ITEM          MONEY,
+    REAL_ITEM           REAL,
+    FLOAT_ITEM          FLOAT,
+    CHAR_ITEM           CHAR(4),
+    VARCHAR_ITEM        VARCHAR(8),
+    TEXT_ITEM           TEXT,
+    NCHAR_ITEM          NCHAR(4),
+    NVARCHAR_ITEM       NVARCHAR(8),
+    NTEXT_ITEM          NTEXT,
+    DATE_ITEM           DATE,
+    DATETIME_ITEM       DATETIME,
+    DATETIME2_ITEM      DATETIME2,
+    DATETIME2_2_ITEM    DATETIME2(2),
+    SMALLDATETIME_ITEM  SMALLDATETIME,
+    TIME_ITEM           TIME,
+    TIME_2_ITEM         TIME(2),
+    PRIMARY KEY (ID)
+);
+INSERT INTO TEST1(ID) VALUES('9999');
+
+DROP TABLE TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９
+CREATE TABLE TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９ (
+    ID                  CHAR(4),
+    TINYINT_ITEM        TINYINT,
+    SMALLINT_ITEM       SMALLINT,
+    INT_ITEM            INT,
+    BIGINT_ITEM         BIGINT,
+    BIT_ITEM            BIT,
+    DECIMAL_ITEM        DECIMAL(12,2),
+    NUMERIC_ITEM        NUMERIC(5,3),
+    SMALLMONEY_ITEM     SMALLMONEY,
+    MONEY_ITEM          MONEY,
+    REAL_ITEM           REAL,
+    FLOAT_ITEM          FLOAT,
+    CHAR_ITEM           CHAR(4),
+    VARCHAR_ITEM        VARCHAR(8),
+    TEXT_ITEM           TEXT,
+    NCHAR_ITEM          NCHAR(4),
+    NVARCHAR_ITEM       NVARCHAR(8),
+    NTEXT_ITEM          NTEXT,
+    DATE_ITEM           DATE,
+    DATETIME_ITEM       DATETIME,
+    DATETIME2_ITEM      DATETIME2,
+    DATETIME2_2_ITEM    DATETIME2(2),
+    SMALLDATETIME_ITEM  SMALLDATETIME,
+    TIME_ITEM           TIME,
+    TIME_2_ITEM         TIME(2),
+    PRIMARY KEY (ID)
+);
+INSERT INTO TEST___Ａ１２３４５６７８９Ｂ１２３４５６７８９Ｃ１２３４５６７８９Ｄ１２３４５６７８９Ｅ１２３４５６７８９Ｆ１２３４５６７８９Ｇ１２３４５６７８９Ｈ１２３４５６７８９Ｉ１２３４５６７８９Ｊ１２３４５６７８９Ｋ１２３４５６７８９Ｌ１２３４５６７８９(ID) VALUES('9999');
+
+DROP TABLE TEST_MERGE1;
+CREATE TABLE TEST_MERGE1 (
+    ITEM1 INT,
+    ITEM2 INT,
+    ITEM3 VARCHAR(4),
+    PRIMARY KEY(ITEM1, ITEM2)
+);
+INSERT INTO TEST_MERGE1 VALUES(10, 20, 'A');
+INSERT INTO TEST_MERGE1 VALUES(10, 21, 'B');
+INSERT INTO TEST_MERGE1 VALUES(11, 20, 'C');
+
+DROP TABLE TEST_MERGE2;
+CREATE TABLE TEST_MERGE2 (
+    ITEM1 INT,
+    ITEM2 INT,
+    ITEM3 VARCHAR(4)
+);
+INSERT INTO TEST_MERGE2 VALUES(10, 20, 'A');
+INSERT INTO TEST_MERGE2 VALUES(10, 21, 'B');
+INSERT INTO TEST_MERGE2 VALUES(11, 20, 'C');