embulk-output-mysql 0.8.4 → 0.8.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bd951816256b91bd4c8ea18655cb69cce02d800
-  data.tar.gz: 7f85745159a29910334433785030c998318cefad
+  metadata.gz: 8a6b83378d49fc8b2a03de7296e7da18eab89050
+  data.tar.gz: 5e0d6bcb0fd421179ba89b6284883b87d7cd0e45
 SHA512:
-  metadata.gz: 2defc0817f7e6bf4671131875e51e2c063c1295a4f31b8d1759ba6755888f7a190c448d13e5c1e651f63ba57078fc1b03d73676ffb39a940dcb21878337889b5
-  data.tar.gz: 67b7e2158febcbb3d8c766d8726ddf8ec1f89a597ec64aa089c6d0511f58859c203ebf6ad5fc15373ea3c63746c020406e054e49a778c4d2a88d7598cd5f753c
+  metadata.gz: 6c84ec4e5a7acaa7173a54e0d339d5b38a70cc17d5692cd4a0452c21f186eb6d71c7476a13cbefeb6e03e0f8a60751a9d91bc87f8251fe9c2ee22a895600b65d
+  data.tar.gz: edf085322564c9f9592bc014889e145b72d2f28ccac8b5734bc062b9f826427b28e9934ca00a5b5b80f7d82cf8daf6c278e27ca578e486f8a50b388d8c5aedba

data/src/test/java/org/embulk/output/mysql/RetryTest.java

@@ -29,6 +29,11 @@ import com.google.common.io.Resources;
 
 public class RetryTest
 {
+    private static class Lock
+    {
+        public boolean enabled = true;
+    }
+
     private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/retry/";
 
     private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
@@ -151,7 +156,7 @@ public class RetryTest
             }
         }
 
-        final Object lock = new Object();
+        final Lock lock = new Lock();
 
         Thread thread = new Thread() {
             @Override
@@ -164,16 +169,18 @@ public class RetryTest
                             for (int i = 1000000; i < 1260000; i++) {
                                 statement.execute("insert into test1 values('B" + i + "', 0)");
                             }
+                            System.out.println("# Inserted many records.");
 
                             synchronized (lock) {
                                 lock.notify();
+                                lock.enabled = false;
                             }
 
-                            System.out.println("##1");
-
+                            System.out.println("# Insert 'A1249000'.");
                             statement.execute("insert into test1 values('A1249010', 0)");
                             Thread.sleep(5000);
                             // deadlock will occur
+                            System.out.println("# Insert 'A1249010'.");
                             statement.execute("insert into test1 values('A1249000', 0)");
                             conn.rollback();
                         }
@@ -186,7 +193,175 @@ public class RetryTest
         thread.start();
 
         synchronized (lock) {
-            lock.wait();
+            if (lock.enabled) {
+                lock.wait();
+            }
+        }
+
+        //Path in1 = toPath("test1_flushed_multiple_times.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test1.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(expected1.toString()));
+    }
+
+    @Test
+    public void testRetry_MultipleTimes() throws Exception
+    {
+        Thread thread1 = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 100; i < 110; i++) {
+                                statement.execute("insert into test1 values('B" + i + "', 0)");
+                            }
+
+                            statement.execute("insert into test1 values('A003', 0)");
+                            Thread.sleep(3000);
+                            // deadlock will occur
+                            statement.execute("insert into test1 values('A002', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread1.start();
+
+        Thread thread2 = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 100; i < 110; i++) {
+                                statement.execute("insert into test1 values('C" + i + "', 0)");
+                            }
+
+                            statement.execute("insert into test1 values('A004', 0)");
+                            Thread.sleep(6000);
+                            // deadlock will occur
+                            statement.execute("insert into test1 values('A001', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread2.start();
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test1.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test1_expected.csv")));
+    }
+
+    // records will be partially committed
+    @Test
+    public void testRetry_Large_MultipleTimes() throws Exception
+    {
+        Path in1 = embulk.createTempFile("csv");
+        StringBuilder expected1 = new StringBuilder();
+
+        try (BufferedWriter writer = new BufferedWriter(new FileWriter(in1.toFile()))) {
+            writer.write("id:string,num:long");
+            writer.newLine();
+
+            for (int i = 1000000; i < 1250000; i++) {
+                writer.write("A" + i + "," + i);
+                writer.newLine();
+
+                expected1.append("A" + i + "," + i + "\n");
+            }
+        }
+
+        final Lock lock1 = new Lock();
+        final Lock lock2 = new Lock();
+
+        Thread thread1 = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 1000000; i < 1260000; i++) {
+                                statement.execute("insert into test1 values('B" + i + "', 0)");
+                            }
+                            System.out.println("# Inserted many records (1).");
+
+                            synchronized (lock1) {
+                                lock1.notify();
+                                lock1.enabled = false;
+                            }
+
+                            System.out.println("# Insert 'A1249010'.");
+                            statement.execute("insert into test1 values('A1249010', 0)");
+                            Thread.sleep(5000);
+                            // deadlock will occur
+                            System.out.println("# Insert 'A1249000'.");
+                            statement.execute("insert into test1 values('A1249000', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread1.start();
+
+        Thread thread2 = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 1000000; i < 1260000; i++) {
+                                statement.execute("insert into test1 values('C" + i + "', 0)");
+                            }
+                            System.out.println("# Inserted many records (2).");
+
+                            synchronized (lock2) {
+                                lock2.notify();
+                                lock2.enabled = false;
+                            }
+
+                            System.out.println("# Insert 'A1249030'.");
+                            statement.execute("insert into test1 values('A1249030', 0)");
+                            Thread.sleep(20000);
+                            // deadlock will occur
+                            System.out.println("# Insert 'A1249020'.");
+                            statement.execute("insert into test1 values('A1249020', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread2.start();
+
+        synchronized (lock1) {
+            if (lock1.enabled) {
+                lock1.wait();
+            }
+        }
+        synchronized (lock2) {
+            if (lock2.enabled) {
+                lock2.wait();
+            }
         }
 
         //Path in1 = toPath("test1_flushed_multiple_times.csv");

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mysql
 version: !ruby/object:Gem::Version
-  version: 0.8.4
+  version: 0.8.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-08 00:00:00.000000000 Z
+date: 2019-05-10 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.4.jar
-- classpath/embulk-output-mysql-0.8.4.jar
+- classpath/embulk-output-jdbc-0.8.5.jar
+- classpath/embulk-output-mysql-0.8.5.jar
 - default_jdbc_driver/mysql-connector-java-5.1.44.jar
 - lib/embulk/output/mysql.rb
 - src/main/java/org/embulk/output/MySQLOutputPlugin.java