embulk-output-jdbc 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6f3b78600d620d03f0332b8f0f883e19ae0fa4df
-  data.tar.gz: ccd435e6f57e287920de400cad04d32e0e58322c
+  metadata.gz: 22fb481777c119acfc41002c96d50fb4bd336afd
+  data.tar.gz: 7757ee98f9ff666421f81efe882020df6f22aa7a
 SHA512:
-  metadata.gz: 72a1e2506e4499a24ea405d63262ddcf0fd853d6f970516b457f965a66ab28d1bd38e1963083e6aa33eaf33cad7f244a2bfa291ac7a96ae850764d2c9b99711a
-  data.tar.gz: afdf7da10965fa9f8c5d5c8dd5d9d584f68221c882a75db3b29c72a0d1a6306ee46b3e3759d45c6ed4d64d14b1bc79d5d033516fd63904c44b15b3cd1b968526
+  metadata.gz: 6e5fb5117260aaa6b8b42b3f0582b33fb5f842c3f20b349f2ca13120904053428f2e0edb7c87fdee1a32332243c421e1700d4ef09b03bcb9b224f8562a8c3399
+  data.tar.gz: 29f4a992a7fce0809993b044f7833d1baf620a1a1dc3527acfba917671ac784ce6bf8f20498c38fb56232fe03730dbdf600e5dd16764dbf8695bf29c72d480a0

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -13,8 +13,10 @@ import java.sql.Types;
 import java.sql.ResultSet;
 import java.sql.DatabaseMetaData;
 import java.sql.SQLException;
+
 import org.slf4j.Logger;
 import org.joda.time.DateTimeZone;
+
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -25,6 +27,7 @@ import com.google.common.base.Throwables;
 import com.google.common.collect.Lists;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -47,6 +50,7 @@ import org.embulk.output.jdbc.setter.ColumnSetter;
 import org.embulk.output.jdbc.setter.ColumnSetterFactory;
 import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
 import org.embulk.spi.util.RetryExecutor.Retryable;
+
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
 
@@ -102,9 +106,30 @@ public abstract class AbstractJdbcOutputPlugin
         public void setIntermediateTables(Optional<List<String>> names);
     }
 
+    public static enum LengthSemantics
+    {
+        BYTES {
+            @Override
+            public int countLength(Charset charset, String s)
+            {
+                return charset.encode(s).remaining();
+            }
+        },
+        CHARACTERS {
+            @Override
+            public int countLength(Charset charset, String s)
+            {
+                return s.length();
+            }
+        };
+
+        public abstract int countLength(Charset charset, String s);
+    }
+
     public static class Features
     {
         private int maxTableNameLength = 64;
+        private LengthSemantics tableNameLengthSemantics = LengthSemantics.BYTES;
         private Set<Mode> supportedModes = ImmutableSet.copyOf(Mode.values());
         private boolean ignoreMergeKeys = false;
 
@@ -124,6 +149,18 @@ public abstract class AbstractJdbcOutputPlugin
             return this;
         }
 
+        public LengthSemantics getTableNameLengthSemantics()
+        {
+            return tableNameLengthSemantics;
+        }
+
+        @JsonProperty
+        public Features setTableNameLengthSemantics(LengthSemantics tableNameLengthSemantics)
+        {
+            this.tableNameLengthSemantics = tableNameLengthSemantics;
+            return this;
+        }
+
         @JsonProperty
         public Set<Mode> getSupportedModes()
         {
@@ -412,12 +449,14 @@ public abstract class AbstractJdbcOutputPlugin
             // direct modify mode doesn't need intermediate tables.
             ImmutableList.Builder<String> intermTableNames = ImmutableList.builder();
             if (mode.tempTablePerTask()) {
-                String namePrefix = generateIntermediateTableNamePrefix(task.getTable(), con, 3, task.getFeatures().getMaxTableNameLength());
+                String namePrefix = generateIntermediateTableNamePrefix(task.getTable(), con, 3,
+                        task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
                 for (int i=0; i < taskCount; i++) {
                     intermTableNames.add(namePrefix + String.format("%03d", i));
                 }
             } else {
-                String name = generateIntermediateTableNamePrefix(task.getTable(), con, 0, task.getFeatures().getMaxTableNameLength());
+                String name = generateIntermediateTableNamePrefix(task.getTable(), con, 0,
+                        task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
                 intermTableNames.add(name);
             }
             // create the intermediate tables here
@@ -452,7 +491,7 @@ public abstract class AbstractJdbcOutputPlugin
 
         // validate column_options
         newColumnSetters(
-                new ColumnSetterFactory(null, task.getDefaultTimeZone()),  // TODO create a dummy BatchInsert
+                newColumnSetterFactory(null, task.getDefaultTimeZone()),  // TODO create a dummy BatchInsert
                 task.getTargetTableSchema(), schema,
                 task.getColumnOptions());
 
@@ -491,7 +530,13 @@ public abstract class AbstractJdbcOutputPlugin
         }
     }
 
-    protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con, int suffixLength, int maxLength) throws SQLException
+    protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, DateTimeZone defaultTimeZone)
+    {
+        return new ColumnSetterFactory(batch, defaultTimeZone);
+    }
+
+    protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con,
+            int suffixLength, int maxLength, LengthSemantics lengthSemantics) throws SQLException
     {
         Charset tableNameCharset = con.getTableNameCharset();
         String tableName = baseTableName;
@@ -500,7 +545,7 @@ public abstract class AbstractJdbcOutputPlugin
 
         // way to count length of table name varies by DBMSs (bytes or characters),
         // so truncate swap table name by one character.
-        while (!checkTableNameLength(tableName + "_" + uniqueSuffix, tableNameCharset, suffixLength, maxLength)) {
+        while (!checkTableNameLength(tableName + "_" + uniqueSuffix, tableNameCharset, suffixLength, maxLength, lengthSemantics)) {
             if (uniqueSuffix.length() > 8 + suffix.length()) {
                 // truncate transaction unique name
                 // (include 8 characters of the transaction name at least)
@@ -572,9 +617,10 @@ public abstract class AbstractJdbcOutputPlugin
                     });
     }
 
-    private boolean checkTableNameLength(String tableName, Charset tableNameCharset, int suffixLength, int maxLength)
+    private boolean checkTableNameLength(String tableName, Charset tableNameCharset,
+            int suffixLength, int maxLength, LengthSemantics lengthSemantics)
     {
-        return tableNameCharset.encode(tableName).remaining() + suffixLength <= maxLength;
+        return lengthSemantics.countLength(tableNameCharset, tableName) + suffixLength <= maxLength;
     }
 
     protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
@@ -773,7 +819,7 @@ public abstract class AbstractJdbcOutputPlugin
             PageReader reader = new PageReader(schema);
 
             List<ColumnSetter> columnSetters = newColumnSetters(
-                    new ColumnSetterFactory(batch, task.getDefaultTimeZone()),
+                    newColumnSetterFactory(batch, task.getDefaultTimeZone()),
                     task.getTargetTableSchema(), schema,
                     task.getColumnOptions());
             JdbcSchema insertIntoSchema = filterSkipColumns(task.getTargetTableSchema());

data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java

@@ -153,6 +153,16 @@ public class JdbcOutputConnection
         return sb.toString();
     }
 
+    protected String buildRenameTableSql(String fromTable, String toTable)
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append("ALTER TABLE ");
+        quoteIdentifierString(sb, fromTable);
+        sb.append(" RENAME TO ");
+        quoteIdentifierString(sb, toTable);
+        return sb.toString();
+    }
+
     public static enum ColumnDeclareType
     {
         SIMPLE,
@@ -208,6 +218,7 @@ public class JdbcOutputConnection
 
     private static final String[] STANDARD_SIZE_AND_SCALE_TYPE_NAMES = new String[] {
         "DECIMAL",
+        "NUMERIC",
     };
 
     protected ColumnDeclareType getColumnDeclareType(String convertedTypeName, JdbcColumn col)
@@ -346,13 +357,7 @@ public class JdbcOutputConnection
         try {
             dropTableIfExists(stmt, toTable);
 
-            StringBuilder sb = new StringBuilder();
-            sb.append("ALTER TABLE ");
-            quoteIdentifierString(sb, fromTable);
-            sb.append(" RENAME TO ");
-            quoteIdentifierString(sb, toTable);
-            String sql = sb.toString();
-            executeUpdate(stmt, sql);
+            executeUpdate(stmt, buildRenameTableSql(fromTable, toTable));
 
             commitIfNecessary(connection);
         } catch (SQLException ex) {

data/src/main/java/org/embulk/output/jdbc/setter/SqlDateColumnSetter.java

@@ -10,7 +10,7 @@ import org.embulk.output.jdbc.BatchInsert;
 public class SqlDateColumnSetter
         extends ColumnSetter
 {
-    private final Calendar calendar;
+    protected final Calendar calendar;
 
     public SqlDateColumnSetter(BatchInsert batch, JdbcColumn column,
             DefaultValueSetter defaultValue,

data/src/main/java/org/embulk/output/jdbc/setter/SqlTimeColumnSetter.java

@@ -10,7 +10,7 @@ import org.embulk.output.jdbc.BatchInsert;
 public class SqlTimeColumnSetter
         extends ColumnSetter
 {
-    private final Calendar calendar;
+    protected final Calendar calendar;
 
     public SqlTimeColumnSetter(BatchInsert batch, JdbcColumn column,
             DefaultValueSetter defaultValue,

data/src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java

@@ -10,7 +10,7 @@ import org.embulk.output.jdbc.BatchInsert;
 public class SqlTimestampColumnSetter
         extends ColumnSetter
 {
-    private final Calendar calendar;
+    protected final Calendar calendar;
 
     public SqlTimestampColumnSetter(BatchInsert batch, JdbcColumn column,
             DefaultValueSetter defaultValue,

data/src/test/java/org/embulk/output/AbstractJdbcOutputPluginTest.java

@@ -0,0 +1,117 @@
+package org.embulk.output;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.google.common.io.Files;
+
+public abstract class AbstractJdbcOutputPluginTest
+{
+    protected void dropTable(String table) throws SQLException
+    {
+        String sql = String.format("DROP TABLE %s", table);
+        executeSQL(sql, true);
+    }
+
+    protected List<List<Object>> select(String table) throws SQLException
+    {
+        try (Connection connection = connect()) {
+            try (Statement statement = connection.createStatement()) {
+                List<List<Object>> rows = new ArrayList<List<Object>>();
+                String sql = String.format("SELECT * FROM %s", table);
+                System.out.println(sql);
+                try (ResultSet resultSet = statement.executeQuery(sql)) {
+                    while (resultSet.next()) {
+                        List<Object> row = new ArrayList<Object>();
+                        for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
+                            row.add(getValue(resultSet, i));
+                        }
+                        rows.add(row);
+                    }
+                }
+                // cannot sort by CLOB, so sort by Java
+                Collections.sort(rows, new Comparator<List<Object>>() {
+                    @Override
+                    public int compare(List<Object> o1, List<Object> o2) {
+                        return o1.toString().compareTo(o2.toString());
+                    }
+                });
+                return rows;
+            }
+        }
+
+    }
+
+    protected Object getValue(ResultSet resultSet, int index) throws SQLException
+    {
+        return resultSet.getObject(index);
+    }
+
+    protected void executeSQL(String sql) throws SQLException
+    {
+        executeSQL(sql, false);
+    }
+
+    protected void executeSQL(String sql, boolean ignoreError) throws SQLException
+    {
+        try (Connection connection = connect()) {
+            try {
+                connection.setAutoCommit(true);
+
+                try (Statement statement = connection.createStatement()) {
+                    System.out.println(String.format("Execute SQL : \"%s\".", sql));
+                    statement.execute(sql);
+                }
+
+            } catch (SQLException e) {
+                if (!ignoreError) {
+                    throw e;
+                }
+            }
+        }
+    }
+
+    protected String convertYml(String ymlName) throws Exception
+    {
+        StringBuilder builder = new StringBuilder();
+        Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
+        for (String line : Files.readLines(convertPath(ymlName), Charset.defaultCharset())) {
+            line = convertYmlLine(line);
+            Matcher matcher = pathPrefixPattern.matcher(line);
+            if (matcher.matches()) {
+                int group = 2;
+                builder.append(line.substring(0, matcher.start(group)));
+                builder.append(convertPath(matcher.group(group)).getAbsolutePath());
+                builder.append(line.substring(matcher.end(group)));
+            } else {
+                builder.append(line);
+            }
+            builder.append(System.lineSeparator());
+        }
+        return builder.toString();
+    }
+
+    protected String convertYmlLine(String line)
+    {
+        return line;
+    }
+
+    protected File convertPath(String name) throws URISyntaxException
+    {
+        return new File(getClass().getResource(name).toURI());
+    }
+
+    protected abstract Connection connect() throws SQLException;
+
+}

data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java

@@ -0,0 +1,78 @@
+package org.embulk.output.tester;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.EmbulkEmbed.Bootstrap;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.InjectedPluginSource;
+
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+public class EmbulkPluginTester
+{
+    private static class PluginDefinition
+    {
+        public final Class<?> iface;
+        public final String name;
+        public final Class<?> impl;
+
+
+        public PluginDefinition(Class<?> iface, String name, Class<?> impl)
+        {
+            this.iface = iface;
+            this.name = name;
+            this.impl = impl;
+        }
+
+    }
+
+    private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
+
+    private EmbulkEmbed embulk;
+
+    public EmbulkPluginTester()
+    {
+    }
+
+    public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
+    {
+        addPlugin(iface, name, impl);
+    }
+
+    public void addPlugin(Class<?> iface, String name, Class<?> impl)
+    {
+        plugins.add(new PluginDefinition(iface, name, impl));
+    }
+
+    public void run(String yml) throws Exception
+    {
+        if (embulk == null) {
+            Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+            bootstrap.addModules(new Module()
+            {
+                @Override
+                public void configure(Binder binder)
+                {
+                    for (PluginDefinition plugin : plugins) {
+                        InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
+                    }
+                }
+            });
+            embulk = bootstrap.initializeCloseable();
+        }
+        ConfigSource config = embulk.newConfigLoader().fromYamlString(yml);
+        embulk.run(config);
+    }
+
+    public void destroy()
+    {
+        if (embulk != null) {
+            embulk.destroy();
+            embulk = null;
+        }
+    }
+
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.5.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-14 00:00:00.000000000 Z
+date: 2016-01-15 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -52,8 +52,10 @@ files:
 - src/main/java/org/embulk/output/jdbc/setter/SqlTimeColumnSetter.java
 - src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java
 - src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java
+- src/test/java/org/embulk/output/AbstractJdbcOutputPluginTest.java
 - src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
-- classpath/embulk-output-jdbc-0.4.5.jar
+- src/test/java/org/embulk/output/tester/EmbulkPluginTester.java
+- classpath/embulk-output-jdbc-0.5.0.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0