embulk-output-jdbc 0.6.5 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: edd24bb5ee6ad68663c74eaa12c99d2f07634d56
-  data.tar.gz: bad5af8ddc392000053fad1145e855ef9b53327e
+  metadata.gz: 1e7b878973e6e97c5149c469886b1d11c9efc8d9
+  data.tar.gz: 3af9509130f455d733b78de7d0805907d259bdc2
 SHA512:
-  metadata.gz: b824193cebbd1c6132e4e98e6b08ca2d03bdc82e9dbbb62b12cec0540a471366a42e99f8936479423de51070b32fd9724f9968860e7e19e27c30968fa1075875
-  data.tar.gz: 0512de6ab6d2091d74f37039ce9b2b6e0a73dd770ebb783891b6837b39e17f643fe1b230852ee33465ebeb0e09e276f909c0f262c340bc5c24d37aa4c9e19925
+  metadata.gz: 586210422b7503a9921dcf879ab13b2fd6691521cddb5afb37ef51349c844173f6ac4b6506b048e8f01d08f4f172724341b4e6a40097c8c508b8f17119248659
+  data.tar.gz: cf113f39192e5ec7dbba13c33947c1095dcf03c1297a0fbfebc251c219725a4a8ffc0580551d2001ff45cf3ea865e62faa5af133a5baa2dae70f7cd109d4ce84

data/README.md

@@ -0,0 +1,99 @@
+# Generic JDBC output plugin for Embulk
+
+Generic JDBC output plugin for Embulk loads records to a database using a JDBC driver. If the database follows ANSI SQL standards and JDBC standards strictly, this plugin works. But because of many incompatibilities, use case of this plugin is very limited. It's recommended to use specific plugins for the databases.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depends on the mode. see below.
+* **Resume supported**: depends on the mode. see below.
+
+## Configuration
+
+- **driver_path**: path to the jar file of the JDBC driver (e.g. 'sqlite-jdbc-3.8.7.jar') (string, optional)
+- **driver_class**: class name of the JDBC driver (e.g. 'org.sqlite.JDBC') (string, required)
+- **url**: URL of the JDBC connection (e.g. 'jdbc:sqlite:mydb.sqlite3') (string, required)
+- **user**: database login user name (string, optional)
+- **password**: database login password (string, optional)
+- **schema**: destination schema name (string, default: use default schema)
+- **table**: destination table name (string, required)
+- **options**: extra JDBC properties (hash, default: {})
+- **retry_limit** max retry count for database operations (integer, default: 12)
+- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
+- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
+- **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See below (string, required)
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **max_table_name_length**: maximum length of table name in this RDBMS (integer, default: 256)
+- **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert and truncate_insert modes), when it creates the target table (replace mode), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
+  - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
+  - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
+  - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+
+## Modes
+
+* **insert**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ...` query. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes. This mode successfully writes all rows, or fails with writing zero rows.
+  * Resumable: Yes.
+* **insert_direct**:
+  * Behavior: This mode inserts rows to the target table directly. If the target table doesn't exist, it is created automatically.
+  * Transactional: No. If fails, the target table could have some rows inserted.
+  * Resumable: No.
+* **truncate_insert**:
+  * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
+  * Transactional: Yes.
+  * Resumable: Yes.
+* **replace**:
+  * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
+  * Transactional: No. If fails, the target table could be dropped.
+  * Resumable: No.
+* **merge**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, merges the intermediate tables into the target table. Namely, if primary keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes.
+  * Resumable: Yes.
+* **merge_direct**:
+  * Behavior: This mode merges rows to the target table directly. Namely, if primary keys of an input record already exist in the target table, the target record is updated by the input record, otherwise the input record is inserted. If the target table doesn't exist, it is created automatically.
+  * Transactional: No.
+  * Resumable: No.
+
+## Example
+
+```yaml
+out:
+  type: jdbc
+  driver_path: /usr/local/nz/lib/nzjdbc3.jar
+  driver_class: org.netezza.Driver
+  url: jdbc:jdbc:netezza://127.0.0.1:5480/mydb
+  user: myuser
+  password: "mypassword"
+  table: my_table
+  mode: insert
+```
+
+Advanced configuration:
+
+```yaml
+out:
+  type: jdbc
+  driver_path: /usr/local/nz/lib/nzjdbc3.jar
+  driver_class: org.netezza.Driver
+  url: jdbc:jdbc:netezza://127.0.0.1:5480/mydb
+  user: myuser
+  password: "mypassword"
+  table: my_table
+  options: {loglevel: 2}
+  mode: insert_direct
+  column_options:
+    my_col_1: {type: 'VARCHAR(255)'}
+    my_col_3: {type: 'INT NOT NULL'}
+    my_col_4: {value_type: string, timestamp_format: `%Y-%m-%d %H:%M:%S %z`, timezone: '-0700'}
+    my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/src/test/java/org/embulk/output/AbstractJdbcOutputPluginTest.java

@@ -1,6 +1,9 @@
 package org.embulk.output;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.net.URISyntaxException;
 import java.nio.charset.Charset;
 import java.sql.Connection;
@@ -11,17 +14,115 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import org.embulk.config.ConfigException;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.tester.EmbulkPluginTester;
+import org.embulk.output.tester.EmbulkPluginTester.PluginDefinition;
+import org.yaml.snakeyaml.Yaml;
 
 import com.google.common.io.Files;
 
+import static java.util.Locale.ENGLISH;
+
 public abstract class AbstractJdbcOutputPluginTest
 {
-    protected static boolean enabled;
-    protected static EmbulkPluginTester tester = new EmbulkPluginTester();
+    private static final String CONFIG_FILE_NAME = "tests.yml";
+
+    protected boolean enabled;
+    // TODO:destroy EmbulkPluginTester after test
+    protected EmbulkPluginTester tester = new EmbulkPluginTester();
+    private String pluginName;
+    private Map<String, ?> testConfigurations;
+
+    protected AbstractJdbcOutputPluginTest()
+    {
+        try {
+            prepare();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected abstract void prepare() throws SQLException;
+
+
+    private Map<String, ?> getTestConfigs()
+    {
+        if (testConfigurations == null) {
+            for (PluginDefinition pluginDefinition : tester.getPlugins()) {
+                if (AbstractJdbcOutputPlugin.class.isAssignableFrom(pluginDefinition.impl)) {
+                    pluginName = pluginDefinition.name;
+                    break;
+                }
+            }
+
+            Yaml yaml = new Yaml();
+            File configFile = new File(CONFIG_FILE_NAME);
+            if (!configFile.exists()) {
+                configFile = new File("../" + CONFIG_FILE_NAME);
+                if (!configFile.exists()) {
+                    throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't exist.",
+                            CONFIG_FILE_NAME));
+                }
+            }
+
+            try {
+                InputStreamReader reader = new InputStreamReader(new FileInputStream(configFile), Charset.forName("UTF8"));
+                try {
+                    Map<String, ?> allTestConfigs = (Map<String, ?>)yaml.load(reader);
+                    if (!allTestConfigs.containsKey(pluginName)) {
+                        throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't contain \"%s\" element.",
+                                CONFIG_FILE_NAME, pluginName));
+                    }
+                    testConfigurations = (Map<String, ?>)allTestConfigs.get(pluginName);
+                } finally {
+                    reader.close();
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return testConfigurations;
+    }
+
+    private Object getTestConfig(String name)
+    {
+        Map<String, ?> testConfigs = getTestConfigs();
+        if (!testConfigs.containsKey(name)) {
+            throw new ConfigException(String.format(ENGLISH, "\"%s\" element in \"%s\" doesn't contain \"%s\" element.",
+                    pluginName, CONFIG_FILE_NAME, name));
+        }
+        return testConfigs.get(name);
+    }
+
+    protected String getHost()
+    {
+        return (String)getTestConfig("host");
+    }
+
+    protected int getPort()
+    {
+        return (Integer)getTestConfig("port");
+    }
+
+    protected String getUser()
+    {
+        return (String)getTestConfig("user");
+    }
+
+    protected String getPassword()
+    {
+        return (String)getTestConfig("password");
+    }
+
+    protected String getDatabase()
+    {
+        return (String)getTestConfig("database");
+    }
 
     protected void dropTable(String table) throws SQLException
     {
@@ -122,6 +223,11 @@ public abstract class AbstractJdbcOutputPluginTest
 
     protected String convertYmlLine(String line)
     {
+        line = line.replaceAll("#host#", getHost());
+        line = line.replaceAll("#port#", Integer.toString(getPort()));
+        line = line.replaceAll("#database#", getDatabase());
+        line = line.replaceAll("#user#", getUser());
+        line = line.replaceAll("#password#", getPassword());
         return line;
     }
 

data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java

@@ -13,7 +13,7 @@ import com.google.inject.Module;
 
 public class EmbulkPluginTester
 {
-    private static class PluginDefinition
+    public static class PluginDefinition
     {
         public final Class<?> iface;
         public final String name;
@@ -47,6 +47,11 @@ public class EmbulkPluginTester
         plugins.add(new PluginDefinition(iface, name, impl));
     }
 
+    public List<PluginDefinition> getPlugins()
+    {
+        return plugins;
+    }
+
     public void run(String yml) throws Exception
     {
         if (embulk == null) {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.6.5
+  version: 0.7.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-11 00:00:00.000000000 Z
+date: 2016-10-26 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -17,8 +17,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.6.5.jar
+- classpath/embulk-output-jdbc-0.7.0.jar
 - lib/embulk/output/jdbc.rb
 - src/main/java/org/embulk/output/JdbcOutputPlugin.java
 - src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java