embulk-input-jdbc 0.7.4 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f0e8890c29d644c993bd0507a380802fb4c0e2c
-  data.tar.gz: 535b176cb211f3115482534f2a6b0373923c9e15
+  metadata.gz: 121506005fab6a020ff1ab8e908f07e49dd7c5fb
+  data.tar.gz: 85081ddc42add58819ac8b4a0a9a5e8e52c1f129
 SHA512:
-  metadata.gz: 4f774a9727295f35f9e16c06e9fd57d6918398f44ab070a55606b62a9402fa3857b1cb1823f67ef2388e1a016c016a443d2d40bb3b41c7fda28c0f45991378bd
-  data.tar.gz: 2fe9ba6cbf6d2cc768f0d641371e0a08e464ee9cbfa91bf5016bea97ac9bf1e1a24ecb56110e9fe49c22251a9f1df56c59b9115bfae73c2622f2bcc02227a37d
+  metadata.gz: 13c70bfde5dcc988607ea3f77dc84a70402b90878c7bf536ef5e5d3a2530e370b5b75f2eac04dabaabbccbbac9180db60c9612bbebfbf0116c5761adcaae22e2
+  data.tar.gz: 41c9d0e17c2a825c740d2e0467dc3b1582b8f1d8f6649d8f42d5940ef9e138be731a679a4364c5823a13bc2b46d0949fbf623c00c383a109bd2bd110609c3283

data/README.md

@@ -0,0 +1,148 @@
+# Generic JDBC input plugin for Embulk
+
+Generic JDBC input plugin for Embulk loads records from a database using a JDBC driver. If the database follows ANSI SQL standards and JDBC standards strictly, this plugin works. But because of many incompatibilities, use case of this plugin is very limited. It's recommended to use specific plugins for the databases.
+
+## Overview
+
+* **Plugin type**: input
+* **Resume supported**: yes
+
+## Configuration
+
+- **driver_path**: path to the jar file of the JDBC driver (e.g. 'sqlite-jdbc-3.8.7.jar') (string, optional)
+- **driver_class**: class name of the JDBC driver (e.g. 'org.sqlite.JDBC') (string, required)
+- **url**: URL of the JDBC connection (e.g. 'jdbc:sqlite:mydb.sqlite3') (string, required)
+- **user**: database login user name (string, optional)
+- **password**: database login password (string, default: optional)
+- **schema**: destination schema name (string, default: use the default schema)
+- **fetch_rows**: number of rows to fetch one time (integer, default: 10000)
+- **connect_timeout**: not supported.
+- **socket_timeout**: timeout for executing the query. 0 means no timeout. (integer (seconds), default: 1800)
+- **options**: extra JDBC properties (hash, default: {})
+- If you write SQL directly,
+  - **query**: SQL to run (string)
+- If **query** is not set,
+  - **table**: destination table name (string, required)
+  - **select**: expression of select (e.g. `id, created_at`) (string, default: "*")
+  - **where**: WHERE condition to filter the rows (string, default: no-condition)
+  - **order_by**: expression of ORDER BY to sort rows (e.g. `created_at DESC, id ASC`) (string, default: not sorted)
+- **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
+- **default_column_options**: column_options for each JDBC type as default. Key is a JDBC type (e.g. 'DATE', 'BIGINT'). Value is same as column_options's value.
+- **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
+  - **value_type**: embulk get values from database as this value_type. Typically, the value_type determines `getXXX` method of `java.sql.PreparedStatement`. `value_type: json` is an exception which uses `getString` and parses the result as a JSON string.
+  (string, default: depends on the sql type of the column. Available values options are: `long`, `double`, `float`, `decimal`, `boolean`, `string`, `json`, `date`, `time`, `timestamp`)
+  - **type**: Column values are converted to this embulk type.
+  Available values options are: `boolean`, `long`, `double`, `string`, `json`, `timestamp`).
+  By default, the embulk type is determined according to the sql type of the column (or value_type if specified).
+  - **timestamp_format**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted by this timestamp_format. And if the embulk type is `timestamp`, this timestamp_format may be used in the output plugin. For example, stdout plugin use the timestamp_format, but *csv formatter plugin doesn't use*. (string, default : `%Y-%m-%d` for `date`, `%H:%M:%S` for `time`, `%Y-%m-%d %H:%M:%S` for `timestamp`)
+  - **timezone**: If the sql type of the column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted in this timezone.
+(string, value of default_timezone option is used by default)
+- **after_select**: if set, this SQL will be executed after the SELECT query in the same transaction.
+
+
+## Incremental loading
+
+Incremental loading uses monotonically increasing unique columns (such as auto-increment id) to load records inserted (or updated) after last execution.
+
+First, if `incremental: true` is set, this plugin loads all records with additional ORDER BY. For example, if `incremental_columns: [updated_at, id]` option is set, query will be as following:
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+ORDER BY updated_at, id
+```
+
+When bulk data loading finishes successfully, it outputs `last_record: ` paramater as config-diff so that next execution uses it.
+
+At the next execution, when `last_record: ` is also set, this plugin generates additional WHERE conditions to load records larger than the last record. For example, if `last_record: ["2017-01-01 00:32:12", 5291]` is set,
+
+```
+SELECT * FROM (
+  ...original query is here...
+)
+WHERE created_at > '2017-01-01 00:32:12' OR (created_at = '2017-01-01 00:32:12' AND id > 5291)
+ORDER BY updated_at, id
+```
+
+Then, it updates `last_record: ` so that next execution uses the updated last_record.
+
+**IMPORTANT**: If you set `incremental_columns: ` option, make sure that there is an index on the columns to avoid full table scan. For this example, following index should be created:
+
+```
+CREATE INDEX embulk_incremental_loading_index ON table (updated_at, id);
+```
+
+Recommended usage is to leave `incremental_columns` unset and let this plugin automatically finds an auto-increment primary key. Currently, only strings and integers are supported as incremental_columns.
+
+
+## Example
+
+```yaml
+in:
+  type: jdbc
+  driver_path: /opt/oracle/ojdbc6.jar
+  driver_class: oracle.jdbc.driver.OracleDriver
+  url: jdbc:oracle:thin:@127.0.0.1:1521:mydb
+  user: myuser
+  password: "mypassword"
+  table: "my_table"
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+  order_by: "col1 DESC"
+```
+
+This configuration will generate following SQL:
+
+```
+SELECT col1, col2, col3
+FROM "my_table"
+WHERE col4 != 'a'
+ORDER BY col1 DESC
+```
+
+If you need a complex SQL,
+
+```yaml
+in:
+  type: jdbc
+  driver_path: /opt/oracle/ojdbc6.jar
+  driver_class: oracle.jdbc.driver.OracleDriver
+  url: jdbc:oracle:thin:@127.0.0.1:1521:mydb
+  user: myuser
+  password: "mypassword"
+  query: |
+    SELECT t1.id, t1.name, t2.id AS t2_id, t2.name AS t2_name
+    FROM table1 AS t1
+    LEFT JOIN table2 AS t2
+      ON t1.id = t2.t1_id
+```
+
+Advanced configuration:
+
+```yaml
+in:
+  type: jdbc
+  driver_path: /opt/oracle/ojdbc6.jar
+  driver_class: oracle.jdbc.driver.OracleDriver
+  url: jdbc:oracle:thin:@127.0.0.1:1521:mydb
+  user: myuser
+  password: "mypassword"
+  table: "my_table"
+  select: "col1, col2, col3"
+  where: "col4 != 'a'"
+  default_column_options:
+    DATE: { type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
+    BIGINT: { type: string }
+  column_options:
+    col1: {type: long}
+    col3: {type: string, timestamp_format: "%Y/%m/%d", timezone: "+0900"}
+  after_select: "update my_table set col5 = '1' where col4 != 'a'"
+
+```
+
+## Build
+
+```
+$ ./gradlew gem
+```

data/src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java

@@ -0,0 +1,254 @@
+package org.embulk.input;
+
+import static java.util.Locale.ENGLISH;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.embulk.config.ConfigException;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.tester.EmbulkPluginTester;
+import org.embulk.input.tester.EmbulkPluginTester.PluginDefinition;
+import org.yaml.snakeyaml.Yaml;
+
+import com.google.common.io.Files;
+
+public abstract class AbstractJdbcInputPluginTest
+{
+    private static final String CONFIG_FILE_NAME = "tests.yml";
+
+    protected boolean enabled;
+    // TODO:destroy EmbulkPluginTester after test
+    protected EmbulkPluginTester tester = new EmbulkPluginTester();
+    private String pluginName;
+    private Map<String, ?> testConfigurations;
+
+    protected AbstractJdbcInputPluginTest()
+    {
+        try {
+            prepare();
+        } catch (SQLException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    protected abstract void prepare() throws SQLException;
+
+
+    private Map<String, ?> getTestConfigs()
+    {
+        if (testConfigurations == null) {
+            for (PluginDefinition pluginDefinition : tester.getPlugins()) {
+                if (AbstractJdbcInputPlugin.class.isAssignableFrom(pluginDefinition.impl)) {
+                    pluginName = pluginDefinition.name;
+                    break;
+                }
+            }
+
+            Yaml yaml = new Yaml();
+            File configFile = new File(CONFIG_FILE_NAME);
+            if (!configFile.exists()) {
+                configFile = new File("../" + CONFIG_FILE_NAME);
+                if (!configFile.exists()) {
+                    throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't exist.",
+                            CONFIG_FILE_NAME));
+                }
+            }
+
+            try {
+                InputStreamReader reader = new InputStreamReader(new FileInputStream(configFile), Charset.forName("UTF8"));
+                try {
+                    Map<String, ?> allTestConfigs = (Map<String, ?>)yaml.load(reader);
+                    if (!allTestConfigs.containsKey(pluginName)) {
+                        throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't contain \"%s\" element.",
+                                CONFIG_FILE_NAME, pluginName));
+                    }
+                    testConfigurations = (Map<String, ?>)allTestConfigs.get(pluginName);
+                } finally {
+                    reader.close();
+                }
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        return testConfigurations;
+    }
+
+    protected Object getTestConfig(String name, boolean required)
+    {
+        Map<String, ?> testConfigs = getTestConfigs();
+        if (!testConfigs.containsKey(name)) {
+            if (required) {
+                throw new ConfigException(String.format(ENGLISH, "\"%s\" element in \"%s\" doesn't contain \"%s\" element.",
+                        pluginName, CONFIG_FILE_NAME, name));
+            }
+            return null;
+        }
+        return testConfigs.get(name);
+    }
+
+    protected Object getTestConfig(String name)
+    {
+        return getTestConfig(name, true);
+    }
+
+    protected String getHost()
+    {
+        return (String)getTestConfig("host");
+    }
+
+    protected int getPort()
+    {
+        return (Integer)getTestConfig("port");
+    }
+
+    protected String getUser()
+    {
+        return (String)getTestConfig("user");
+    }
+
+    protected String getPassword()
+    {
+        return (String)getTestConfig("password");
+    }
+
+    protected String getDatabase()
+    {
+        return (String)getTestConfig("database");
+    }
+
+    protected void dropTable(String table) throws SQLException
+    {
+        String sql = String.format("DROP TABLE %s", table);
+        executeSQL(sql, true);
+    }
+
+    protected List<List<Object>> select(String table) throws SQLException
+    {
+        try (Connection connection = connect()) {
+            try (Statement statement = connection.createStatement()) {
+                List<List<Object>> rows = new ArrayList<List<Object>>();
+                String sql = String.format("SELECT * FROM %s", table);
+                System.out.println(sql);
+                try (ResultSet resultSet = statement.executeQuery(sql)) {
+                    while (resultSet.next()) {
+                        List<Object> row = new ArrayList<Object>();
+                        for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
+                            row.add(getValue(resultSet, i));
+                        }
+                        rows.add(row);
+                    }
+                }
+                // cannot sort by CLOB, so sort by Java
+                Collections.sort(rows, new Comparator<List<Object>>() {
+                    @Override
+                    public int compare(List<Object> o1, List<Object> o2) {
+                        return o1.toString().compareTo(o2.toString());
+                    }
+                });
+                return rows;
+            }
+        }
+
+    }
+
+    protected Object getValue(ResultSet resultSet, int index) throws SQLException
+    {
+        return resultSet.getObject(index);
+    }
+
+    protected void executeSQL(String sql) throws SQLException
+    {
+        executeSQL(sql, false);
+    }
+
+    protected void executeSQL(String sql, boolean ignoreError) throws SQLException
+    {
+        if (!enabled) {
+            return;
+        }
+
+        try (Connection connection = connect()) {
+            try {
+                connection.setAutoCommit(true);
+
+                try (Statement statement = connection.createStatement()) {
+                    System.out.println(String.format("Execute SQL : \"%s\".", sql));
+                    statement.execute(sql);
+                }
+
+            } catch (SQLException e) {
+                if (!ignoreError) {
+                    throw e;
+                }
+            }
+        }
+    }
+
+    protected void test(String ymlPath) throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        tester.run(convertYml(ymlPath));
+    }
+
+    protected String convertYml(String ymlName) throws Exception
+    {
+        StringBuilder builder = new StringBuilder();
+        Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
+        for (String line : Files.readLines(convertPath(ymlName), Charset.forName("UTF8"))) {
+            line = convertYmlLine(line);
+            Matcher matcher = pathPrefixPattern.matcher(line);
+            if (matcher.matches()) {
+                int group = 2;
+                builder.append(line.substring(0, matcher.start(group)));
+                builder.append(convertPath(matcher.group(group)).getAbsolutePath());
+                builder.append(line.substring(matcher.end(group)));
+            } else {
+                builder.append(line);
+            }
+            builder.append(System.lineSeparator());
+        }
+        return builder.toString();
+    }
+
+    protected String convertYmlLine(String line)
+    {
+        line = line.replaceAll("#host#", getHost());
+        line = line.replaceAll("#port#", Integer.toString(getPort()));
+        line = line.replaceAll("#database#", getDatabase());
+        line = line.replaceAll("#user#", getUser());
+        line = line.replaceAll("#password#", getPassword());
+        return line;
+    }
+
+    protected File convertPath(String name) throws URISyntaxException
+    {
+        return new File(getClass().getResource(name).toURI());
+    }
+
+    protected List<String> read(String path) throws IOException
+    {
+        return Files.readLines(new File(path), Charset.forName("UTF8"));
+    }
+
+    protected abstract Connection connect() throws SQLException;
+
+}

data/src/test/java/org/embulk/input/{EmbulkPluginTester.java → tester/EmbulkPluginTester.java}

@@ -1,6 +1,5 @@
-package org.embulk.input;
+package org.embulk.input.tester;
 
-import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -14,7 +13,7 @@ import com.google.inject.Module;
 
 public class EmbulkPluginTester
 {
-    private static class PluginDefinition
+    public static class PluginDefinition
     {
         public final Class<?> iface;
         public final String name;
@@ -48,7 +47,12 @@ public class EmbulkPluginTester
         plugins.add(new PluginDefinition(iface, name, impl));
     }
 
-    public void run(String ymlPath) throws Exception
+    public List<PluginDefinition> getPlugins()
+    {
+        return plugins;
+    }
+
+    public void run(String yml) throws Exception
     {
         if (embulk == null) {
             Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
@@ -64,8 +68,7 @@ public class EmbulkPluginTester
             });
             embulk = bootstrap.initializeCloseable();
         }
-
-        ConfigSource config = embulk.newConfigLoader().fromYamlFile(new File(ymlPath));
+        ConfigSource config = embulk.newConfigLoader().fromYamlString(yml);
         embulk.run(config);
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.8.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-28 00:00:00.000000000 Z
+date: 2016-10-13 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -17,8 +17,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.7.4.jar
+- classpath/embulk-input-jdbc-0.8.0.jar
 - lib/embulk/input/jdbc.rb
 - src/main/java/org/embulk/input/JdbcInputPlugin.java
 - src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
@@ -43,7 +44,8 @@ files:
 - src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
-- src/test/java/org/embulk/input/EmbulkPluginTester.java
+- src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java
+- src/test/java/org/embulk/input/tester/EmbulkPluginTester.java
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0