embulk-output-sqlserver 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 76d8fb06a79d4382fab1f378b165ab10518fe9ef
4
+ data.tar.gz: cc766e5eeff96a7a1e74e00893a01beb58a13e83
5
+ SHA512:
6
+ metadata.gz: e39ecf1f59e3b3c555733821925e0a098be114b1d15844c6bcb1f8349de93168cb46bbfc999f427f5231f061d89bfae509e875d9d480e366224e639385dccd54
7
+ data.tar.gz: 45419b194e670602d291ff1e1ee5ecedb0c7a6d9b236340e056ce82a37b1cdc7838dd01cc03ebafa41470f10b5cef673ea25f85a902b4a625ffaab7501a00968
data/README.md ADDED
@@ -0,0 +1,95 @@
1
+ # SQL Server output plugins for Embulk
2
+
3
+ SQL Server output plugins for Embulk loads records to SQL Server.
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: output
8
+ * **Load all or nothing**: depnds on the mode. see bellow.
9
+ * **Resume supported**: depnds on the mode. see bellow.
10
+
11
+ ## Configuration
12
+
13
+ - **driver_path**: path to the jar file of the SQL Server JDBC driver (string)
14
+ - **host**: database host name (string, required)
15
+ - **port**: database port number (integer, default: 1433)
16
+ - **integratedSecutiry**: whether to use integrated authentication or not. The `sqljdbc_auth.dll` must be located on Java library path if using integrated authentication. : (boolean, default: false)
17
+ ```
18
+ rem C:\drivers\sqljdbc_auth.dll
19
+ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
20
+ ```
21
+ - **user**: database login user name (string, required if not using integrated authentication)
22
+ - **password**: database login password (string, default: "")
23
+ - **instance**: destination instance name (string, default: use the default instance)
24
+ - **database**: destination database name (string, default: use the default database)
25
+ - **url**: URL of the JDBC connection (string, optional)
26
+ - **table**: destination table name (string, required)
27
+ - **options**: extra connection properties (hash, default: {})
28
+ - **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See bellow. (string, required)
29
+ - **batch_size**: size of a single batch insert (integer, default: 16777216)
30
+ - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
31
+ - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
32
+ - **type**: type of a column when this plugin creates new tables (e.g. `VARCHAR(255)`, `INTEGER NOT NULL UNIQUE`). This used when this plugin creates intermediate tables (insert, insert_truncate and merge modes), when it creates the target table (insert_direct, merge_direct and replace modes), and when it creates nonexistent target table automatically. (string, default: depends on input column type. `BIGINT` if input column type is long, `BOOLEAN` if boolean, `DOUBLE PRECISION` if double, `CLOB` if string, `TIMESTAMP` if timestamp)
33
+ - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on input column type. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `null`, `pass`)
34
+ - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
35
+ - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
36
+
37
+ ### Modes
38
+
39
+ * **insert**:
40
+ * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `INSERT INTO <target_table> SELECT * FROM <intermediate_table_1> UNION ALL SELECT * FROM <intermediate_table_2> UNION ALL ...` query. If the target table doesn't exist, it is created automatically.
41
+ * Transactional: Yes. This mode successfully writes all rows, or fails with writing zero rows.
42
+ * Resumable: Yes.
43
+ * **insert_direct**:
44
+ * Behavior: This mode inserts rows to the target table directly. If the target table doesn't exist, it is created automatically.
45
+ * Transactional: No. If fails, the target table could have some rows inserted.
46
+ * Resumable: No.
47
+ * **truncate_insert**:
48
+ * Behavior: Same with `insert` mode excepting that it truncates the target table right before the last `INSERT ...` query.
49
+ * Transactional: Yes.
50
+ * Resumable: Yes.
51
+ * **replace**:
52
+ * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
53
+ * Transactional: No. If fails, the target table could be dropped (because SQL Server can't rollback DDL).
54
+ * Resumable: No.
55
+
56
+ ### Example
57
+
58
+ ```yaml
59
+ out:
60
+ type: sqlserver
61
+ driver_path: C:\drivers\sqljdbc41.jar
62
+ host: localhost
63
+ user: myuser
64
+ password: ""
65
+ instance: MSSQLSERVER
66
+ database: my_database
67
+ table: my_table
68
+ mode: insert
69
+ ```
70
+
71
+ Advanced configuration:
72
+
73
+ ```yaml
74
+ out:
75
+ type: sqlserver
76
+ driver_path: C:\drivers\sqljdbc41.jar
77
+ host: localhost
78
+ user: myuser
79
+ password: ""
80
+ instance: MSSQLSERVER
81
+ database: my_database
82
+ table: my_table
83
+ mode: insert_direct
84
+ column_options:
85
+ my_col_1: {type: 'TEXT'}
86
+ my_col_3: {type: 'INT NOT NULL'}
87
+ my_col_4: {value_type: string, timestamp_format: `%Y-%m-%d %H:%M:%S %z`, timezone: '-0700'}
88
+ my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
89
+ ```
90
+
91
+ ### Build
92
+
93
+ ```
94
+ $ ./gradlew gem
95
+ ```
data/build.gradle ADDED
@@ -0,0 +1,6 @@
1
+ [compileTestJava]*.options*.encoding = 'UTF-8'
2
+
3
+ dependencies {
4
+ compile project(':embulk-output-jdbc')
5
+ testCompile files('../embulk-output-jdbc/build/classes/test/')
6
+ }
@@ -0,0 +1,3 @@
1
+ Embulk::JavaPlugin.register_output(
2
+ :sqlserver, "org.embulk.output.SQLServerOutputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
@@ -0,0 +1,151 @@
1
+ package org.embulk.output;
2
+
3
+ import java.io.IOException;
4
+ import java.sql.SQLException;
5
+ import java.util.List;
6
+ import java.util.Properties;
7
+
8
+ import org.embulk.config.Config;
9
+ import org.embulk.config.ConfigDefault;
10
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
11
+ import org.embulk.output.jdbc.BatchInsert;
12
+ import org.embulk.output.jdbc.StandardBatchInsert;
13
+ import org.embulk.output.jdbc.setter.ColumnSetterFactory;
14
+ import org.embulk.output.sqlserver.SQLServerOutputConnector;
15
+ import org.embulk.output.sqlserver.setter.SQLServerColumnSetterFactory;
16
+ import org.joda.time.DateTimeZone;
17
+
18
+ import com.google.common.base.Optional;
19
+ import com.google.common.collect.ImmutableSet;
20
+
21
+ public class SQLServerOutputPlugin
22
+ extends AbstractJdbcOutputPlugin
23
+ {
24
+ public interface SQLServerPluginTask
25
+ extends PluginTask
26
+ {
27
+ @Config("driver_path")
28
+ @ConfigDefault("null")
29
+ public Optional<String> getDriverPath();
30
+
31
+ @Config("host")
32
+ @ConfigDefault("null")
33
+ public Optional<String> getHost();
34
+
35
+ @Config("port")
36
+ @ConfigDefault("1433")
37
+ public int getPort();
38
+
39
+ @Config("instance")
40
+ @ConfigDefault("null")
41
+ public Optional<String> getInstance();
42
+
43
+ @Config("database")
44
+ @ConfigDefault("null")
45
+ public Optional<String> getDatabase();
46
+
47
+ @Config("integratedSecurity")
48
+ @ConfigDefault("null")
49
+ public Optional<Boolean> getIntegratedSecurity();
50
+
51
+ @Config("url")
52
+ @ConfigDefault("null")
53
+ public Optional<String> getUrl();
54
+
55
+ @Config("user")
56
+ @ConfigDefault("null")
57
+ public Optional<String> getUser();
58
+
59
+ @Config("password")
60
+ @ConfigDefault("\"\"")
61
+ public Optional<String> getPassword();
62
+
63
+ }
64
+
65
+ @Override
66
+ protected Class<? extends PluginTask> getTaskClass()
67
+ {
68
+ return SQLServerPluginTask.class;
69
+ }
70
+
71
+ @Override
72
+ protected Features getFeatures(PluginTask task)
73
+ {
74
+ return new Features()
75
+ .setMaxTableNameLength(128)
76
+ .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
77
+ .setIgnoreMergeKeys(false);
78
+ }
79
+
80
+ @Override
81
+ protected SQLServerOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
82
+ {
83
+ SQLServerPluginTask sqlServerTask = (SQLServerPluginTask) task;
84
+
85
+ if (sqlServerTask.getDriverPath().isPresent()) {
86
+ loadDriverJar(sqlServerTask.getDriverPath().get());
87
+ }
88
+
89
+ String url;
90
+ if (sqlServerTask.getUrl().isPresent()) {
91
+ if (sqlServerTask.getHost().isPresent()
92
+ || sqlServerTask.getInstance().isPresent()
93
+ || sqlServerTask.getDatabase().isPresent()
94
+ || sqlServerTask.getIntegratedSecurity().isPresent()) {
95
+ throw new IllegalArgumentException("'host', 'port', 'instance', 'database' and 'integratedSecurity' parameters are invalid if 'url' parameter is set.");
96
+ }
97
+ url = sqlServerTask.getUrl().get();
98
+ } else {
99
+ if (!sqlServerTask.getHost().isPresent()) {
100
+ throw new IllegalArgumentException("Field 'host' is not set.");
101
+ }
102
+ if (!sqlServerTask.getDatabase().isPresent()) {
103
+ throw new IllegalArgumentException("Field 'database' is not set.");
104
+ }
105
+ StringBuilder urlBuilder = new StringBuilder();
106
+ if (sqlServerTask.getInstance().isPresent()) {
107
+ urlBuilder.append(String.format("jdbc:sqlserver://%s\\%s:%d",
108
+ sqlServerTask.getHost().get(), sqlServerTask.getInstance().get(), sqlServerTask.getPort()));
109
+ } else {
110
+ urlBuilder.append(String.format("jdbc:sqlserver://%s:%d",
111
+ sqlServerTask.getHost().get(), sqlServerTask.getPort()));
112
+ }
113
+ if (sqlServerTask.getDatabase().isPresent()) {
114
+ urlBuilder.append(";databaseName=" + sqlServerTask.getDatabase().get());
115
+ }
116
+ if (sqlServerTask.getIntegratedSecurity().isPresent() && sqlServerTask.getIntegratedSecurity().get()) {
117
+ urlBuilder.append(";integratedSecurity=" + sqlServerTask.getIntegratedSecurity().get());
118
+ } else {
119
+ if (!sqlServerTask.getUser().isPresent()) {
120
+ throw new IllegalArgumentException("Field 'user' is not set.");
121
+ }
122
+ if (!sqlServerTask.getPassword().isPresent()) {
123
+ throw new IllegalArgumentException("Field 'password' is not set.");
124
+ }
125
+ }
126
+ url = urlBuilder.toString();
127
+ }
128
+
129
+
130
+ Properties props = new Properties();
131
+ props.putAll(sqlServerTask.getOptions());
132
+
133
+ props.setProperty("user", sqlServerTask.getUser().get());
134
+ logger.info("Connecting to {} options {}", url, props);
135
+ props.setProperty("password", sqlServerTask.getPassword().get());
136
+
137
+ return new SQLServerOutputConnector(url, props, null);
138
+ }
139
+
140
+ @Override
141
+ protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
142
+ {
143
+ return new StandardBatchInsert(getConnector(task, true), mergeKeys);
144
+ }
145
+
146
+ @Override
147
+ protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, DateTimeZone defaultTimeZone)
148
+ {
149
+ return new SQLServerColumnSetterFactory(batch, defaultTimeZone);
150
+ }
151
+ }
@@ -0,0 +1,124 @@
1
+ package org.embulk.output.sqlserver;
2
+
3
+ import java.sql.Connection;
4
+ import java.sql.SQLException;
5
+ import java.sql.Statement;
6
+ import java.util.Arrays;
7
+
8
+ import org.embulk.output.jdbc.JdbcColumn;
9
+ import org.embulk.output.jdbc.JdbcOutputConnection;
10
+ import org.embulk.output.jdbc.JdbcSchema;
11
+
12
+ public class SQLServerOutputConnection
13
+ extends JdbcOutputConnection
14
+ {
15
+ public SQLServerOutputConnection(Connection connection, String schemaName, boolean autoCommit)
16
+ throws SQLException
17
+ {
18
+ super(connection, schemaName);
19
+ connection.setAutoCommit(autoCommit);
20
+ }
21
+
22
+ @Override
23
+ protected String buildRenameTableSql(String fromTable, String toTable)
24
+ {
25
+ StringBuilder sb = new StringBuilder();
26
+ sb.append("EXEC sp_rename ");
27
+ sb.append(quoteIdentifierString(fromTable));
28
+ sb.append(", ");
29
+ sb.append(quoteIdentifierString(toTable));
30
+ sb.append(", 'OBJECT'");
31
+ return sb.toString();
32
+ }
33
+
34
+ @Override
35
+ protected String buildColumnTypeName(JdbcColumn c)
36
+ {
37
+ switch(c.getSimpleTypeName()) {
38
+ case "BOOLEAN":
39
+ return "BIT";
40
+ case "CLOB":
41
+ return "TEXT";
42
+ case "TIMESTAMP":
43
+ return "DATETIME2";
44
+ default:
45
+ return super.buildColumnTypeName(c);
46
+ }
47
+ }
48
+
49
+ @Override
50
+ protected void setSearchPath(String schema) throws SQLException
51
+ {
52
+ // NOP
53
+ }
54
+
55
+ @Override
56
+ public void dropTableIfExists(String tableName) throws SQLException
57
+ {
58
+ if (tableExists(tableName)) {
59
+ dropTable(tableName);
60
+ }
61
+ }
62
+
63
+ @Override
64
+ protected void dropTableIfExists(Statement stmt, String tableName) throws SQLException
65
+ {
66
+ if (tableExists(tableName)) {
67
+ dropTable(stmt, tableName);
68
+ }
69
+ }
70
+
71
+ @Override
72
+ public void createTableIfNotExists(String tableName, JdbcSchema schema) throws SQLException
73
+ {
74
+ if (!tableExists(tableName)) {
75
+ createTable(tableName, schema);
76
+ }
77
+ }
78
+
79
+ public void createTable(String tableName, JdbcSchema schema) throws SQLException
80
+ {
81
+ Statement stmt = connection.createStatement();
82
+ try {
83
+ String sql = buildCreateTableSql(tableName, schema);
84
+ executeUpdate(stmt, sql);
85
+ commitIfNecessary(connection);
86
+ } catch (SQLException ex) {
87
+ throw safeRollback(connection, ex);
88
+ } finally {
89
+ stmt.close();
90
+ }
91
+ }
92
+
93
+ protected String buildCreateTableSql(String name, JdbcSchema schema)
94
+ {
95
+ StringBuilder sb = new StringBuilder();
96
+
97
+ sb.append("CREATE TABLE ");
98
+ quoteIdentifierString(sb, name);
99
+ sb.append(buildCreateTableSchemaSql(schema));
100
+ return sb.toString();
101
+ }
102
+
103
+ private static final String[] SIMPLE_TYPE_NAMES = {
104
+ "BIT", "FLOAT",
105
+ };
106
+
107
+ @Override
108
+ protected ColumnDeclareType getColumnDeclareType(String convertedTypeName, JdbcColumn col)
109
+ {
110
+ if (Arrays.asList(SIMPLE_TYPE_NAMES).contains(convertedTypeName)) {
111
+ return ColumnDeclareType.SIMPLE;
112
+ }
113
+ return super.getColumnDeclareType(convertedTypeName, col);
114
+ }
115
+
116
+ /*
117
+ @Override
118
+ public Charset getTableNameCharset() throws SQLException
119
+ {
120
+ return getOracleCharset().getJavaCharset();
121
+ }
122
+
123
+ */
124
+ }
@@ -0,0 +1,49 @@
1
+ package org.embulk.output.sqlserver;
2
+
3
+ import java.sql.Connection;
4
+ import java.sql.DriverManager;
5
+ import java.sql.SQLException;
6
+ import java.util.Properties;
7
+
8
+ import org.embulk.output.jdbc.JdbcOutputConnector;
9
+
10
+ public class SQLServerOutputConnector
11
+ implements JdbcOutputConnector
12
+ {
13
+ private final String url;
14
+ private final Properties properties;
15
+ private final String schemaName;
16
+
17
+ public SQLServerOutputConnector(String url, Properties properties, String schemaName)
18
+ {
19
+ try {
20
+ Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
21
+ } catch (Exception ex) {
22
+ throw new RuntimeException(ex);
23
+ }
24
+ this.url = url;
25
+ this.properties = properties;
26
+ this.schemaName = schemaName;
27
+ }
28
+
29
+ @Override
30
+ public SQLServerOutputConnection connect(boolean autoCommit) throws SQLException
31
+ {
32
+ Connection c = DriverManager.getConnection(url, properties);
33
+ if (c == null) {
34
+ // driver.connect returns null when url is "jdbc:mysql://...".
35
+ throw new SQLException("Invalid url : " + url);
36
+ }
37
+
38
+ try {
39
+ SQLServerOutputConnection con = new SQLServerOutputConnection(c, schemaName, autoCommit);
40
+ c = null;
41
+ return con;
42
+
43
+ } finally {
44
+ if (c != null) {
45
+ c.close();
46
+ }
47
+ }
48
+ }
49
+ }