embulk-input-postgresql 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +42 -0
- data/build.gradle +7 -0
- data/classpath/embulk-input-jdbc-0.1.0.jar +0 -0
- data/classpath/embulk-input-postgresql-0.1.0.jar +0 -0
- data/classpath/jna-4.1.0.jar +0 -0
- data/classpath/jna-platform-4.1.0.jar +0 -0
- data/classpath/postgresql-9.4-1200-jdbc41.jar +0 -0
- data/classpath/slf4j-simple-1.7.7.jar +0 -0
- data/classpath/waffle-jna-1.7.jar +0 -0
- data/lib/embulk/input/postgresql.rb +3 -0
- data/src/main/java/org/embulk/input/PostgreSQLInputPlugin.java +60 -0
- data/src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java +60 -0
- metadata +56 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2a8135b43af2ab08ff290956926610cab241f9da
|
4
|
+
data.tar.gz: 8c6ad6f3da2c5a86a69225fbc0c18fc82f2bfcfc
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4263cbf52d0f00aa26d19d621baf2a3c14a31bd90f5e2505a4e891fdaee9118d2c87977ac48dd84d65c1051f224a7c74b50fe4be75aa485ac0c05d2c539ef544
|
7
|
+
data.tar.gz: 0f81e7b36f57a492cb67508d6a813807351d91b0ce9ce30a0fdb9c56fa0b45a665c78abc077d0e85f91732d5ec8b89d55ae83c50144ee7d67dc3c6044022d5a0
|
data/README.md
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
# PostgreSQL input plugins for Embulk
|
2
|
+
|
3
|
+
PostgreSQL input plugins for Embulk loads records from PostgreSQL.
|
4
|
+
|
5
|
+
## Overview
|
6
|
+
|
7
|
+
* **Plugin type**: input
|
8
|
+
* **Resume supported**: yes
|
9
|
+
|
10
|
+
## Configuration
|
11
|
+
|
12
|
+
- **host**: database host name (string, required)
|
13
|
+
- **port**: database port number (integer, 5432)
|
14
|
+
- **user**: database login user name (string, required)
|
15
|
+
- **password**: database login password (string, default: "")
|
16
|
+
- **database**: destination database name (string, required)
|
17
|
+
- **schema**: destination name (string, default: "public")
|
18
|
+
- **table**: destination name (string, required)
|
19
|
+
- **select**: comma-separated list of columns to select (string, default: "*")
|
20
|
+
- **where**: WHERE condition to filter the rows (string, default: no-condition)
|
21
|
+
- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
|
22
|
+
- **options**: extra JDBC properties (hash, default: {})
|
23
|
+
|
24
|
+
## Example
|
25
|
+
|
26
|
+
```yaml
|
27
|
+
in:
|
28
|
+
type: postgresql
|
29
|
+
host: localhost
|
30
|
+
user: myuser
|
31
|
+
password: ""
|
32
|
+
database: my_database
|
33
|
+
table: my_table
|
34
|
+
select: "col1, col2, col3"
|
35
|
+
where: "col4 != 'a'"
|
36
|
+
```
|
37
|
+
|
38
|
+
## Build
|
39
|
+
|
40
|
+
```
|
41
|
+
$ ./gradlew gem
|
42
|
+
```
|
data/build.gradle
ADDED
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -0,0 +1,60 @@
|
|
1
|
+
package org.embulk.input;
|
2
|
+
|
3
|
+
import java.util.Properties;
|
4
|
+
import java.sql.Connection;
|
5
|
+
import java.sql.Driver;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import com.google.common.base.Throwables;
|
8
|
+
import org.embulk.config.Config;
|
9
|
+
import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
|
10
|
+
import org.embulk.input.postgresql.PostgreSQLInputConnection;
|
11
|
+
|
12
|
+
public class PostgreSQLInputPlugin
|
13
|
+
extends AbstractJdbcInputPlugin
|
14
|
+
{
|
15
|
+
private static final String DEFAULT_SCHEMA = "public";
|
16
|
+
private static final int DEFAULT_PORT = 5432;
|
17
|
+
|
18
|
+
private static final Driver driver = new org.postgresql.Driver();
|
19
|
+
|
20
|
+
@Override
|
21
|
+
protected PostgreSQLInputConnection newConnection(PluginTask task) throws SQLException
|
22
|
+
{
|
23
|
+
String url = String.format("jdbc:postgresql://%s:%d/%s",
|
24
|
+
task.getHost(), task.getPort().or(DEFAULT_PORT), task.getDatabase());
|
25
|
+
|
26
|
+
Properties props = new Properties();
|
27
|
+
props.setProperty("user", task.getUser());
|
28
|
+
props.setProperty("password", task.getPassword());
|
29
|
+
props.setProperty("loginTimeout", "300"); // seconds
|
30
|
+
props.setProperty("socketTimeout", "1800"); // seconds
|
31
|
+
|
32
|
+
// Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
|
33
|
+
// Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
|
34
|
+
props.setProperty("tcpKeepAlive", "true");
|
35
|
+
|
36
|
+
// TODO
|
37
|
+
//switch task.getSssl() {
|
38
|
+
//when "disable":
|
39
|
+
// break;
|
40
|
+
//when "enable":
|
41
|
+
// props.setProperty("sslfactory", "org.postgresql.ssl.NonValidatingFactory"); // disable server-side validation
|
42
|
+
//when "verify":
|
43
|
+
// props.setProperty("ssl", "true");
|
44
|
+
// break;
|
45
|
+
//}
|
46
|
+
|
47
|
+
props.putAll(task.getOptions());
|
48
|
+
|
49
|
+
Connection con = driver.connect(url, props);
|
50
|
+
try {
|
51
|
+
PostgreSQLInputConnection c = new PostgreSQLInputConnection(con, task.getSchema().or(DEFAULT_SCHEMA));
|
52
|
+
con = null;
|
53
|
+
return c;
|
54
|
+
} finally {
|
55
|
+
if (con != null) {
|
56
|
+
con.close();
|
57
|
+
}
|
58
|
+
}
|
59
|
+
}
|
60
|
+
}
|
@@ -0,0 +1,60 @@
|
|
1
|
+
package org.embulk.input.postgresql;
|
2
|
+
|
3
|
+
import java.sql.Connection;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.ResultSet;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import org.slf4j.Logger;
|
8
|
+
import org.embulk.spi.Exec;
|
9
|
+
import org.embulk.input.jdbc.JdbcInputConnection;
|
10
|
+
|
11
|
+
public class PostgreSQLInputConnection
|
12
|
+
extends JdbcInputConnection
|
13
|
+
{
|
14
|
+
private final Logger logger = Exec.getLogger(PostgreSQLInputConnection.class);
|
15
|
+
|
16
|
+
public PostgreSQLInputConnection(Connection connection, String schemaName)
|
17
|
+
throws SQLException
|
18
|
+
{
|
19
|
+
super(connection, schemaName);
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
protected CursorSelect newBatchSelect(String select, int fetchRows) throws SQLException
|
24
|
+
{
|
25
|
+
executeUpdate("DECLARE cur NO SCROLL CURSOR FOR "+select);
|
26
|
+
|
27
|
+
String fetchSql = "FETCH FORWARD "+fetchRows+" FROM cur";
|
28
|
+
return new CursorSelect(fetchSql, connection.prepareStatement(fetchSql));
|
29
|
+
}
|
30
|
+
|
31
|
+
public class CursorSelect
|
32
|
+
implements BatchSelect
|
33
|
+
{
|
34
|
+
private final String fetchSql;
|
35
|
+
private final PreparedStatement fetchStatement;
|
36
|
+
|
37
|
+
public CursorSelect(String fetchSql, PreparedStatement fetchStatement) throws SQLException
|
38
|
+
{
|
39
|
+
this.fetchSql = fetchSql;
|
40
|
+
this.fetchStatement = fetchStatement;
|
41
|
+
}
|
42
|
+
|
43
|
+
public ResultSet fetch() throws SQLException
|
44
|
+
{
|
45
|
+
logger.info("SQL: " + fetchSql);
|
46
|
+
long startTime = System.currentTimeMillis();
|
47
|
+
|
48
|
+
ResultSet rs = fetchStatement.executeQuery();
|
49
|
+
|
50
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
51
|
+
logger.info(String.format("> %.2f seconds", seconds));
|
52
|
+
return rs;
|
53
|
+
}
|
54
|
+
|
55
|
+
public void close() throws SQLException
|
56
|
+
{
|
57
|
+
// TODO close?
|
58
|
+
}
|
59
|
+
}
|
60
|
+
}
|
metadata
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-input-postgresql
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- FURUHASHI Sadayuki
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-02-16 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: JDBC input plugin is an Embulk plugin that loads records from JDBC so that any output plugins can receive the records. Search the output plugins by "embulk-output" keyword.
|
14
|
+
email:
|
15
|
+
- frsyuki@users.sourceforge.jp
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- README.md
|
21
|
+
- build.gradle
|
22
|
+
- lib/embulk/input/postgresql.rb
|
23
|
+
- src/main/java/org/embulk/input/PostgreSQLInputPlugin.java
|
24
|
+
- src/main/java/org/embulk/input/postgresql/PostgreSQLInputConnection.java
|
25
|
+
- classpath/embulk-input-jdbc-0.1.0.jar
|
26
|
+
- classpath/embulk-input-postgresql-0.1.0.jar
|
27
|
+
- classpath/jna-4.1.0.jar
|
28
|
+
- classpath/jna-platform-4.1.0.jar
|
29
|
+
- classpath/postgresql-9.4-1200-jdbc41.jar
|
30
|
+
- classpath/slf4j-simple-1.7.7.jar
|
31
|
+
- classpath/waffle-jna-1.7.jar
|
32
|
+
homepage: https://github.com/embulk/embulk-input-jdbc
|
33
|
+
licenses:
|
34
|
+
- Apache 2.0
|
35
|
+
metadata: {}
|
36
|
+
post_install_message:
|
37
|
+
rdoc_options: []
|
38
|
+
require_paths:
|
39
|
+
- lib
|
40
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
41
|
+
requirements:
|
42
|
+
- - '>='
|
43
|
+
- !ruby/object:Gem::Version
|
44
|
+
version: '0'
|
45
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - '>='
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '0'
|
50
|
+
requirements: []
|
51
|
+
rubyforge_project:
|
52
|
+
rubygems_version: 2.1.9
|
53
|
+
signing_key:
|
54
|
+
specification_version: 4
|
55
|
+
summary: JDBC input plugin for Embulk
|
56
|
+
test_files: []
|