embulk-input-athena 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +13 -0
- data/Dockerfile +8 -0
- data/LICENSE +21 -0
- data/README.md +46 -0
- data/build.gradle +101 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/docker-compose.yml +10 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +5 -0
- data/gradlew +172 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/athena.rb +3 -0
- data/src/main/java/org/embulk/input/athena/AthenaInputConnection.java +49 -0
- data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java +202 -0
- data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1 +192 -0
- data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java +674 -0
- data/src/main/java/org/embulk/input/jdbc/JdbcColumn.java +58 -0
- data/src/main/java/org/embulk/input/jdbc/JdbcColumnOption.java +31 -0
- data/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java +397 -0
- data/src/main/java/org/embulk/input/jdbc/JdbcLiteral.java +38 -0
- data/src/main/java/org/embulk/input/jdbc/JdbcSchema.java +55 -0
- data/src/main/java/org/embulk/input/jdbc/Ssl.java +37 -0
- data/src/main/java/org/embulk/input/jdbc/ToString.java +54 -0
- data/src/main/java/org/embulk/input/jdbc/ToStringMap.java +35 -0
- data/src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java +105 -0
- data/src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java +45 -0
- data/src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java +38 -0
- data/src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java +59 -0
- data/src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java +56 -0
- data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java +21 -0
- data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java +207 -0
- data/src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java +37 -0
- data/src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java +66 -0
- data/src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java +66 -0
- data/src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java +57 -0
- data/src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java +70 -0
- data/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java +96 -0
- data/src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java +37 -0
- data/src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java +36 -0
- data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java +83 -0
- data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java +75 -0
- data/src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java +5 -0
- metadata +258 -0
@@ -0,0 +1,38 @@
|
|
1
|
+
package org.embulk.input.jdbc;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
5
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
6
|
+
|
7
|
+
public class JdbcLiteral
|
8
|
+
{
|
9
|
+
private final int columnIndex;
|
10
|
+
private final JsonNode value;
|
11
|
+
|
12
|
+
@JsonCreator
|
13
|
+
public JdbcLiteral(
|
14
|
+
@JsonProperty("columnIndex") int columnIndex,
|
15
|
+
@JsonProperty("value") JsonNode value)
|
16
|
+
{
|
17
|
+
this.columnIndex = columnIndex;
|
18
|
+
this.value = value;
|
19
|
+
}
|
20
|
+
|
21
|
+
@JsonProperty("columnIndex")
|
22
|
+
public int getColumnIndex()
|
23
|
+
{
|
24
|
+
return columnIndex;
|
25
|
+
}
|
26
|
+
|
27
|
+
@JsonProperty("value")
|
28
|
+
public JsonNode getValue()
|
29
|
+
{
|
30
|
+
return value;
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public String toString()
|
35
|
+
{
|
36
|
+
return value.toString();
|
37
|
+
}
|
38
|
+
}
|
@@ -0,0 +1,55 @@
|
|
1
|
+
package org.embulk.input.jdbc;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
6
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
7
|
+
|
8
|
+
public class JdbcSchema
|
9
|
+
{
|
10
|
+
private List<JdbcColumn> columns;
|
11
|
+
|
12
|
+
@JsonCreator
|
13
|
+
public JdbcSchema(List<JdbcColumn> columns)
|
14
|
+
{
|
15
|
+
this.columns = columns;
|
16
|
+
}
|
17
|
+
|
18
|
+
@JsonValue
|
19
|
+
public List<JdbcColumn> getColumns()
|
20
|
+
{
|
21
|
+
return columns;
|
22
|
+
}
|
23
|
+
|
24
|
+
public int getCount()
|
25
|
+
{
|
26
|
+
return columns.size();
|
27
|
+
}
|
28
|
+
|
29
|
+
public JdbcColumn getColumn(int i)
|
30
|
+
{
|
31
|
+
return columns.get(i);
|
32
|
+
}
|
33
|
+
|
34
|
+
public String getColumnName(int i)
|
35
|
+
{
|
36
|
+
return columns.get(i).getName();
|
37
|
+
}
|
38
|
+
|
39
|
+
public Optional<Integer> findColumn(String caseInsensitiveName)
|
40
|
+
{
|
41
|
+
// find by case sensitive first
|
42
|
+
for (int i = 0; i < columns.size(); i++) {
|
43
|
+
if (getColumn(i).getName().equals(caseInsensitiveName)) {
|
44
|
+
return Optional.of(i);
|
45
|
+
}
|
46
|
+
}
|
47
|
+
// find by case insensitive
|
48
|
+
for (int i = 0; i < columns.size(); i++) {
|
49
|
+
if (getColumn(i).getName().equalsIgnoreCase(caseInsensitiveName)) {
|
50
|
+
return Optional.of(i);
|
51
|
+
}
|
52
|
+
}
|
53
|
+
return Optional.absent();
|
54
|
+
}
|
55
|
+
}
|
@@ -0,0 +1,37 @@
|
|
1
|
+
package org.embulk.input.jdbc;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
5
|
+
|
6
|
+
import org.embulk.config.ConfigException;
|
7
|
+
|
8
|
+
public enum Ssl
|
9
|
+
{
|
10
|
+
ENABLE,
|
11
|
+
DISABLE,
|
12
|
+
VERIFY;
|
13
|
+
|
14
|
+
@JsonValue
|
15
|
+
@Override
|
16
|
+
public String toString()
|
17
|
+
{
|
18
|
+
return this.name().toLowerCase();
|
19
|
+
}
|
20
|
+
|
21
|
+
@JsonCreator
|
22
|
+
public static Ssl fromString(String value)
|
23
|
+
{
|
24
|
+
switch(value) {
|
25
|
+
case "enable":
|
26
|
+
case "true":
|
27
|
+
return ENABLE;
|
28
|
+
case "disable":
|
29
|
+
case "false":
|
30
|
+
return DISABLE;
|
31
|
+
case "verify":
|
32
|
+
return VERIFY;
|
33
|
+
default:
|
34
|
+
throw new ConfigException(String.format("Unknown SSL value '%s'. Supported values are enable, true, disable, false or verify.", value));
|
35
|
+
}
|
36
|
+
}
|
37
|
+
}
|
@@ -0,0 +1,54 @@
|
|
1
|
+
package org.embulk.input.jdbc;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
5
|
+
import com.fasterxml.jackson.databind.JsonMappingException;
|
6
|
+
import com.fasterxml.jackson.databind.node.NullNode;
|
7
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
8
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
9
|
+
|
10
|
+
public class ToString
|
11
|
+
{
|
12
|
+
private final String string;
|
13
|
+
|
14
|
+
public ToString(String string)
|
15
|
+
{
|
16
|
+
this.string = string;
|
17
|
+
}
|
18
|
+
|
19
|
+
@JsonCreator
|
20
|
+
ToString(Optional<JsonNode> option) throws JsonMappingException
|
21
|
+
{
|
22
|
+
JsonNode node = option.or(NullNode.getInstance());
|
23
|
+
if (node.isTextual()) {
|
24
|
+
this.string = node.textValue();
|
25
|
+
} else if (node.isValueNode()) {
|
26
|
+
this.string = node.toString();
|
27
|
+
} else {
|
28
|
+
throw new JsonMappingException(String.format("Arrays and objects are invalid: '%s'", node));
|
29
|
+
}
|
30
|
+
}
|
31
|
+
|
32
|
+
@Override
|
33
|
+
public boolean equals(Object obj)
|
34
|
+
{
|
35
|
+
if (!(obj instanceof ToString)) {
|
36
|
+
return false;
|
37
|
+
}
|
38
|
+
ToString o = (ToString) obj;
|
39
|
+
return string.equals(o.string);
|
40
|
+
}
|
41
|
+
|
42
|
+
@Override
|
43
|
+
public int hashCode()
|
44
|
+
{
|
45
|
+
return string.hashCode();
|
46
|
+
}
|
47
|
+
|
48
|
+
@JsonValue
|
49
|
+
@Override
|
50
|
+
public String toString()
|
51
|
+
{
|
52
|
+
return string;
|
53
|
+
}
|
54
|
+
}
|
@@ -0,0 +1,35 @@
|
|
1
|
+
package org.embulk.input.jdbc;
|
2
|
+
|
3
|
+
import java.util.Map;
|
4
|
+
import java.util.HashMap;
|
5
|
+
import java.util.Properties;
|
6
|
+
import com.google.common.base.Function;
|
7
|
+
import com.google.common.collect.Maps;
|
8
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
9
|
+
|
10
|
+
// TODO copied from embulk-output-jdbc. Move this class to embulk-core spi.unit.
|
11
|
+
public class ToStringMap
|
12
|
+
extends HashMap<String, String>
|
13
|
+
{
|
14
|
+
@JsonCreator
|
15
|
+
ToStringMap(Map<String, ToString> map)
|
16
|
+
{
|
17
|
+
super(Maps.transformValues(map, new Function<ToString, String>() {
|
18
|
+
public String apply(ToString value)
|
19
|
+
{
|
20
|
+
if (value == null) {
|
21
|
+
return "null";
|
22
|
+
} else {
|
23
|
+
return value.toString();
|
24
|
+
}
|
25
|
+
}
|
26
|
+
}));
|
27
|
+
}
|
28
|
+
|
29
|
+
public Properties toProperties()
|
30
|
+
{
|
31
|
+
Properties props = new Properties();
|
32
|
+
props.putAll(this);
|
33
|
+
return props;
|
34
|
+
}
|
35
|
+
}
|
@@ -0,0 +1,105 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import java.sql.ResultSet;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
7
|
+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
8
|
+
import org.embulk.spi.Column;
|
9
|
+
import org.embulk.spi.ColumnVisitor;
|
10
|
+
import org.embulk.spi.PageBuilder;
|
11
|
+
import org.embulk.spi.type.Type;
|
12
|
+
import org.embulk.spi.DataException;
|
13
|
+
import static java.util.Locale.ENGLISH;
|
14
|
+
|
15
|
+
public abstract class AbstractColumnGetter implements ColumnGetter, ColumnVisitor
|
16
|
+
{
|
17
|
+
protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
|
18
|
+
|
19
|
+
protected final PageBuilder to;
|
20
|
+
private final Type toType;
|
21
|
+
|
22
|
+
public AbstractColumnGetter(PageBuilder to, Type toType)
|
23
|
+
{
|
24
|
+
this.to = to;
|
25
|
+
this.toType = toType;
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public void getAndSet(ResultSet from, int fromIndex,
|
30
|
+
Column toColumn) throws SQLException
|
31
|
+
{
|
32
|
+
fetch(from, fromIndex);
|
33
|
+
if (from.wasNull()) {
|
34
|
+
to.setNull(toColumn);
|
35
|
+
} else {
|
36
|
+
toColumn.visit(this);
|
37
|
+
}
|
38
|
+
}
|
39
|
+
|
40
|
+
protected abstract void fetch(ResultSet from, int fromIndex) throws SQLException;
|
41
|
+
|
42
|
+
@Override
|
43
|
+
public void booleanColumn(Column column)
|
44
|
+
{
|
45
|
+
to.setNull(column);
|
46
|
+
}
|
47
|
+
|
48
|
+
@Override
|
49
|
+
public void longColumn(Column column)
|
50
|
+
{
|
51
|
+
to.setNull(column);
|
52
|
+
}
|
53
|
+
|
54
|
+
@Override
|
55
|
+
public void doubleColumn(Column column)
|
56
|
+
{
|
57
|
+
to.setNull(column);
|
58
|
+
}
|
59
|
+
|
60
|
+
@Override
|
61
|
+
public void stringColumn(Column column)
|
62
|
+
{
|
63
|
+
to.setNull(column);
|
64
|
+
}
|
65
|
+
|
66
|
+
@Override
|
67
|
+
public void jsonColumn(Column column)
|
68
|
+
{
|
69
|
+
to.setNull(column);
|
70
|
+
}
|
71
|
+
|
72
|
+
@Override
|
73
|
+
public void timestampColumn(Column column)
|
74
|
+
{
|
75
|
+
to.setNull(column);
|
76
|
+
}
|
77
|
+
|
78
|
+
@Override
|
79
|
+
public Type getToType()
|
80
|
+
{
|
81
|
+
if (toType == null) {
|
82
|
+
return getDefaultToType();
|
83
|
+
}
|
84
|
+
return toType;
|
85
|
+
}
|
86
|
+
|
87
|
+
protected abstract Type getDefaultToType();
|
88
|
+
|
89
|
+
@Override
|
90
|
+
public JsonNode encodeToJson()
|
91
|
+
{
|
92
|
+
throw new DataException(String.format(ENGLISH,
|
93
|
+
"Column type '%s' set at incremental_columns option is not supported",
|
94
|
+
getToType()));
|
95
|
+
}
|
96
|
+
|
97
|
+
@Override
|
98
|
+
public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
|
99
|
+
throws SQLException
|
100
|
+
{
|
101
|
+
throw new DataException(String.format(ENGLISH,
|
102
|
+
"Converting last_record value %s to column index %d is not supported",
|
103
|
+
fromValue.toString(), toIndex));
|
104
|
+
}
|
105
|
+
}
|
@@ -0,0 +1,45 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import java.sql.ResultSet;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
7
|
+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
8
|
+
import org.embulk.spi.Column;
|
9
|
+
import org.embulk.spi.ColumnVisitor;
|
10
|
+
import org.embulk.spi.PageBuilder;
|
11
|
+
import org.embulk.spi.type.Type;
|
12
|
+
import org.embulk.spi.DataException;
|
13
|
+
import static java.util.Locale.ENGLISH;
|
14
|
+
|
15
|
+
public abstract class AbstractIncrementalHandler implements ColumnGetter
|
16
|
+
{
|
17
|
+
protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
|
18
|
+
|
19
|
+
protected ColumnGetter next;
|
20
|
+
|
21
|
+
public AbstractIncrementalHandler(ColumnGetter next)
|
22
|
+
{
|
23
|
+
this.next = next;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public void getAndSet(ResultSet from, int fromIndex,
|
28
|
+
Column toColumn) throws SQLException
|
29
|
+
{
|
30
|
+
next.getAndSet(from, fromIndex, toColumn);
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public Type getToType()
|
35
|
+
{
|
36
|
+
return next.getToType();
|
37
|
+
}
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public abstract JsonNode encodeToJson();
|
41
|
+
|
42
|
+
@Override
|
43
|
+
public abstract void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
|
44
|
+
throws SQLException;
|
45
|
+
}
|
@@ -0,0 +1,38 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import org.embulk.spi.Column;
|
4
|
+
import org.embulk.spi.PageBuilder;
|
5
|
+
import org.embulk.spi.time.Timestamp;
|
6
|
+
import org.embulk.spi.time.TimestampFormatter;
|
7
|
+
import org.embulk.spi.type.Type;
|
8
|
+
|
9
|
+
public abstract class AbstractTimestampColumnGetter
|
10
|
+
extends AbstractColumnGetter
|
11
|
+
{
|
12
|
+
protected final TimestampFormatter timestampFormatter;
|
13
|
+
protected Timestamp value;
|
14
|
+
|
15
|
+
public AbstractTimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter)
|
16
|
+
{
|
17
|
+
super(to, toType);
|
18
|
+
|
19
|
+
this.timestampFormatter = timestampFormatter;
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
public void stringColumn(Column column)
|
24
|
+
{
|
25
|
+
to.setString(column, timestampFormatter.format(value));
|
26
|
+
}
|
27
|
+
|
28
|
+
@Override
|
29
|
+
public void jsonColumn(Column column) {
|
30
|
+
throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public void timestampColumn(Column column)
|
35
|
+
{
|
36
|
+
to.setTimestamp(column, value);
|
37
|
+
}
|
38
|
+
}
|
@@ -0,0 +1,59 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import java.math.BigDecimal;
|
4
|
+
import java.sql.ResultSet;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import org.embulk.spi.Column;
|
7
|
+
import org.embulk.spi.PageBuilder;
|
8
|
+
import org.embulk.spi.type.Type;
|
9
|
+
import org.embulk.spi.type.Types;
|
10
|
+
|
11
|
+
public class BigDecimalColumnGetter
|
12
|
+
extends AbstractColumnGetter
|
13
|
+
{
|
14
|
+
protected BigDecimal value;
|
15
|
+
|
16
|
+
public BigDecimalColumnGetter(PageBuilder to, Type toType)
|
17
|
+
{
|
18
|
+
super(to, toType);
|
19
|
+
}
|
20
|
+
|
21
|
+
@Override
|
22
|
+
protected void fetch(ResultSet from, int fromIndex) throws SQLException
|
23
|
+
{
|
24
|
+
value = from.getBigDecimal(fromIndex);
|
25
|
+
}
|
26
|
+
|
27
|
+
@Override
|
28
|
+
protected Type getDefaultToType()
|
29
|
+
{
|
30
|
+
return Types.DOUBLE;
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public void booleanColumn(Column column)
|
35
|
+
{
|
36
|
+
to.setBoolean(column, value.signum() > 0);
|
37
|
+
}
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public void longColumn(Column column)
|
41
|
+
{
|
42
|
+
to.setLong(column, value.longValue());
|
43
|
+
}
|
44
|
+
|
45
|
+
@Override
|
46
|
+
public void doubleColumn(Column column)
|
47
|
+
{
|
48
|
+
// rounded value could be Double.NEGATIVE_INFINITY or Double.POSITIVE_INFINITY.
|
49
|
+
double rounded = value.doubleValue();
|
50
|
+
to.setDouble(column, rounded);
|
51
|
+
}
|
52
|
+
|
53
|
+
@Override
|
54
|
+
public void stringColumn(Column column)
|
55
|
+
{
|
56
|
+
to.setString(column, value.toPlainString());
|
57
|
+
}
|
58
|
+
|
59
|
+
}
|