embulk-filter-column 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/CHANGELOG.md +10 -0
- data/README.md +6 -0
- data/build.gradle +8 -3
- data/config/checkstyle/checkstyle.xml +127 -0
- data/example/add_columns.yml +1 -0
- data/example/columns.yml +2 -0
- data/example/drop_columns.yml +1 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/filter/ColumnFilterPlugin.java +133 -83
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 572e638f806833f390196e60aa5791c0f23aff76
|
|
4
|
+
data.tar.gz: db719c3a30d31e86bea2f75afaa09849c92ec631
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6a4c06e967fc14763e386326166ae4b0ae9247fca8a158dc41e25a9bb430e26430d70fb2603abd29fedfb54f822a2fac152078628f62eb3fe816bb31a7882d45
|
|
7
|
+
data.tar.gz: edb08b9d1b7fc6e53a41d86ab82a61a7f848737bf37f6baa0ec6048c5475c5040db4f165fb5306a21f4fef6e69b7afdce4f78855381c1c5b48e5aaec4fa3b303
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
}
|
|
6
7
|
import com.github.jrubygradle.JRubyExec
|
|
7
8
|
repositories {
|
|
@@ -12,17 +13,21 @@ configurations {
|
|
|
12
13
|
provided
|
|
13
14
|
}
|
|
14
15
|
|
|
15
|
-
version = "0.
|
|
16
|
+
version = "0.4.0"
|
|
16
17
|
sourceCompatibility = 1.7
|
|
17
18
|
targetCompatibility = 1.7
|
|
18
19
|
|
|
19
20
|
dependencies {
|
|
20
|
-
compile "org.embulk:embulk-core:0.
|
|
21
|
-
provided "org.embulk:embulk-core:0.
|
|
21
|
+
compile "org.embulk:embulk-core:0.8.2"
|
|
22
|
+
provided "org.embulk:embulk-core:0.8.2"
|
|
22
23
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
|
23
24
|
testCompile "junit:junit:4.+"
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
checkstyle {
|
|
28
|
+
toolVersion = '6.7'
|
|
29
|
+
}
|
|
30
|
+
|
|
26
31
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
27
32
|
doFirst { file("classpath").deleteDir() }
|
|
28
33
|
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<module name="FileTabCharacter"/>
|
|
7
|
+
<module name="NewlineAtEndOfFile">
|
|
8
|
+
<property name="lineSeparator" value="lf"/>
|
|
9
|
+
</module>
|
|
10
|
+
<module name="RegexpMultiline">
|
|
11
|
+
<property name="format" value="\r"/>
|
|
12
|
+
<property name="message" value="Line contains carriage return"/>
|
|
13
|
+
</module>
|
|
14
|
+
<module name="RegexpMultiline">
|
|
15
|
+
<property name="format" value=" \n"/>
|
|
16
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
17
|
+
</module>
|
|
18
|
+
<module name="RegexpMultiline">
|
|
19
|
+
<property name="format" value="\{\n\n"/>
|
|
20
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
21
|
+
</module>
|
|
22
|
+
<module name="RegexpMultiline">
|
|
23
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
24
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
25
|
+
</module>
|
|
26
|
+
<module name="RegexpMultiline">
|
|
27
|
+
<property name="format" value="\n\n\n"/>
|
|
28
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
29
|
+
</module>
|
|
30
|
+
<module name="RegexpMultiline">
|
|
31
|
+
<property name="format" value="\n\n\Z"/>
|
|
32
|
+
<property name="message" value="Blank line before end of file"/>
|
|
33
|
+
</module>
|
|
34
|
+
<module name="RegexpMultiline">
|
|
35
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
36
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
37
|
+
</module>
|
|
38
|
+
|
|
39
|
+
<module name="TreeWalker">
|
|
40
|
+
<module name="EmptyBlock">
|
|
41
|
+
<property name="option" value="text"/>
|
|
42
|
+
<property name="tokens" value="
|
|
43
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
44
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="EmptyStatement"/>
|
|
47
|
+
<module name="EmptyForInitializerPad"/>
|
|
48
|
+
<module name="EmptyForIteratorPad">
|
|
49
|
+
<property name="option" value="space"/>
|
|
50
|
+
</module>
|
|
51
|
+
<module name="MethodParamPad">
|
|
52
|
+
<property name="allowLineBreaks" value="true"/>
|
|
53
|
+
<property name="option" value="nospace"/>
|
|
54
|
+
</module>
|
|
55
|
+
<module name="ParenPad"/>
|
|
56
|
+
<module name="TypecastParenPad"/>
|
|
57
|
+
<module name="NeedBraces"/>
|
|
58
|
+
<module name="LeftCurly">
|
|
59
|
+
<property name="option" value="nl"/>
|
|
60
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="LeftCurly">
|
|
63
|
+
<property name="option" value="eol"/>
|
|
64
|
+
<property name="tokens" value="
|
|
65
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
66
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
67
|
+
</module>
|
|
68
|
+
<module name="RightCurly">
|
|
69
|
+
<property name="option" value="alone"/>
|
|
70
|
+
</module>
|
|
71
|
+
<module name="GenericWhitespace"/>
|
|
72
|
+
<module name="WhitespaceAfter"/>
|
|
73
|
+
<module name="NoWhitespaceBefore"/>
|
|
74
|
+
|
|
75
|
+
<module name="UpperEll"/>
|
|
76
|
+
<module name="DefaultComesLast"/>
|
|
77
|
+
<module name="ArrayTypeStyle"/>
|
|
78
|
+
<module name="MultipleVariableDeclarations"/>
|
|
79
|
+
<module name="ModifierOrder"/>
|
|
80
|
+
<module name="OneStatementPerLine"/>
|
|
81
|
+
<module name="StringLiteralEquality"/>
|
|
82
|
+
<module name="MutableException"/>
|
|
83
|
+
<module name="EqualsHashCode"/>
|
|
84
|
+
<module name="InnerAssignment"/>
|
|
85
|
+
<module name="InterfaceIsType"/>
|
|
86
|
+
<module name="HideUtilityClassConstructor"/>
|
|
87
|
+
|
|
88
|
+
<module name="MemberName"/>
|
|
89
|
+
<module name="LocalVariableName"/>
|
|
90
|
+
<module name="LocalFinalVariableName"/>
|
|
91
|
+
<module name="TypeName"/>
|
|
92
|
+
<module name="PackageName"/>
|
|
93
|
+
<module name="ParameterName"/>
|
|
94
|
+
<module name="StaticVariableName"/>
|
|
95
|
+
<module name="ClassTypeParameterName">
|
|
96
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="MethodTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
|
|
102
|
+
<module name="AvoidStarImport"/>
|
|
103
|
+
<module name="RedundantImport"/>
|
|
104
|
+
<module name="UnusedImports"/>
|
|
105
|
+
<module name="ImportOrder">
|
|
106
|
+
<property name="groups" value="*,javax,java"/>
|
|
107
|
+
<property name="separated" value="true"/>
|
|
108
|
+
<property name="option" value="bottom"/>
|
|
109
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
110
|
+
</module>
|
|
111
|
+
|
|
112
|
+
<module name="WhitespaceAround">
|
|
113
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
114
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
115
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
116
|
+
<property name="tokens" value="
|
|
117
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
118
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
119
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
120
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
121
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
122
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
123
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
124
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
125
|
+
</module>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
data/example/add_columns.yml
CHANGED
data/example/columns.yml
CHANGED
|
@@ -20,6 +20,7 @@ in:
|
|
|
20
20
|
- {name: id, type: long}
|
|
21
21
|
- {name: name, type: string}
|
|
22
22
|
- {name: score, type: double}
|
|
23
|
+
- {name: json, type: json}
|
|
23
24
|
filters:
|
|
24
25
|
- type: column
|
|
25
26
|
columns:
|
|
@@ -28,5 +29,6 @@ filters:
|
|
|
28
29
|
- {name: foo, default: 1, type: long}
|
|
29
30
|
- {name: id}
|
|
30
31
|
- {name: copy_score, src: score}
|
|
32
|
+
- {name: json, default: "{\"foo\": \"FOO\"}"}
|
|
31
33
|
out:
|
|
32
34
|
type: stdout
|
data/example/drop_columns.yml
CHANGED
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -1,46 +1,44 @@
|
|
|
1
1
|
package org.embulk.filter;
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import
|
|
3
|
+
import com.google.common.base.Optional;
|
|
4
|
+
import com.google.common.base.Throwables;
|
|
5
5
|
import com.google.common.collect.ImmutableList;
|
|
6
|
-
import org.slf4j.Logger;
|
|
7
6
|
|
|
8
7
|
import org.embulk.config.Config;
|
|
9
8
|
import org.embulk.config.ConfigDefault;
|
|
10
|
-
import org.embulk.config.ConfigDiff;
|
|
11
|
-
import org.embulk.config.ConfigSource;
|
|
12
9
|
import org.embulk.config.ConfigException;
|
|
10
|
+
import org.embulk.config.ConfigSource;
|
|
13
11
|
import org.embulk.config.Task;
|
|
14
12
|
import org.embulk.config.TaskSource;
|
|
15
13
|
|
|
16
|
-
import org.embulk.spi.
|
|
17
|
-
import org.embulk.spi.
|
|
18
|
-
import org.embulk.spi.type.LongType;
|
|
19
|
-
import org.embulk.spi.type.DoubleType;
|
|
20
|
-
import org.embulk.spi.type.StringType;
|
|
21
|
-
import org.embulk.spi.type.TimestampType;
|
|
22
|
-
|
|
23
|
-
import org.embulk.spi.FilterPlugin;
|
|
14
|
+
import org.embulk.spi.Column;
|
|
15
|
+
import org.embulk.spi.ColumnVisitor;
|
|
24
16
|
import org.embulk.spi.Exec;
|
|
17
|
+
import org.embulk.spi.FilterPlugin;
|
|
25
18
|
import org.embulk.spi.Page;
|
|
26
19
|
import org.embulk.spi.PageBuilder;
|
|
27
20
|
import org.embulk.spi.PageOutput;
|
|
28
21
|
import org.embulk.spi.PageReader;
|
|
29
22
|
import org.embulk.spi.Schema;
|
|
30
|
-
import org.embulk.spi.
|
|
31
|
-
import org.embulk.spi.
|
|
32
|
-
import org.embulk.spi.ColumnVisitor;
|
|
33
|
-
|
|
34
|
-
import org.joda.time.DateTimeZone;
|
|
23
|
+
import org.embulk.spi.SchemaConfigException;
|
|
24
|
+
import org.embulk.spi.json.JsonParser;
|
|
35
25
|
import org.embulk.spi.time.Timestamp;
|
|
36
|
-
import org.embulk.spi.time.TimestampParser;
|
|
37
26
|
import org.embulk.spi.time.TimestampParseException;
|
|
38
|
-
import
|
|
27
|
+
import org.embulk.spi.time.TimestampParser;
|
|
28
|
+
import org.embulk.spi.type.BooleanType;
|
|
29
|
+
import org.embulk.spi.type.DoubleType;
|
|
30
|
+
import org.embulk.spi.type.LongType;
|
|
31
|
+
import org.embulk.spi.type.StringType;
|
|
32
|
+
import org.embulk.spi.type.JsonType;
|
|
33
|
+
import org.embulk.spi.type.TimestampType;
|
|
34
|
+
import org.embulk.spi.type.Type;
|
|
39
35
|
|
|
40
|
-
import org.
|
|
41
|
-
import org.
|
|
42
|
-
import
|
|
43
|
-
|
|
36
|
+
import org.joda.time.DateTimeZone;
|
|
37
|
+
import org.msgpack.value.Value;
|
|
38
|
+
import org.slf4j.Logger;
|
|
39
|
+
|
|
40
|
+
import java.util.HashMap;
|
|
41
|
+
import java.util.List;
|
|
44
42
|
|
|
45
43
|
public class ColumnFilterPlugin implements FilterPlugin
|
|
46
44
|
{
|
|
@@ -116,7 +114,7 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
116
114
|
ImmutableList.Builder<Column> builder = ImmutableList.builder();
|
|
117
115
|
int i = 0;
|
|
118
116
|
if (dropColumns.size() > 0) {
|
|
119
|
-
for (Column inputColumn: inputSchema.getColumns()) {
|
|
117
|
+
for (Column inputColumn : inputSchema.getColumns()) {
|
|
120
118
|
String name = inputColumn.getName();
|
|
121
119
|
boolean matched = false;
|
|
122
120
|
for (ColumnConfig dropColumn : dropColumns) {
|
|
@@ -130,7 +128,8 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
130
128
|
builder.add(outputColumn);
|
|
131
129
|
}
|
|
132
130
|
}
|
|
133
|
-
}
|
|
131
|
+
}
|
|
132
|
+
else if (columns.size() > 0) {
|
|
134
133
|
for (ColumnConfig column : columns) {
|
|
135
134
|
String name = column.getName();
|
|
136
135
|
Optional<Type> type = column.getType();
|
|
@@ -151,8 +150,9 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
151
150
|
throw new SchemaConfigException(String.format("columns: Column src '%s' is not found in inputSchema. Column '%s' does not have \"type\" and \"default\"", srcName, name));
|
|
152
151
|
}
|
|
153
152
|
}
|
|
154
|
-
}
|
|
155
|
-
|
|
153
|
+
}
|
|
154
|
+
else {
|
|
155
|
+
for (Column inputColumn : inputSchema.getColumns()) {
|
|
156
156
|
Column outputColumn = new Column(i++, inputColumn.getName(), inputColumn.getType());
|
|
157
157
|
builder.add(outputColumn);
|
|
158
158
|
}
|
|
@@ -191,9 +191,10 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
191
191
|
control.run(task.dump(), outputSchema);
|
|
192
192
|
}
|
|
193
193
|
|
|
194
|
-
private Column getColumn(String name, Schema schema)
|
|
194
|
+
private Column getColumn(String name, Schema schema)
|
|
195
|
+
{
|
|
195
196
|
// hash should be faster, though
|
|
196
|
-
for (Column column: schema.getColumns()) {
|
|
197
|
+
for (Column column : schema.getColumns()) {
|
|
197
198
|
if (column.getName().equals(name)) {
|
|
198
199
|
return column;
|
|
199
200
|
}
|
|
@@ -201,22 +202,24 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
201
202
|
return null;
|
|
202
203
|
}
|
|
203
204
|
|
|
204
|
-
private String getSrc(String name, List<ColumnConfig> columnConfigs)
|
|
205
|
+
private String getSrc(String name, List<ColumnConfig> columnConfigs)
|
|
206
|
+
{
|
|
205
207
|
for (ColumnConfig columnConfig : columnConfigs) {
|
|
206
208
|
if (columnConfig.getName().equals(name) &&
|
|
207
209
|
columnConfig.getSrc().isPresent()) {
|
|
208
|
-
return (String)columnConfig.getSrc().get();
|
|
210
|
+
return (String) columnConfig.getSrc().get();
|
|
209
211
|
}
|
|
210
212
|
}
|
|
211
213
|
return null;
|
|
212
214
|
}
|
|
213
215
|
|
|
214
|
-
private Object getDefault(String name, Type type, List<ColumnConfig> columnConfigs, PluginTask task)
|
|
216
|
+
private Object getDefault(String name, Type type, List<ColumnConfig> columnConfigs, PluginTask task)
|
|
217
|
+
{
|
|
215
218
|
for (ColumnConfig columnConfig : columnConfigs) {
|
|
216
219
|
if (columnConfig.getName().equals(name)) {
|
|
217
220
|
if (type instanceof BooleanType) {
|
|
218
221
|
if (columnConfig.getDefault().isPresent()) {
|
|
219
|
-
return (Boolean)columnConfig.getDefault().get();
|
|
222
|
+
return (Boolean) columnConfig.getDefault().get();
|
|
220
223
|
}
|
|
221
224
|
}
|
|
222
225
|
else if (type instanceof LongType) {
|
|
@@ -231,29 +234,38 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
231
234
|
}
|
|
232
235
|
else if (type instanceof StringType) {
|
|
233
236
|
if (columnConfig.getDefault().isPresent()) {
|
|
234
|
-
return (String)columnConfig.getDefault().get();
|
|
237
|
+
return (String) columnConfig.getDefault().get();
|
|
235
238
|
}
|
|
236
239
|
}
|
|
240
|
+
else if (type instanceof JsonType) {
|
|
241
|
+
if (columnConfig.getDefault().isPresent()) {
|
|
242
|
+
JsonParser parser = new JsonParser();
|
|
243
|
+
return parser.parse((String) columnConfig.getDefault().get());
|
|
244
|
+
}
|
|
245
|
+
}
|
|
237
246
|
else if (type instanceof TimestampType) {
|
|
238
247
|
if (columnConfig.getDefault().isPresent()) {
|
|
239
|
-
String time = (String)columnConfig.getDefault().get();
|
|
248
|
+
String time = (String) columnConfig.getDefault().get();
|
|
240
249
|
String format = null;
|
|
241
250
|
if (columnConfig.getFormat().isPresent()) {
|
|
242
251
|
format = columnConfig.getFormat().get();
|
|
243
|
-
}
|
|
252
|
+
}
|
|
253
|
+
else {
|
|
244
254
|
format = task.getDefaultTimestampFormat();
|
|
245
255
|
}
|
|
246
256
|
DateTimeZone timezone = null;
|
|
247
257
|
if (columnConfig.getTimeZone().isPresent()) {
|
|
248
258
|
timezone = columnConfig.getTimeZone().get();
|
|
249
|
-
}
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
250
261
|
timezone = task.getDefaultTimeZone();
|
|
251
262
|
}
|
|
252
263
|
TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
|
|
253
264
|
try {
|
|
254
|
-
Timestamp
|
|
255
|
-
return
|
|
256
|
-
}
|
|
265
|
+
Timestamp defaultValue = parser.parse(time);
|
|
266
|
+
return defaultValue;
|
|
267
|
+
}
|
|
268
|
+
catch (TimestampParseException ex) {
|
|
257
269
|
throw Throwables.propagate(ex);
|
|
258
270
|
}
|
|
259
271
|
}
|
|
@@ -272,7 +284,7 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
272
284
|
|
|
273
285
|
// Map outputColumn => inputColumn
|
|
274
286
|
final HashMap<Column, Column> outputInputColumnMap = new HashMap<Column, Column>();
|
|
275
|
-
for (Column outputColumn: outputSchema.getColumns()) {
|
|
287
|
+
for (Column outputColumn : outputSchema.getColumns()) {
|
|
276
288
|
String name = outputColumn.getName();
|
|
277
289
|
String srcName = getSrc(name, task.getColumns());
|
|
278
290
|
if (srcName == null) {
|
|
@@ -287,16 +299,16 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
287
299
|
|
|
288
300
|
// Map outputColumn => default value if present
|
|
289
301
|
final HashMap<Column, Object> outputDefaultMap = new HashMap<Column, Object>();
|
|
290
|
-
for (Column outputColumn: outputSchema.getColumns()) {
|
|
302
|
+
for (Column outputColumn : outputSchema.getColumns()) {
|
|
291
303
|
String name = outputColumn.getName();
|
|
292
304
|
Type type = outputColumn.getType();
|
|
293
305
|
|
|
294
|
-
Object
|
|
295
|
-
if (
|
|
296
|
-
|
|
306
|
+
Object defaultValue = getDefault(name, type, task.getColumns(), task);
|
|
307
|
+
if (defaultValue == null) {
|
|
308
|
+
defaultValue = getDefault(name, type, task.getAddColumns(), task);
|
|
297
309
|
}
|
|
298
|
-
if (
|
|
299
|
-
outputDefaultMap.put(outputColumn,
|
|
310
|
+
if (defaultValue != null) {
|
|
311
|
+
outputDefaultMap.put(outputColumn, defaultValue);
|
|
300
312
|
}
|
|
301
313
|
}
|
|
302
314
|
|
|
@@ -306,17 +318,20 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
306
318
|
private ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
|
|
307
319
|
|
|
308
320
|
@Override
|
|
309
|
-
public void finish()
|
|
321
|
+
public void finish()
|
|
322
|
+
{
|
|
310
323
|
pageBuilder.finish();
|
|
311
324
|
}
|
|
312
325
|
|
|
313
326
|
@Override
|
|
314
|
-
public void close()
|
|
327
|
+
public void close()
|
|
328
|
+
{
|
|
315
329
|
pageBuilder.close();
|
|
316
330
|
}
|
|
317
331
|
|
|
318
332
|
@Override
|
|
319
|
-
public void add(Page page)
|
|
333
|
+
public void add(Page page)
|
|
334
|
+
{
|
|
320
335
|
pageReader.setPage(page);
|
|
321
336
|
|
|
322
337
|
while (pageReader.nextRecord()) {
|
|
@@ -325,84 +340,119 @@ public class ColumnFilterPlugin implements FilterPlugin
|
|
|
325
340
|
}
|
|
326
341
|
}
|
|
327
342
|
|
|
328
|
-
class ColumnVisitorImpl implements ColumnVisitor
|
|
343
|
+
class ColumnVisitorImpl implements ColumnVisitor
|
|
344
|
+
{
|
|
329
345
|
private final PageBuilder pageBuilder;
|
|
330
346
|
|
|
331
|
-
ColumnVisitorImpl(PageBuilder pageBuilder)
|
|
347
|
+
ColumnVisitorImpl(PageBuilder pageBuilder)
|
|
348
|
+
{
|
|
332
349
|
this.pageBuilder = pageBuilder;
|
|
333
350
|
}
|
|
334
351
|
|
|
335
352
|
@Override
|
|
336
|
-
public void booleanColumn(Column outputColumn)
|
|
353
|
+
public void booleanColumn(Column outputColumn)
|
|
354
|
+
{
|
|
337
355
|
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
338
356
|
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
339
|
-
Boolean
|
|
340
|
-
if (
|
|
341
|
-
pageBuilder.setBoolean(outputColumn,
|
|
342
|
-
}
|
|
357
|
+
Boolean defaultValue = (Boolean) outputDefaultMap.get(outputColumn);
|
|
358
|
+
if (defaultValue != null) {
|
|
359
|
+
pageBuilder.setBoolean(outputColumn, defaultValue.booleanValue());
|
|
360
|
+
}
|
|
361
|
+
else {
|
|
343
362
|
pageBuilder.setNull(outputColumn);
|
|
344
363
|
}
|
|
345
|
-
}
|
|
364
|
+
}
|
|
365
|
+
else {
|
|
346
366
|
pageBuilder.setBoolean(outputColumn, pageReader.getBoolean(inputColumn));
|
|
347
367
|
}
|
|
348
368
|
}
|
|
349
369
|
|
|
350
370
|
@Override
|
|
351
|
-
public void longColumn(Column outputColumn)
|
|
371
|
+
public void longColumn(Column outputColumn)
|
|
372
|
+
{
|
|
352
373
|
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
353
374
|
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
354
|
-
Long
|
|
355
|
-
if (
|
|
356
|
-
pageBuilder.setLong(outputColumn,
|
|
357
|
-
}
|
|
375
|
+
Long defaultValue = (Long) outputDefaultMap.get(outputColumn);
|
|
376
|
+
if (defaultValue != null) {
|
|
377
|
+
pageBuilder.setLong(outputColumn, defaultValue.longValue());
|
|
378
|
+
}
|
|
379
|
+
else {
|
|
358
380
|
pageBuilder.setNull(outputColumn);
|
|
359
381
|
}
|
|
360
|
-
}
|
|
382
|
+
}
|
|
383
|
+
else {
|
|
361
384
|
pageBuilder.setLong(outputColumn, pageReader.getLong(inputColumn));
|
|
362
385
|
}
|
|
363
386
|
}
|
|
364
387
|
|
|
365
388
|
@Override
|
|
366
|
-
public void doubleColumn(Column outputColumn)
|
|
389
|
+
public void doubleColumn(Column outputColumn)
|
|
390
|
+
{
|
|
367
391
|
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
368
392
|
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
369
|
-
Double
|
|
370
|
-
if (
|
|
371
|
-
pageBuilder.setDouble(outputColumn,
|
|
372
|
-
}
|
|
393
|
+
Double defaultValue = (Double) outputDefaultMap.get(outputColumn);
|
|
394
|
+
if (defaultValue != null) {
|
|
395
|
+
pageBuilder.setDouble(outputColumn, defaultValue.doubleValue());
|
|
396
|
+
}
|
|
397
|
+
else {
|
|
373
398
|
pageBuilder.setNull(outputColumn);
|
|
374
399
|
}
|
|
375
|
-
}
|
|
400
|
+
}
|
|
401
|
+
else {
|
|
376
402
|
pageBuilder.setDouble(outputColumn, pageReader.getDouble(inputColumn));
|
|
377
403
|
}
|
|
378
404
|
}
|
|
379
405
|
|
|
380
406
|
@Override
|
|
381
|
-
public void stringColumn(Column outputColumn)
|
|
407
|
+
public void stringColumn(Column outputColumn)
|
|
408
|
+
{
|
|
382
409
|
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
383
410
|
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
384
|
-
String
|
|
385
|
-
if (
|
|
386
|
-
pageBuilder.setString(outputColumn,
|
|
387
|
-
}
|
|
411
|
+
String defaultValue = (String) outputDefaultMap.get(outputColumn);
|
|
412
|
+
if (defaultValue != null) {
|
|
413
|
+
pageBuilder.setString(outputColumn, defaultValue);
|
|
414
|
+
}
|
|
415
|
+
else {
|
|
388
416
|
pageBuilder.setNull(outputColumn);
|
|
389
417
|
}
|
|
390
|
-
}
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
391
420
|
pageBuilder.setString(outputColumn, pageReader.getString(inputColumn));
|
|
392
421
|
}
|
|
393
422
|
}
|
|
394
423
|
|
|
395
424
|
@Override
|
|
396
|
-
public void
|
|
425
|
+
public void jsonColumn(Column outputColumn)
|
|
426
|
+
{
|
|
397
427
|
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
398
428
|
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
399
|
-
|
|
400
|
-
if (
|
|
401
|
-
pageBuilder.
|
|
402
|
-
}
|
|
429
|
+
Value defaultValue = (Value) outputDefaultMap.get(outputColumn);
|
|
430
|
+
if (defaultValue != null) {
|
|
431
|
+
pageBuilder.setJson(outputColumn, defaultValue);
|
|
432
|
+
}
|
|
433
|
+
else {
|
|
403
434
|
pageBuilder.setNull(outputColumn);
|
|
404
435
|
}
|
|
405
|
-
}
|
|
436
|
+
}
|
|
437
|
+
else {
|
|
438
|
+
pageBuilder.setJson(outputColumn, pageReader.getJson(inputColumn));
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
@Override
|
|
443
|
+
public void timestampColumn(Column outputColumn)
|
|
444
|
+
{
|
|
445
|
+
Column inputColumn = outputInputColumnMap.get(outputColumn);
|
|
446
|
+
if (inputColumn == null || pageReader.isNull(inputColumn)) {
|
|
447
|
+
Timestamp defaultValue = (Timestamp) outputDefaultMap.get(outputColumn);
|
|
448
|
+
if (defaultValue != null) {
|
|
449
|
+
pageBuilder.setTimestamp(outputColumn, defaultValue);
|
|
450
|
+
}
|
|
451
|
+
else {
|
|
452
|
+
pageBuilder.setNull(outputColumn);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
else {
|
|
406
456
|
pageBuilder.setTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
|
|
407
457
|
}
|
|
408
458
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-column
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-02-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -51,9 +51,11 @@ files:
|
|
|
51
51
|
- LICENSE.txt
|
|
52
52
|
- README.md
|
|
53
53
|
- build.gradle
|
|
54
|
+
- config/checkstyle/checkstyle.xml
|
|
54
55
|
- example/add_columns.yml
|
|
55
56
|
- example/columns.yml
|
|
56
57
|
- example/drop_columns.yml
|
|
58
|
+
- example/example.csv
|
|
57
59
|
- gradle/wrapper/gradle-wrapper.jar
|
|
58
60
|
- gradle/wrapper/gradle-wrapper.properties
|
|
59
61
|
- gradlew
|
|
@@ -61,7 +63,7 @@ files:
|
|
|
61
63
|
- lib/embulk/filter/column.rb
|
|
62
64
|
- src/main/java/org/embulk/filter/ColumnFilterPlugin.java
|
|
63
65
|
- src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
|
|
64
|
-
- classpath/embulk-filter-column-0.
|
|
66
|
+
- classpath/embulk-filter-column-0.4.0.jar
|
|
65
67
|
homepage: https://github.com/sonots/embulk-filter-column
|
|
66
68
|
licenses:
|
|
67
69
|
- MIT
|