embulk-filter-expand_json 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -0
- data/build.gradle +1 -1
- data/classpath/{embulk-filter-expand_json-0.1.1.jar → embulk-filter-expand_json-0.1.2.jar} +0 -0
- data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java +26 -0
- data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java +45 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4eb62c73f09245613f9c6950d72f840434d6cc16
|
4
|
+
data.tar.gz: b46a7998eeab53f56fe524a6746d9c266ee45a53
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dd2395c19f4113da5d4c98624ebc8d4c3a2b268bb68a9e0091cc4955ea5fb1696665c1541d9bcf183ba6b8e9b2f76291c340face70e17c6ffbfdd949024beb31
|
7
|
+
data.tar.gz: d1005e506f9f61dcb06e08da8c396983ed04797a25d8d5add72828a14fe19ee1b77ec7c391dfd0518d6f1717a1e1c436769ddedc413ad068e1b343cee2102124
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
0.1.2 (2016-05-31)
|
2
|
+
==================
|
3
|
+
- [Enhancement] Add a validation to check column duplication
|
4
|
+
- https://github.com/civitaspo/embulk-filter-expand_json/pull/21
|
5
|
+
|
1
6
|
0.1.1 (2016-05-02)
|
2
7
|
==================
|
3
8
|
- [New Feature] Add keep_expanding_json_column option
|
data/build.gradle
CHANGED
Binary file
|
@@ -17,6 +17,7 @@ import org.embulk.spi.time.TimestampParser;
|
|
17
17
|
import org.embulk.spi.type.Types;
|
18
18
|
import org.slf4j.Logger;
|
19
19
|
|
20
|
+
import java.util.ArrayList;
|
20
21
|
import java.util.List;
|
21
22
|
|
22
23
|
public class ExpandJsonFilterPlugin
|
@@ -64,8 +65,10 @@ public class ExpandJsonFilterPlugin
|
|
64
65
|
throw new ConfigException(String.format("A column specified as json_column_name option must be string or json type: %s",
|
65
66
|
new Object[] {jsonColumn.toString()}));
|
66
67
|
}
|
68
|
+
validateExpandedColumns(task.getExpandedColumns());
|
67
69
|
|
68
70
|
Schema outputSchema = buildOutputSchema(task, inputSchema);
|
71
|
+
validateOutputSchema(outputSchema);
|
69
72
|
control.run(task.dump(), outputSchema);
|
70
73
|
}
|
71
74
|
|
@@ -124,4 +127,27 @@ public class ExpandJsonFilterPlugin
|
|
124
127
|
return new Schema(builder.build());
|
125
128
|
}
|
126
129
|
|
130
|
+
private void validateExpandedColumns(List<ColumnConfig> expandedColumns)
|
131
|
+
{
|
132
|
+
List<String> columnList = new ArrayList<>();
|
133
|
+
for (ColumnConfig columnConfig: expandedColumns) {
|
134
|
+
String columnName = columnConfig.getName();
|
135
|
+
if (columnList.contains(columnName)) {
|
136
|
+
throw new ConfigException(String.format("Column config for '%s' is duplicated at 'expanded_columns' option", columnName));
|
137
|
+
}
|
138
|
+
columnList.add(columnName);
|
139
|
+
}
|
140
|
+
}
|
141
|
+
|
142
|
+
private void validateOutputSchema(Schema outputSchema)
|
143
|
+
{
|
144
|
+
List<String> columnList = new ArrayList<>();
|
145
|
+
for (Column column: outputSchema.getColumns()) {
|
146
|
+
String columnName = column.getName();
|
147
|
+
if (columnList.contains(columnName)) {
|
148
|
+
throw new ConfigException(String.format("Output column '%s' is duplicated. Please check 'expanded_columns' option and Input plugin's settings.", columnName));
|
149
|
+
}
|
150
|
+
columnList.add(columnName);
|
151
|
+
}
|
152
|
+
}
|
127
153
|
}
|
@@ -164,6 +164,51 @@ public class TestExpandJsonFilterPlugin
|
|
164
164
|
config.loadConfig(PluginTask.class);
|
165
165
|
}
|
166
166
|
|
167
|
+
@Test
|
168
|
+
public void testThrowExceptionDuplicatedExpandedColumns()
|
169
|
+
{
|
170
|
+
String configYaml = "" +
|
171
|
+
"type: expand_json\n" +
|
172
|
+
"json_column_name: _c0\n" +
|
173
|
+
"expanded_columns:\n" +
|
174
|
+
" - {name: _c1, type: string}\n" +
|
175
|
+
" - {name: _c1, type: string}";
|
176
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
177
|
+
schema = schema("_c0", STRING, "_c1", STRING);
|
178
|
+
|
179
|
+
exception.expect(ConfigException.class);
|
180
|
+
exception.expectMessage("Column config for '_c1' is duplicated at 'expanded_columns' option");
|
181
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control() {
|
182
|
+
@Override
|
183
|
+
public void run(TaskSource taskSource, Schema schema)
|
184
|
+
{
|
185
|
+
// do nothing
|
186
|
+
}
|
187
|
+
});
|
188
|
+
}
|
189
|
+
|
190
|
+
@Test
|
191
|
+
public void testThrowExceptionDuplicatedOutputColumns()
|
192
|
+
{
|
193
|
+
String configYaml = "" +
|
194
|
+
"type: expand_json\n" +
|
195
|
+
"json_column_name: _c0\n" +
|
196
|
+
"expanded_columns:\n" +
|
197
|
+
" - {name: _c1, type: string}";
|
198
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
199
|
+
schema = schema("_c0", STRING, "_c0", STRING, "_c1", STRING);
|
200
|
+
|
201
|
+
exception.expect(ConfigException.class);
|
202
|
+
exception.expectMessage("Output column '_c1' is duplicated. Please check 'expanded_columns' option and Input plugin's settings.");
|
203
|
+
expandJsonFilterPlugin.transaction(config, schema, new Control() {
|
204
|
+
@Override
|
205
|
+
public void run(TaskSource taskSource, Schema schema)
|
206
|
+
{
|
207
|
+
// do nothing
|
208
|
+
}
|
209
|
+
});
|
210
|
+
}
|
211
|
+
|
167
212
|
@Test
|
168
213
|
public void testDefaultValue()
|
169
214
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-expand_json
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-05-
|
11
|
+
date: 2016-05-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -65,7 +65,7 @@ files:
|
|
65
65
|
- src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
|
66
66
|
- classpath/accessors-smart-1.1.jar
|
67
67
|
- classpath/asm-5.0.3.jar
|
68
|
-
- classpath/embulk-filter-expand_json-0.1.
|
68
|
+
- classpath/embulk-filter-expand_json-0.1.2.jar
|
69
69
|
- classpath/json-path-2.2.0.jar
|
70
70
|
- classpath/json-smart-2.2.1.jar
|
71
71
|
- classpath/slf4j-api-1.7.16.jar
|