embulk-filter-column 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java +2 -2
- data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java +294 -0
- data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java +47 -69
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 827fdada3d29078b86a59877f66c9e2ebf1f8793
|
|
4
|
+
data.tar.gz: 268ce04ccace8ef31c24aa989bc0ebebc99f3015
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1f59e24aac1088d4d420a1fdb2f02614049bbeaa7afe262689f09860d80d6139a38eabc26cbcacab770a0ee1ecfcd4f9d98fcbe73a64cc344aba3463c50322b4
|
|
7
|
+
data.tar.gz: 81eaf28c207fcdd325cfc62de40acea4dfc1b79555f764d936e420a9bbb185f4a0b908844c875f3c07e441c02bccef66f1a3d8b26bb5310c8b384e0c3d81307f
|
data/CHANGELOG.md
CHANGED
data/build.gradle
CHANGED
|
@@ -244,13 +244,13 @@ public class ColumnVisitorImpl implements ColumnVisitor
|
|
|
244
244
|
pageBuilder.setNull(outputColumn);
|
|
245
245
|
}
|
|
246
246
|
else {
|
|
247
|
-
String jsonPath = new StringBuilder("$.").append(
|
|
247
|
+
String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
|
|
248
248
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, defaultValue));
|
|
249
249
|
}
|
|
250
250
|
}
|
|
251
251
|
else {
|
|
252
252
|
Value value = pageReader.getJson(inputColumn);
|
|
253
|
-
String jsonPath = new StringBuilder("$.").append(
|
|
253
|
+
String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
|
|
254
254
|
pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, value));
|
|
255
255
|
}
|
|
256
256
|
}
|
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
package org.embulk.filter.column;
|
|
2
|
+
|
|
3
|
+
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
|
4
|
+
|
|
5
|
+
import org.embulk.EmbulkTestRuntime;
|
|
6
|
+
import org.embulk.config.ConfigLoader;
|
|
7
|
+
import org.embulk.config.ConfigSource;
|
|
8
|
+
import org.embulk.spi.Exec;
|
|
9
|
+
import org.embulk.spi.Page;
|
|
10
|
+
import org.embulk.spi.PageBuilder;
|
|
11
|
+
import org.embulk.spi.PageReader;
|
|
12
|
+
import org.embulk.spi.PageTestUtils;
|
|
13
|
+
import org.embulk.spi.Schema;
|
|
14
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
15
|
+
import org.embulk.spi.time.Timestamp;
|
|
16
|
+
import org.embulk.spi.util.Pages;
|
|
17
|
+
import org.junit.Before;
|
|
18
|
+
import org.junit.Rule;
|
|
19
|
+
import org.junit.Test;
|
|
20
|
+
import org.msgpack.value.ValueFactory;
|
|
21
|
+
|
|
22
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
23
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
|
24
|
+
import static org.embulk.spi.type.Types.JSON;
|
|
25
|
+
import static org.embulk.spi.type.Types.LONG;
|
|
26
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
27
|
+
import static org.junit.Assert.assertEquals;
|
|
28
|
+
|
|
29
|
+
import static org.embulk.spi.type.Types.STRING;
|
|
30
|
+
|
|
31
|
+
import java.util.List;
|
|
32
|
+
|
|
33
|
+
public class TestColumnVisitorImpl {
|
|
34
|
+
@Rule
|
|
35
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
36
|
+
|
|
37
|
+
@Before
|
|
38
|
+
public void createReasource()
|
|
39
|
+
{
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
private ConfigSource config()
|
|
43
|
+
{
|
|
44
|
+
return runtime.getExec().newConfigSource();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
private PluginTask taskFromYamlString(String... lines)
|
|
48
|
+
{
|
|
49
|
+
StringBuilder builder = new StringBuilder();
|
|
50
|
+
for (String line : lines) {
|
|
51
|
+
builder.append(line).append("\n");
|
|
52
|
+
}
|
|
53
|
+
String yamlString = builder.toString();
|
|
54
|
+
|
|
55
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
|
56
|
+
ConfigSource config = loader.fromYamlString(yamlString);
|
|
57
|
+
return config.loadConfig(PluginTask.class);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private List<Object[]> filter(PluginTask task, Schema inputSchema, Object ... objects)
|
|
61
|
+
{
|
|
62
|
+
MockPageOutput output = new MockPageOutput();
|
|
63
|
+
Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
|
|
64
|
+
PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
|
|
65
|
+
PageReader pageReader = new PageReader(inputSchema);
|
|
66
|
+
ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
|
|
67
|
+
|
|
68
|
+
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
|
|
69
|
+
for (Page page : pages) {
|
|
70
|
+
pageReader.setPage(page);
|
|
71
|
+
|
|
72
|
+
while (pageReader.nextRecord()) {
|
|
73
|
+
outputSchema.visitColumns(visitor);
|
|
74
|
+
pageBuilder.addRecord();
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
pageBuilder.finish();
|
|
78
|
+
pageBuilder.close();
|
|
79
|
+
return Pages.toObjects(outputSchema, output.pages);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
@Test
|
|
83
|
+
public void visit_Columns_WithDrop()
|
|
84
|
+
{
|
|
85
|
+
PluginTask task = taskFromYamlString(
|
|
86
|
+
"type: column",
|
|
87
|
+
"columns:",
|
|
88
|
+
" - {name: timestamp}",
|
|
89
|
+
" - {name: string}",
|
|
90
|
+
" - {name: boolean}",
|
|
91
|
+
" - {name: long}",
|
|
92
|
+
" - {name: double}",
|
|
93
|
+
" - {name: json}");
|
|
94
|
+
Schema inputSchema = Schema.builder()
|
|
95
|
+
.add("timestamp", TIMESTAMP)
|
|
96
|
+
.add("string", STRING)
|
|
97
|
+
.add("boolean", BOOLEAN)
|
|
98
|
+
.add("long", LONG)
|
|
99
|
+
.add("double", DOUBLE)
|
|
100
|
+
.add("json", JSON)
|
|
101
|
+
.add("remove_me", STRING)
|
|
102
|
+
.build();
|
|
103
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
104
|
+
Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
|
|
105
|
+
Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me");
|
|
106
|
+
|
|
107
|
+
assertEquals(2, records.size());
|
|
108
|
+
|
|
109
|
+
Object[] record;
|
|
110
|
+
{
|
|
111
|
+
record = records.get(0);
|
|
112
|
+
assertEquals(6, record.length);
|
|
113
|
+
assertEquals(Timestamp.ofEpochSecond(0), record[0]);
|
|
114
|
+
assertEquals("string", record[1]);
|
|
115
|
+
assertEquals(new Boolean(true), record[2]);
|
|
116
|
+
assertEquals(new Long(0), record[3]);
|
|
117
|
+
assertEquals(new Double(0.5), record[4]);
|
|
118
|
+
assertEquals(ValueFactory.newString("json"), record[5]);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
@Test
|
|
123
|
+
public void visit_Columns_WithDefault()
|
|
124
|
+
{
|
|
125
|
+
PluginTask task = taskFromYamlString(
|
|
126
|
+
"type: column",
|
|
127
|
+
"columns:",
|
|
128
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
|
129
|
+
" - {name: string, type: string, default: string}",
|
|
130
|
+
" - {name: boolean, type: boolean, default: true}",
|
|
131
|
+
" - {name: long, type: long, default: 0}",
|
|
132
|
+
" - {name: double, type: double, default: 0.5}",
|
|
133
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
|
134
|
+
Schema inputSchema = Schema.builder()
|
|
135
|
+
.add("timestamp", TIMESTAMP)
|
|
136
|
+
.add("string", STRING)
|
|
137
|
+
.add("boolean", BOOLEAN)
|
|
138
|
+
.add("long", LONG)
|
|
139
|
+
.add("double", DOUBLE)
|
|
140
|
+
.add("json", JSON)
|
|
141
|
+
.add("remove_me", STRING)
|
|
142
|
+
.build();
|
|
143
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
144
|
+
Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
|
|
145
|
+
null, null, null, null, null, null, "remove_me");
|
|
146
|
+
|
|
147
|
+
assertEquals(2, records.size());
|
|
148
|
+
|
|
149
|
+
Object[] record;
|
|
150
|
+
{
|
|
151
|
+
record = records.get(0);
|
|
152
|
+
assertEquals(6, record.length);
|
|
153
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
|
|
154
|
+
assertEquals("string", record[1]);
|
|
155
|
+
assertEquals(new Boolean(true), record[2]);
|
|
156
|
+
assertEquals(new Long(0), record[3]);
|
|
157
|
+
assertEquals(new Double(0.5), record[4]);
|
|
158
|
+
assertEquals(ValueFactory.newString("json"), record[5]);
|
|
159
|
+
}
|
|
160
|
+
{
|
|
161
|
+
record = records.get(1);
|
|
162
|
+
assertEquals(6, record.length);
|
|
163
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
|
|
164
|
+
assertEquals("string", record[1]);
|
|
165
|
+
assertEquals(new Boolean(true), record[2]);
|
|
166
|
+
assertEquals(new Long(0), record[3]);
|
|
167
|
+
assertEquals(new Double(0.5), record[4]);
|
|
168
|
+
assertEquals("{\"foo\":\"bar\"}", record[5].toString());
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
@Test
|
|
173
|
+
public void visit_Columns_WithSrc()
|
|
174
|
+
{
|
|
175
|
+
PluginTask task = taskFromYamlString(
|
|
176
|
+
"type: column",
|
|
177
|
+
"columns:",
|
|
178
|
+
" - {name: copy, src: src}");
|
|
179
|
+
Schema inputSchema = Schema.builder()
|
|
180
|
+
.add("src", STRING)
|
|
181
|
+
.build();
|
|
182
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
183
|
+
"src");
|
|
184
|
+
|
|
185
|
+
assertEquals(1, records.size());
|
|
186
|
+
|
|
187
|
+
Object[] record;
|
|
188
|
+
{
|
|
189
|
+
record = records.get(0);
|
|
190
|
+
assertEquals(1, record.length);
|
|
191
|
+
assertEquals("src", record[0]);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
@Test
|
|
196
|
+
public void visit_DropColumns()
|
|
197
|
+
{
|
|
198
|
+
PluginTask task = taskFromYamlString(
|
|
199
|
+
"type: column",
|
|
200
|
+
"drop_columns:",
|
|
201
|
+
" - {name: timestamp}",
|
|
202
|
+
" - {name: string}",
|
|
203
|
+
" - {name: boolean}",
|
|
204
|
+
" - {name: long}",
|
|
205
|
+
" - {name: double}",
|
|
206
|
+
" - {name: json}");
|
|
207
|
+
Schema inputSchema = Schema.builder()
|
|
208
|
+
.add("timestamp", TIMESTAMP)
|
|
209
|
+
.add("string", STRING)
|
|
210
|
+
.add("boolean", BOOLEAN)
|
|
211
|
+
.add("long", LONG)
|
|
212
|
+
.add("double", DOUBLE)
|
|
213
|
+
.add("json", JSON)
|
|
214
|
+
.add("keep_me", STRING)
|
|
215
|
+
.build();
|
|
216
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
217
|
+
Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "keep_me",
|
|
218
|
+
null, null, null, null, null, null, "keep_me");
|
|
219
|
+
|
|
220
|
+
assertEquals(2, records.size());
|
|
221
|
+
|
|
222
|
+
Object[] record;
|
|
223
|
+
{
|
|
224
|
+
record = records.get(0);
|
|
225
|
+
assertEquals(1, record.length);
|
|
226
|
+
assertEquals("keep_me", record[0]);
|
|
227
|
+
}
|
|
228
|
+
{
|
|
229
|
+
record = records.get(1);
|
|
230
|
+
assertEquals(1, record.length);
|
|
231
|
+
assertEquals("keep_me", record[0]);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
@Test
|
|
236
|
+
public void visit_AddColumns_WithDefault()
|
|
237
|
+
{
|
|
238
|
+
PluginTask task = taskFromYamlString(
|
|
239
|
+
"type: column",
|
|
240
|
+
"add_columns:",
|
|
241
|
+
" - {name: timestamp, type: timestamp, default: 2015-07-13, format: \"%Y-%m-%d\", timezone: UTC}",
|
|
242
|
+
" - {name: string, type: string, default: string}",
|
|
243
|
+
" - {name: boolean, type: boolean, default: true}",
|
|
244
|
+
" - {name: long, type: long, default: 0}",
|
|
245
|
+
" - {name: double, type: double, default: 0.5}",
|
|
246
|
+
" - {name: json, type: json, default: \"{\\\"foo\\\":\\\"bar\\\"}\" }");
|
|
247
|
+
Schema inputSchema = Schema.builder()
|
|
248
|
+
.add("keep_me", STRING)
|
|
249
|
+
.build();
|
|
250
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
251
|
+
"keep_me",
|
|
252
|
+
"keep_me");
|
|
253
|
+
|
|
254
|
+
assertEquals(2, records.size());
|
|
255
|
+
|
|
256
|
+
Object[] record;
|
|
257
|
+
{
|
|
258
|
+
record = records.get(0);
|
|
259
|
+
assertEquals(7, record.length);
|
|
260
|
+
assertEquals("keep_me", record[0]);
|
|
261
|
+
assertEquals(Timestamp.ofEpochSecond(1436745600), record[1]);
|
|
262
|
+
assertEquals("string", record[2]);
|
|
263
|
+
assertEquals(new Boolean(true), record[3]);
|
|
264
|
+
assertEquals(new Long(0), record[4]);
|
|
265
|
+
assertEquals(new Double(0.5), record[5]);
|
|
266
|
+
assertEquals("{\"foo\":\"bar\"}", record[6].toString());
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
@Test
|
|
271
|
+
public void visit_AddColumns_WithSrc()
|
|
272
|
+
{
|
|
273
|
+
PluginTask task = taskFromYamlString(
|
|
274
|
+
"type: column",
|
|
275
|
+
"add_columns:",
|
|
276
|
+
" - {name: copy, src: src}");
|
|
277
|
+
Schema inputSchema = Schema.builder()
|
|
278
|
+
.add("src", STRING)
|
|
279
|
+
.build();
|
|
280
|
+
List<Object[]> records = filter(task, inputSchema,
|
|
281
|
+
"src");
|
|
282
|
+
|
|
283
|
+
assertEquals(1, records.size());
|
|
284
|
+
|
|
285
|
+
Object[] record;
|
|
286
|
+
{
|
|
287
|
+
record = records.get(0);
|
|
288
|
+
assertEquals(2, record.length);
|
|
289
|
+
assertEquals("src", record[0]);
|
|
290
|
+
assertEquals("src", record[1]);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
}
|
|
@@ -1,51 +1,29 @@
|
|
|
1
1
|
package org.embulk.filter.column;
|
|
2
2
|
|
|
3
|
-
import com.google.common.base.Optional;
|
|
4
3
|
import com.google.common.collect.Lists;
|
|
5
|
-
import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
|
|
6
4
|
import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
|
|
7
5
|
|
|
8
6
|
import org.embulk.EmbulkTestRuntime;
|
|
9
7
|
import org.embulk.config.ConfigLoader;
|
|
10
8
|
import org.embulk.config.ConfigSource;
|
|
11
|
-
import org.embulk.config.TaskSource;
|
|
12
9
|
import org.embulk.spi.Column;
|
|
13
10
|
import org.embulk.spi.Exec;
|
|
14
|
-
import org.embulk.spi.FileInput;
|
|
15
|
-
import org.embulk.spi.ParserPlugin;
|
|
16
11
|
import org.embulk.spi.Schema;
|
|
17
|
-
import org.embulk.spi.SchemaConfig;
|
|
18
|
-
import org.embulk.spi.type.Type;
|
|
19
|
-
import org.joda.time.DateTimeZone;
|
|
20
12
|
import org.junit.Before;
|
|
21
13
|
import org.junit.Rule;
|
|
22
14
|
import org.junit.Test;
|
|
23
15
|
|
|
24
|
-
import static junit.framework.TestCase.assertFalse;
|
|
25
|
-
import static org.junit.Assert.assertEquals;
|
|
26
|
-
import static org.junit.Assert.fail;
|
|
27
|
-
|
|
28
16
|
import org.msgpack.value.MapValue;
|
|
29
17
|
import org.msgpack.value.Value;
|
|
30
18
|
import org.msgpack.value.ValueFactory;
|
|
31
19
|
|
|
32
|
-
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
33
|
-
import static org.embulk.spi.type.Types.DOUBLE;
|
|
34
20
|
import static org.embulk.spi.type.Types.JSON;
|
|
35
|
-
import static org.embulk.spi.type.Types.LONG;
|
|
36
|
-
import static org.embulk.spi.type.Types.STRING;
|
|
37
|
-
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
38
21
|
import static org.junit.Assert.assertEquals;
|
|
39
|
-
import static org.junit.Assert.
|
|
22
|
+
import static org.junit.Assert.assertFalse;
|
|
40
23
|
import static org.junit.Assert.assertTrue;
|
|
41
|
-
import static org.junit.Assert.fail;
|
|
42
24
|
|
|
43
|
-
import java.io.File;
|
|
44
|
-
import java.io.IOException;
|
|
45
|
-
import java.util.ArrayList;
|
|
46
25
|
import java.util.HashMap;
|
|
47
26
|
import java.util.HashSet;
|
|
48
|
-
import java.util.List;
|
|
49
27
|
|
|
50
28
|
public class TestJsonVisitor {
|
|
51
29
|
@Rule
|
|
@@ -54,7 +32,6 @@ public class TestJsonVisitor {
|
|
|
54
32
|
@Before
|
|
55
33
|
public void createReasource()
|
|
56
34
|
{
|
|
57
|
-
// config = config().set("type", "column");
|
|
58
35
|
}
|
|
59
36
|
|
|
60
37
|
private ConfigSource config()
|
|
@@ -62,12 +39,7 @@ public class TestJsonVisitor {
|
|
|
62
39
|
return runtime.getExec().newConfigSource();
|
|
63
40
|
}
|
|
64
41
|
|
|
65
|
-
private
|
|
66
|
-
{
|
|
67
|
-
return new Schema(Lists.newArrayList(columns));
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
private ConfigSource configFromYamlString(String... lines)
|
|
42
|
+
private PluginTask taskFromYamlString(String... lines)
|
|
71
43
|
{
|
|
72
44
|
StringBuilder builder = new StringBuilder();
|
|
73
45
|
for (String line : lines) {
|
|
@@ -76,12 +48,7 @@ public class TestJsonVisitor {
|
|
|
76
48
|
String yamlString = builder.toString();
|
|
77
49
|
|
|
78
50
|
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
|
79
|
-
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
private PluginTask taskFromYamlString(String... lines)
|
|
83
|
-
{
|
|
84
|
-
ConfigSource config = configFromYamlString(lines);
|
|
51
|
+
ConfigSource config = loader.fromYamlString(yamlString);
|
|
85
52
|
return config.loadConfig(PluginTask.class);
|
|
86
53
|
}
|
|
87
54
|
|
|
@@ -102,9 +69,10 @@ public class TestJsonVisitor {
|
|
|
102
69
|
" - {name: \"$.json1.b.b[1].b\", type: string, default: foo}",
|
|
103
70
|
"drop_columns:",
|
|
104
71
|
" - {name: \"$.json1.c.c[*].c\"}");
|
|
105
|
-
Schema inputSchema =
|
|
106
|
-
|
|
107
|
-
|
|
72
|
+
Schema inputSchema = Schema.builder()
|
|
73
|
+
.add("json1", JSON)
|
|
74
|
+
.add("json2", JSON)
|
|
75
|
+
.build();
|
|
108
76
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
109
77
|
|
|
110
78
|
assertTrue(subject.shouldVisit("$.json1.a.a.a"));
|
|
@@ -131,9 +99,10 @@ public class TestJsonVisitor {
|
|
|
131
99
|
" - {name: $.json1.a.default}",
|
|
132
100
|
" - {name: $.json1.a.copy}",
|
|
133
101
|
" - {name: \"$.json1.a.copy_array[1]\"}");
|
|
134
|
-
Schema inputSchema =
|
|
135
|
-
|
|
136
|
-
|
|
102
|
+
Schema inputSchema = Schema.builder()
|
|
103
|
+
.add("json1", JSON)
|
|
104
|
+
.add("json2", JSON)
|
|
105
|
+
.build();
|
|
137
106
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
138
107
|
|
|
139
108
|
assertFalse(subject.jsonDropColumns.containsKey("$.json1"));
|
|
@@ -163,9 +132,10 @@ public class TestJsonVisitor {
|
|
|
163
132
|
" - {name: $.json1.a.default, type: string, default: foo}",
|
|
164
133
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
165
134
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
166
|
-
Schema inputSchema =
|
|
167
|
-
|
|
168
|
-
|
|
135
|
+
Schema inputSchema = Schema.builder()
|
|
136
|
+
.add("json1", JSON)
|
|
137
|
+
.add("json2", JSON)
|
|
138
|
+
.build();
|
|
169
139
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
170
140
|
|
|
171
141
|
assertFalse(subject.jsonAddColumns.containsKey("$.json1"));
|
|
@@ -202,9 +172,10 @@ public class TestJsonVisitor {
|
|
|
202
172
|
" - {name: $.json1.a.default, type: string, default: foo}",
|
|
203
173
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
204
174
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
205
|
-
Schema inputSchema =
|
|
206
|
-
|
|
207
|
-
|
|
175
|
+
Schema inputSchema = Schema.builder()
|
|
176
|
+
.add("json1", JSON)
|
|
177
|
+
.add("json2", JSON)
|
|
178
|
+
.build();
|
|
208
179
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
209
180
|
|
|
210
181
|
assertFalse(subject.jsonColumns.containsKey("$.json1"));
|
|
@@ -242,9 +213,10 @@ public class TestJsonVisitor {
|
|
|
242
213
|
" - {name: $.json1.a.copy, src: $.json1.a.src}",
|
|
243
214
|
"columns:",
|
|
244
215
|
" - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
|
|
245
|
-
Schema inputSchema =
|
|
246
|
-
|
|
247
|
-
|
|
216
|
+
Schema inputSchema = Schema.builder()
|
|
217
|
+
.add("json1", JSON)
|
|
218
|
+
.add("json2", JSON)
|
|
219
|
+
.build();
|
|
248
220
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
249
221
|
|
|
250
222
|
assertFalse(subject.jsonDropColumns.isEmpty());
|
|
@@ -259,9 +231,10 @@ public class TestJsonVisitor {
|
|
|
259
231
|
"drop_columns:",
|
|
260
232
|
" - {name: $.json1.k1.k1}",
|
|
261
233
|
" - {name: $.json1.k2}");
|
|
262
|
-
Schema inputSchema =
|
|
263
|
-
|
|
264
|
-
|
|
234
|
+
Schema inputSchema = Schema.builder()
|
|
235
|
+
.add("json1", JSON)
|
|
236
|
+
.add("json2", JSON)
|
|
237
|
+
.build();
|
|
265
238
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
266
239
|
|
|
267
240
|
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
|
@@ -284,9 +257,10 @@ public class TestJsonVisitor {
|
|
|
284
257
|
" - {name: $.json1.k3, type: json, default: \"{}\"}",
|
|
285
258
|
" - {name: $.json1.k3.k3, type: string, default: v}",
|
|
286
259
|
" - {name: $.json1.k4, src: $.json1.k2}");
|
|
287
|
-
Schema inputSchema =
|
|
288
|
-
|
|
289
|
-
|
|
260
|
+
Schema inputSchema = Schema.builder()
|
|
261
|
+
.add("json1", JSON)
|
|
262
|
+
.add("json2", JSON)
|
|
263
|
+
.build();
|
|
290
264
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
291
265
|
|
|
292
266
|
// {"k1":{"k1":"v"},"k2":{"k2":"v"}}
|
|
@@ -311,9 +285,10 @@ public class TestJsonVisitor {
|
|
|
311
285
|
" - {name: $.json1.k3, type: json, default: \"{}\"}",
|
|
312
286
|
" - {name: $.json1.k3.k3, type: string, default: v}",
|
|
313
287
|
" - {name: $.json1.k4, src: $.json1.k2}");
|
|
314
|
-
Schema inputSchema =
|
|
315
|
-
|
|
316
|
-
|
|
288
|
+
Schema inputSchema = Schema.builder()
|
|
289
|
+
.add("json1", JSON)
|
|
290
|
+
.add("json2", JSON)
|
|
291
|
+
.build();
|
|
317
292
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
318
293
|
|
|
319
294
|
// {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
|
|
@@ -335,9 +310,10 @@ public class TestJsonVisitor {
|
|
|
335
310
|
"drop_columns:",
|
|
336
311
|
" - {name: \"$.json1.k1[0].k1\"}",
|
|
337
312
|
" - {name: \"$.json1.k2[*]\"}");
|
|
338
|
-
Schema inputSchema =
|
|
339
|
-
|
|
340
|
-
|
|
313
|
+
Schema inputSchema = Schema.builder()
|
|
314
|
+
.add("json1", JSON)
|
|
315
|
+
.add("json2", JSON)
|
|
316
|
+
.build();
|
|
341
317
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
342
318
|
|
|
343
319
|
// {"k1":[{"k1":"v"}[,"k2":["v","v"]}
|
|
@@ -361,9 +337,10 @@ public class TestJsonVisitor {
|
|
|
361
337
|
" - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
|
|
362
338
|
" - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
|
|
363
339
|
" - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
|
|
364
|
-
Schema inputSchema =
|
|
365
|
-
|
|
366
|
-
|
|
340
|
+
Schema inputSchema = Schema.builder()
|
|
341
|
+
.add("json1", JSON)
|
|
342
|
+
.add("json2", JSON)
|
|
343
|
+
.build();
|
|
367
344
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
368
345
|
|
|
369
346
|
// {"k1":[{"k1":"v"}],"k2":["v","v"]}
|
|
@@ -389,9 +366,10 @@ public class TestJsonVisitor {
|
|
|
389
366
|
" - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
|
|
390
367
|
" - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
|
|
391
368
|
" - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
|
|
392
|
-
Schema inputSchema =
|
|
393
|
-
|
|
394
|
-
|
|
369
|
+
Schema inputSchema = Schema.builder()
|
|
370
|
+
.add("json1", JSON)
|
|
371
|
+
.add("json2", JSON)
|
|
372
|
+
.build();
|
|
395
373
|
JsonVisitor subject = jsonVisitor(task, inputSchema);
|
|
396
374
|
|
|
397
375
|
// {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-filter-column
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.5.
|
|
4
|
+
version: 0.5.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Naotoshi Seo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-06-
|
|
11
|
+
date: 2016-06-06 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -67,9 +67,10 @@ files:
|
|
|
67
67
|
- src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
|
|
68
68
|
- src/main/java/org/embulk/filter/column/JsonColumn.java
|
|
69
69
|
- src/main/java/org/embulk/filter/column/JsonVisitor.java
|
|
70
|
+
- src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
|
|
70
71
|
- src/test/java/org/embulk/filter/column/TestJsonColumn.java
|
|
71
72
|
- src/test/java/org/embulk/filter/column/TestJsonVisitor.java
|
|
72
|
-
- classpath/embulk-filter-column-0.5.
|
|
73
|
+
- classpath/embulk-filter-column-0.5.2.jar
|
|
73
74
|
homepage: https://github.com/sonots/embulk-filter-column
|
|
74
75
|
licenses:
|
|
75
76
|
- MIT
|