embulk-filter-base58 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b6646b2971f7197bf9636fad6b586f32637f1afa
|
4
|
+
data.tar.gz: 79e3ed639b082594ce38d77c27a11ede5accf05b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3c2bffac02a0f2191df9fb6c1974e24c74bc7df002ab87ca6fa4f947b87c2abbc2db7e03a67c357999dc71a8e6fe02d5cb0b0b30392c2306f52a015c44beb339
|
7
|
+
data.tar.gz: 34e7d6ad1d928a30ea939695b0f73b2999d02e7ff8e43f5cea1c51e9ef4e177632bb1de49d52dd9d200a021235a22578b70bc2ef99766df4e2cff9b7fa060e0b
|
data/build.gradle
CHANGED
@@ -19,6 +19,7 @@ import org.embulk.spi.Schema;
|
|
19
19
|
import org.embulk.spi.type.Types;
|
20
20
|
import org.slf4j.Logger;
|
21
21
|
|
22
|
+
import java.util.ArrayList;
|
22
23
|
import java.util.HashMap;
|
23
24
|
import java.util.List;
|
24
25
|
import java.util.Map;
|
@@ -86,7 +87,7 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
86
87
|
final Schema outputSchema, final PageOutput output) {
|
87
88
|
|
88
89
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
89
|
-
final Map<String, Base58Column
|
90
|
+
final Map<String, List<Base58Column>> base58ColumnMap = convertBase58ColumnListToMap(task.getColumns());
|
90
91
|
final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns());
|
91
92
|
|
92
93
|
return new PageOutput() {
|
@@ -114,47 +115,49 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
114
115
|
};
|
115
116
|
}
|
116
117
|
|
117
|
-
void setValue(final Map<String, Base58Column
|
118
|
+
void setValue(final Map<String, List<Base58Column>> base58ColumnMap, final Map<String, Column> outputColumnMap, final PageReader reader, final Schema outputSchema, final PageBuilder builder) {
|
118
119
|
|
119
120
|
// Do base58 conversions ahead of iterating output columns
|
120
|
-
final Map<String, String> base58OutputMap = new HashMap<>()
|
121
|
-
for (Base58Column
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
121
|
+
final Map<String, String> base58OutputMap = new HashMap<>();
|
122
|
+
for (List<Base58Column> columnSet : base58ColumnMap.values()) {
|
123
|
+
for (Base58Column column : columnSet) {
|
124
|
+
String inputValue, convertedValue = null;
|
125
|
+
|
126
|
+
// If the original column is not a string, then forget about it (it should be hex)
|
127
|
+
Column originalColumn = outputColumnMap.get(column.getName());
|
128
|
+
if (Types.STRING.equals(originalColumn.getType())) {
|
129
|
+
inputValue = reader.getString(originalColumn);
|
130
|
+
} else {
|
131
|
+
logger.error("cannot convert base58 value of non-string values. name: {}, type: {}, index: {}",
|
132
|
+
originalColumn.getName(),
|
133
|
+
originalColumn.getType(),
|
134
|
+
originalColumn.getIndex());
|
135
|
+
throw new DataException("Unexpected string type in column `" + originalColumn.getName() + "`. Got: " + originalColumn.getType());
|
136
|
+
}
|
135
137
|
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
138
|
+
// Convert the value
|
139
|
+
try {
|
140
|
+
convertedValue = convertValue(inputValue, column.getIsEncode().or(true), column.getPrefix().or(""));
|
141
|
+
} catch (Exception e) {
|
142
|
+
// Failed to do the conversion. Probably misconfigured or malformed value
|
143
|
+
logger.error("failed to encode/decode base58 column value. name: {}, type: {}, index: {}, value: {}, method: {}, prefix: {}, target_name: {}",
|
144
|
+
originalColumn.getName(),
|
145
|
+
originalColumn.getType(),
|
146
|
+
originalColumn.getIndex(),
|
147
|
+
inputValue,
|
148
|
+
column.getIsEncode().get() ? "encode" : "decode",
|
149
|
+
column.getPrefix(),
|
150
|
+
column.getNewName().or(column.getName()));
|
151
|
+
logger.error("base58 conversion exception", e);
|
152
|
+
// Don't crash the import if a single value is screwed up. Just log it for now
|
153
|
+
}
|
152
154
|
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
155
|
+
// Add it to the output mappings
|
156
|
+
if (column.getNewName().isPresent()) {
|
157
|
+
base58OutputMap.put(column.getNewName().get(), convertedValue);
|
158
|
+
} else {
|
159
|
+
base58OutputMap.put(column.getName(), convertedValue);
|
160
|
+
}
|
158
161
|
}
|
159
162
|
}
|
160
163
|
|
@@ -208,10 +211,13 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
208
211
|
}
|
209
212
|
}
|
210
213
|
|
211
|
-
static Map<String, Base58Column
|
212
|
-
Map<String, Base58Column
|
214
|
+
static Map<String, List<Base58Column>> convertBase58ColumnListToMap(List<Base58Column> base58Columns) {
|
215
|
+
Map<String, List<Base58Column>> result = new HashMap<>();
|
213
216
|
for (Base58Column base58Column : base58Columns) {
|
214
|
-
result.
|
217
|
+
if (!result.containsKey(base58Column.getName())) {
|
218
|
+
result.put(base58Column.getName(), new ArrayList<>());
|
219
|
+
}
|
220
|
+
result.get(base58Column.getName()).add(base58Column);
|
215
221
|
}
|
216
222
|
return result;
|
217
223
|
}
|
@@ -56,7 +56,7 @@ public class TestBase58FilterImpl {
|
|
56
56
|
Schema outputSchema = plugin.buildOutputSchema(task, inputSchema);
|
57
57
|
PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
|
58
58
|
PageReader pageReader = new PageReader(inputSchema);
|
59
|
-
final Map<String, Base58FilterPlugin.Base58Column
|
59
|
+
final Map<String, List<Base58FilterPlugin.Base58Column>> base58ColumnMap = Base58FilterPlugin.convertBase58ColumnListToMap(task.getColumns());
|
60
60
|
final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns());
|
61
61
|
|
62
62
|
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
|
@@ -247,6 +247,33 @@ public class TestBase58FilterImpl {
|
|
247
247
|
}
|
248
248
|
}
|
249
249
|
|
250
|
+
@Test
|
251
|
+
public void basicEncodingWithPrefixAndNewNameWithMultipleDestinations()
|
252
|
+
{
|
253
|
+
PluginTask task = taskFromYamlString(
|
254
|
+
"type: base58",
|
255
|
+
"columns:",
|
256
|
+
" - {name: _id, prefix: obj_, new_name: public_id}",
|
257
|
+
" - {name: _id, prefix: old_, new_name: old_public_id}");
|
258
|
+
Schema inputSchema = Schema.builder()
|
259
|
+
.add("_id", STRING)
|
260
|
+
.build();
|
261
|
+
|
262
|
+
List<Object[]> records = filter(task, inputSchema,
|
263
|
+
"00f5f8b37c158c2f12ee1c64");
|
264
|
+
|
265
|
+
assertEquals(1, records.size());
|
266
|
+
|
267
|
+
Object[] record;
|
268
|
+
{
|
269
|
+
record = records.get(0);
|
270
|
+
assertEquals(3, record.length);
|
271
|
+
assertEquals("00f5f8b37c158c2f12ee1c64", record[0]);
|
272
|
+
assertEquals("obj_123zhNEUWPr5ogRQP", record[1]);
|
273
|
+
assertEquals("old_123zhNEUWPr5ogRQP", record[2]);
|
274
|
+
}
|
275
|
+
}
|
276
|
+
|
250
277
|
@Test
|
251
278
|
public void basicDecodingWithPrefixAndNewName()
|
252
279
|
{
|
@@ -272,6 +299,33 @@ public class TestBase58FilterImpl {
|
|
272
299
|
}
|
273
300
|
}
|
274
301
|
|
302
|
+
@Test
|
303
|
+
public void basicDecodingWithPrefixAndNewNameWithMultipleDestinations()
|
304
|
+
{
|
305
|
+
PluginTask task = taskFromYamlString(
|
306
|
+
"type: base58",
|
307
|
+
"columns:",
|
308
|
+
" - {name: public_id, encode: false, prefix: obj_, new_name: _id}",
|
309
|
+
" - {name: public_id, encode: false, prefix: obj_, new_name: old_id}");
|
310
|
+
Schema inputSchema = Schema.builder()
|
311
|
+
.add("public_id", STRING)
|
312
|
+
.build();
|
313
|
+
|
314
|
+
List<Object[]> records = filter(task, inputSchema,
|
315
|
+
"obj_123zhNEUWPr5ogRQP");
|
316
|
+
|
317
|
+
assertEquals(1, records.size());
|
318
|
+
|
319
|
+
Object[] record;
|
320
|
+
{
|
321
|
+
record = records.get(0);
|
322
|
+
assertEquals(3, record.length);
|
323
|
+
assertEquals("obj_123zhNEUWPr5ogRQP", record[0]);
|
324
|
+
assertEquals("00f5f8b37c158c2f12ee1c64", record[1]);
|
325
|
+
assertEquals("00f5f8b37c158c2f12ee1c64", record[2]);
|
326
|
+
}
|
327
|
+
}
|
328
|
+
|
275
329
|
@Test
|
276
330
|
public void badBase58DecodeTurnsColumnValueNull()
|
277
331
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-base58
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Fitzgerald
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-06-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -59,7 +59,7 @@ files:
|
|
59
59
|
- src/main/java/org/embulk/filter/base58/Base58FilterPlugin.java
|
60
60
|
- src/test/java/org/embulk/filter/base58/TestBase58FilterImpl.java
|
61
61
|
- src/test/java/org/embulk/filter/base58/TestBase58FilterPlugin.java
|
62
|
-
- classpath/embulk-filter-base58-0.1.
|
62
|
+
- classpath/embulk-filter-base58-0.1.2.jar
|
63
63
|
homepage: https://github.com/kfitzgerald/embulk-filter-base58
|
64
64
|
licenses:
|
65
65
|
- MIT
|