embulk-filter-base58 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8068d8efa29a9afbd6b88d304b4af1a8da0ccda9
|
4
|
+
data.tar.gz: 60795ea043c03d3b9f7f2e62559adc21caa86108
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 41aebd1779a57dddca726ad4b42184476428322c9a4fbbb5ab9cefe09f61ea8838937c471a5c3b41b4e0fb86e737a98a674238bf0c6e32bed82320184e3730d2
|
7
|
+
data.tar.gz: 421523c5edac63a00fd0c7197fa31c3dc66c0b44fab08175d851200bf15ea7b925496362251ca1f08da2426ae70ed97e8858560514c3be313b14acfaa0bff342
|
data/build.gradle
CHANGED
@@ -88,7 +88,7 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
88
88
|
|
89
89
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
90
90
|
final Map<String, List<Base58Column>> base58ColumnMap = convertBase58ColumnListToMap(task.getColumns());
|
91
|
-
final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns());
|
91
|
+
final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns(), logger);
|
92
92
|
|
93
93
|
return new PageOutput() {
|
94
94
|
private PageReader reader = new PageReader(inputSchema);
|
@@ -117,62 +117,66 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
117
117
|
|
118
118
|
void setValue(final Map<String, List<Base58Column>> base58ColumnMap, final Map<String, Column> outputColumnMap, final PageReader reader, final Schema outputSchema, final PageBuilder builder) {
|
119
119
|
|
120
|
-
//
|
121
|
-
final Map<String,
|
122
|
-
for (List<Base58Column>
|
123
|
-
for (Base58Column
|
120
|
+
// Build a map of output columns that should be mapped to encoded/decoded original columns
|
121
|
+
final Map<String, Base58Column> modifiedColumnMap = new HashMap<>(); // output column -> base58 column config
|
122
|
+
for (List<Base58Column> base58SourceColumns : base58ColumnMap.values()) {
|
123
|
+
for (Base58Column base58Column : base58SourceColumns) {
|
124
|
+
String outputColumnName = base58Column.getNewName().isPresent() ? base58Column.getNewName().get() : base58Column.getName();
|
125
|
+
modifiedColumnMap.put(outputColumnName, base58Column);
|
126
|
+
}
|
127
|
+
}
|
128
|
+
|
129
|
+
// Set values on all our output columns
|
130
|
+
List<Column> columns = outputSchema.getColumns();
|
131
|
+
for (Column outputColumn : columns) {
|
132
|
+
|
133
|
+
// Convert the value if configured to do so
|
134
|
+
if (modifiedColumnMap.containsKey(outputColumn.getName())) {
|
135
|
+
Base58Column base58Column = modifiedColumnMap.get(outputColumn.getName());
|
136
|
+
Column sourceColumn = outputColumnMap.get(base58Column.getName());
|
124
137
|
String inputValue, convertedValue = null;
|
125
138
|
|
126
|
-
//
|
127
|
-
|
128
|
-
|
129
|
-
|
139
|
+
// Don't bother setting it if the source is null
|
140
|
+
if (reader.isNull(sourceColumn)) {
|
141
|
+
builder.setNull(outputColumn);
|
142
|
+
continue;
|
143
|
+
}
|
144
|
+
|
145
|
+
// Get the source value
|
146
|
+
if (Types.STRING.equals(sourceColumn.getType())) {
|
147
|
+
inputValue = reader.getString(sourceColumn);
|
130
148
|
} else {
|
131
149
|
logger.error("cannot convert base58 value of non-string values. name: {}, type: {}, index: {}",
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
throw new DataException("Unexpected string type in column `" +
|
150
|
+
sourceColumn.getName(),
|
151
|
+
sourceColumn.getType(),
|
152
|
+
sourceColumn.getIndex());
|
153
|
+
throw new DataException("Unexpected non-string type in column `" + sourceColumn.getName() + "`. Got: " + sourceColumn.getType());
|
136
154
|
}
|
137
155
|
|
138
|
-
// Convert the value
|
156
|
+
// Convert the source value
|
139
157
|
try {
|
140
|
-
convertedValue = convertValue(inputValue,
|
158
|
+
convertedValue = convertValue(inputValue, base58Column.getIsEncode().or(true), base58Column.getPrefix().or(""));
|
141
159
|
} catch (Exception e) {
|
142
160
|
// Failed to do the conversion. Probably misconfigured or malformed value
|
143
161
|
logger.error("failed to encode/decode base58 column value. name: {}, type: {}, index: {}, value: {}, method: {}, prefix: {}, target_name: {}",
|
144
|
-
|
145
|
-
|
146
|
-
|
162
|
+
sourceColumn.getName(),
|
163
|
+
sourceColumn.getType(),
|
164
|
+
sourceColumn.getIndex(),
|
147
165
|
inputValue,
|
148
|
-
|
149
|
-
|
150
|
-
|
166
|
+
base58Column.getIsEncode().get() ? "encode" : "decode",
|
167
|
+
base58Column.getPrefix(),
|
168
|
+
base58Column.getNewName().or(base58Column.getName()));
|
151
169
|
logger.error("base58 conversion exception", e);
|
152
170
|
// Don't crash the import if a single value is screwed up. Just log it for now
|
153
171
|
}
|
154
172
|
|
155
|
-
|
156
|
-
if (column.getNewName().isPresent()) {
|
157
|
-
base58OutputMap.put(column.getNewName().get(), convertedValue);
|
158
|
-
} else {
|
159
|
-
base58OutputMap.put(column.getName(), convertedValue);
|
160
|
-
}
|
161
|
-
}
|
162
|
-
}
|
163
|
-
|
164
|
-
List<Column> columns = outputSchema.getColumns();
|
165
|
-
for (Column outputColumn : columns) {
|
166
|
-
|
167
|
-
// Did we convert the value for this column?
|
168
|
-
if (base58OutputMap.containsKey(outputColumn.getName())) {
|
169
|
-
String value = base58OutputMap.get(outputColumn.getName());
|
170
|
-
if (value == null) {
|
173
|
+
if (convertedValue == null) {
|
171
174
|
builder.setNull(outputColumn);
|
175
|
+
continue;
|
172
176
|
} else {
|
173
|
-
builder.setString(outputColumn,
|
177
|
+
builder.setString(outputColumn, convertedValue);
|
178
|
+
continue;
|
174
179
|
}
|
175
|
-
continue;
|
176
180
|
}
|
177
181
|
|
178
182
|
// No value?
|
@@ -199,6 +203,9 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
199
203
|
}
|
200
204
|
else if (Types.JSON.equals(outputColumn.getType())) {
|
201
205
|
builder.setJson(outputColumn, reader.getJson(outputColumn));
|
206
|
+
} else {
|
207
|
+
// NO VALUE? It shall be null.
|
208
|
+
builder.setNull(outputColumn);
|
202
209
|
}
|
203
210
|
}
|
204
211
|
}
|
@@ -222,9 +229,12 @@ public class Base58FilterPlugin implements FilterPlugin {
|
|
222
229
|
return result;
|
223
230
|
}
|
224
231
|
|
225
|
-
static Map<String, Column> convertColumnListToMap(List<Column> columns) {
|
232
|
+
static Map<String, Column> convertColumnListToMap(List<Column> columns, Logger logger) {
|
226
233
|
Map<String, Column> result = new HashMap<>();
|
227
234
|
for (Column column : columns) {
|
235
|
+
if (result.containsKey(column.getName())) {
|
236
|
+
logger.warn("Output column ({}) already defined. Do you have duplicate column names in your source?", column.getName());
|
237
|
+
}
|
228
238
|
result.put(column.getName(), column);
|
229
239
|
}
|
230
240
|
return result;
|
@@ -57,7 +57,7 @@ public class TestBase58FilterImpl {
|
|
57
57
|
PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
|
58
58
|
PageReader pageReader = new PageReader(inputSchema);
|
59
59
|
final Map<String, List<Base58FilterPlugin.Base58Column>> base58ColumnMap = Base58FilterPlugin.convertBase58ColumnListToMap(task.getColumns());
|
60
|
-
final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns());
|
60
|
+
final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns(), null);
|
61
61
|
|
62
62
|
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
|
63
63
|
for (Page page : pages) {
|
@@ -141,7 +141,14 @@ public class TestBase58FilterImpl {
|
|
141
141
|
.build();
|
142
142
|
|
143
143
|
List<Object[]> records = filter(task, inputSchema,
|
144
|
-
"54f5f8b37c158c2f12ee1c64",
|
144
|
+
"54f5f8b37c158c2f12ee1c64",
|
145
|
+
Timestamp.ofEpochSecond(0),
|
146
|
+
null,
|
147
|
+
new Boolean(true),
|
148
|
+
new Long(0),
|
149
|
+
new Double(0.5),
|
150
|
+
ValueFactory.newString("json")
|
151
|
+
);
|
145
152
|
|
146
153
|
assertEquals(1, records.size());
|
147
154
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-filter-base58
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kevin Fitzgerald
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -59,7 +59,7 @@ files:
|
|
59
59
|
- src/main/java/org/embulk/filter/base58/Base58FilterPlugin.java
|
60
60
|
- src/test/java/org/embulk/filter/base58/TestBase58FilterImpl.java
|
61
61
|
- src/test/java/org/embulk/filter/base58/TestBase58FilterPlugin.java
|
62
|
-
- classpath/embulk-filter-base58-0.1.
|
62
|
+
- classpath/embulk-filter-base58-0.1.3.jar
|
63
63
|
homepage: https://github.com/kfitzgerald/embulk-filter-base58
|
64
64
|
licenses:
|
65
65
|
- MIT
|