embulk-filter-base58 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b6646b2971f7197bf9636fad6b586f32637f1afa
4
- data.tar.gz: 79e3ed639b082594ce38d77c27a11ede5accf05b
3
+ metadata.gz: 8068d8efa29a9afbd6b88d304b4af1a8da0ccda9
4
+ data.tar.gz: 60795ea043c03d3b9f7f2e62559adc21caa86108
5
5
  SHA512:
6
- metadata.gz: 3c2bffac02a0f2191df9fb6c1974e24c74bc7df002ab87ca6fa4f947b87c2abbc2db7e03a67c357999dc71a8e6fe02d5cb0b0b30392c2306f52a015c44beb339
7
- data.tar.gz: 34e7d6ad1d928a30ea939695b0f73b2999d02e7ff8e43f5cea1c51e9ef4e177632bb1de49d52dd9d200a021235a22578b70bc2ef99766df4e2cff9b7fa060e0b
6
+ metadata.gz: 41aebd1779a57dddca726ad4b42184476428322c9a4fbbb5ab9cefe09f61ea8838937c471a5c3b41b4e0fb86e737a98a674238bf0c6e32bed82320184e3730d2
7
+ data.tar.gz: 421523c5edac63a00fd0c7197fa31c3dc66c0b44fab08175d851200bf15ea7b925496362251ca1f08da2426ae70ed97e8858560514c3be313b14acfaa0bff342
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.1.2"
17
+ version = "0.1.3"
18
18
 
19
19
  dependencies {
20
20
  compile "org.embulk:embulk-core:0.8.+"
@@ -88,7 +88,7 @@ public class Base58FilterPlugin implements FilterPlugin {
88
88
 
89
89
  final PluginTask task = taskSource.loadTask(PluginTask.class);
90
90
  final Map<String, List<Base58Column>> base58ColumnMap = convertBase58ColumnListToMap(task.getColumns());
91
- final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns());
91
+ final Map<String, Column> outputColumnMap = convertColumnListToMap(outputSchema.getColumns(), logger);
92
92
 
93
93
  return new PageOutput() {
94
94
  private PageReader reader = new PageReader(inputSchema);
@@ -117,62 +117,66 @@ public class Base58FilterPlugin implements FilterPlugin {
117
117
 
118
118
  void setValue(final Map<String, List<Base58Column>> base58ColumnMap, final Map<String, Column> outputColumnMap, final PageReader reader, final Schema outputSchema, final PageBuilder builder) {
119
119
 
120
- // Do base58 conversions ahead of iterating output columns
121
- final Map<String, String> base58OutputMap = new HashMap<>();
122
- for (List<Base58Column> columnSet : base58ColumnMap.values()) {
123
- for (Base58Column column : columnSet) {
120
+ // Build a map of output columns that should be mapped to encoded/decoded original columns
121
+ final Map<String, Base58Column> modifiedColumnMap = new HashMap<>(); // output column -> base58 column config
122
+ for (List<Base58Column> base58SourceColumns : base58ColumnMap.values()) {
123
+ for (Base58Column base58Column : base58SourceColumns) {
124
+ String outputColumnName = base58Column.getNewName().isPresent() ? base58Column.getNewName().get() : base58Column.getName();
125
+ modifiedColumnMap.put(outputColumnName, base58Column);
126
+ }
127
+ }
128
+
129
+ // Set values on all our output columns
130
+ List<Column> columns = outputSchema.getColumns();
131
+ for (Column outputColumn : columns) {
132
+
133
+ // Convert the value if configured to do so
134
+ if (modifiedColumnMap.containsKey(outputColumn.getName())) {
135
+ Base58Column base58Column = modifiedColumnMap.get(outputColumn.getName());
136
+ Column sourceColumn = outputColumnMap.get(base58Column.getName());
124
137
  String inputValue, convertedValue = null;
125
138
 
126
- // If the original column is not a string, then forget about it (it should be hex)
127
- Column originalColumn = outputColumnMap.get(column.getName());
128
- if (Types.STRING.equals(originalColumn.getType())) {
129
- inputValue = reader.getString(originalColumn);
139
+ // Don't bother setting it if the source is null
140
+ if (reader.isNull(sourceColumn)) {
141
+ builder.setNull(outputColumn);
142
+ continue;
143
+ }
144
+
145
+ // Get the source value
146
+ if (Types.STRING.equals(sourceColumn.getType())) {
147
+ inputValue = reader.getString(sourceColumn);
130
148
  } else {
131
149
  logger.error("cannot convert base58 value of non-string values. name: {}, type: {}, index: {}",
132
- originalColumn.getName(),
133
- originalColumn.getType(),
134
- originalColumn.getIndex());
135
- throw new DataException("Unexpected string type in column `" + originalColumn.getName() + "`. Got: " + originalColumn.getType());
150
+ sourceColumn.getName(),
151
+ sourceColumn.getType(),
152
+ sourceColumn.getIndex());
153
+ throw new DataException("Unexpected non-string type in column `" + sourceColumn.getName() + "`. Got: " + sourceColumn.getType());
136
154
  }
137
155
 
138
- // Convert the value
156
+ // Convert the source value
139
157
  try {
140
- convertedValue = convertValue(inputValue, column.getIsEncode().or(true), column.getPrefix().or(""));
158
+ convertedValue = convertValue(inputValue, base58Column.getIsEncode().or(true), base58Column.getPrefix().or(""));
141
159
  } catch (Exception e) {
142
160
  // Failed to do the conversion. Probably misconfigured or malformed value
143
161
  logger.error("failed to encode/decode base58 column value. name: {}, type: {}, index: {}, value: {}, method: {}, prefix: {}, target_name: {}",
144
- originalColumn.getName(),
145
- originalColumn.getType(),
146
- originalColumn.getIndex(),
162
+ sourceColumn.getName(),
163
+ sourceColumn.getType(),
164
+ sourceColumn.getIndex(),
147
165
  inputValue,
148
- column.getIsEncode().get() ? "encode" : "decode",
149
- column.getPrefix(),
150
- column.getNewName().or(column.getName()));
166
+ base58Column.getIsEncode().get() ? "encode" : "decode",
167
+ base58Column.getPrefix(),
168
+ base58Column.getNewName().or(base58Column.getName()));
151
169
  logger.error("base58 conversion exception", e);
152
170
  // Don't crash the import if a single value is screwed up. Just log it for now
153
171
  }
154
172
 
155
- // Add it to the output mappings
156
- if (column.getNewName().isPresent()) {
157
- base58OutputMap.put(column.getNewName().get(), convertedValue);
158
- } else {
159
- base58OutputMap.put(column.getName(), convertedValue);
160
- }
161
- }
162
- }
163
-
164
- List<Column> columns = outputSchema.getColumns();
165
- for (Column outputColumn : columns) {
166
-
167
- // Did we convert the value for this column?
168
- if (base58OutputMap.containsKey(outputColumn.getName())) {
169
- String value = base58OutputMap.get(outputColumn.getName());
170
- if (value == null) {
173
+ if (convertedValue == null) {
171
174
  builder.setNull(outputColumn);
175
+ continue;
172
176
  } else {
173
- builder.setString(outputColumn, base58OutputMap.get(outputColumn.getName()));
177
+ builder.setString(outputColumn, convertedValue);
178
+ continue;
174
179
  }
175
- continue;
176
180
  }
177
181
 
178
182
  // No value?
@@ -199,6 +203,9 @@ public class Base58FilterPlugin implements FilterPlugin {
199
203
  }
200
204
  else if (Types.JSON.equals(outputColumn.getType())) {
201
205
  builder.setJson(outputColumn, reader.getJson(outputColumn));
206
+ } else {
207
+ // NO VALUE? It shall be null.
208
+ builder.setNull(outputColumn);
202
209
  }
203
210
  }
204
211
  }
@@ -222,9 +229,12 @@ public class Base58FilterPlugin implements FilterPlugin {
222
229
  return result;
223
230
  }
224
231
 
225
- static Map<String, Column> convertColumnListToMap(List<Column> columns) {
232
+ static Map<String, Column> convertColumnListToMap(List<Column> columns, Logger logger) {
226
233
  Map<String, Column> result = new HashMap<>();
227
234
  for (Column column : columns) {
235
+ if (result.containsKey(column.getName())) {
236
+ logger.warn("Output column ({}) already defined. Do you have duplicate column names in your source?", column.getName());
237
+ }
228
238
  result.put(column.getName(), column);
229
239
  }
230
240
  return result;
@@ -57,7 +57,7 @@ public class TestBase58FilterImpl {
57
57
  PageBuilder pageBuilder = new PageBuilder(runtime.getBufferAllocator(), outputSchema, output);
58
58
  PageReader pageReader = new PageReader(inputSchema);
59
59
  final Map<String, List<Base58FilterPlugin.Base58Column>> base58ColumnMap = Base58FilterPlugin.convertBase58ColumnListToMap(task.getColumns());
60
- final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns());
60
+ final Map<String, Column> outputColumnMap = Base58FilterPlugin.convertColumnListToMap(outputSchema.getColumns(), null);
61
61
 
62
62
  List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema, objects);
63
63
  for (Page page : pages) {
@@ -141,7 +141,14 @@ public class TestBase58FilterImpl {
141
141
  .build();
142
142
 
143
143
  List<Object[]> records = filter(task, inputSchema,
144
- "54f5f8b37c158c2f12ee1c64", Timestamp.ofEpochSecond(0), null, new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"));
144
+ "54f5f8b37c158c2f12ee1c64",
145
+ Timestamp.ofEpochSecond(0),
146
+ null,
147
+ new Boolean(true),
148
+ new Long(0),
149
+ new Double(0.5),
150
+ ValueFactory.newString("json")
151
+ );
145
152
 
146
153
  assertEquals(1, records.size());
147
154
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-filter-base58
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kevin Fitzgerald
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-06-26 00:00:00.000000000 Z
11
+ date: 2017-06-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -59,7 +59,7 @@ files:
59
59
  - src/main/java/org/embulk/filter/base58/Base58FilterPlugin.java
60
60
  - src/test/java/org/embulk/filter/base58/TestBase58FilterImpl.java
61
61
  - src/test/java/org/embulk/filter/base58/TestBase58FilterPlugin.java
62
- - classpath/embulk-filter-base58-0.1.2.jar
62
+ - classpath/embulk-filter-base58-0.1.3.jar
63
63
  homepage: https://github.com/kfitzgerald/embulk-filter-base58
64
64
  licenses:
65
65
  - MIT