embulk-filter-split_column 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7296372eb3a2f58d15e07ebdd2afeae7c5a22589
-  data.tar.gz: 2bc5b5a43317e58ec8aa59eabeff32b1277f8de5
+  metadata.gz: 53a619abdc297f96cee13340709ceee54c3ce911
+  data.tar.gz: b92e6154f1481d505e5d03d850b1041ccc846dd2
 SHA512:
-  metadata.gz: c5e55cda448c31d069a401f7032997392436812b5dc9fc4c52d082ba95b1ddc73c6ae9027a7bb68a446365af3a0ad542acde25c455b08e56da3eea1a462cb648
-  data.tar.gz: 0b341b2cf82a6262a5183fcc4bb5368a76809837659fa62c7b2f2f46e5b58bc23f891f709d7d1f22ba193196f1ddc8c430dd6ab6ab0ac066e5f548f5a663a576
+  metadata.gz: 8be926fd3253406702e7816b802d7a56af41259f5dbd35e50f9fe9934fa9a659311418459fc733d2b5e925425d9233e8104a96b691fd536a12bf663407d73669
+  data.tar.gz: 59f4001fa6fffaa405eeb2f1d8b4d1bce44af286c6d8f59a34a05638f0edcd94254c8a1efdaffa0d2af5511de8e23870faba411646a3ab47347ff08318982b63

data/README.md

@@ -82,6 +82,7 @@ filters:
 - 0.1.0: first release
 - 0.1.1: bugfix
 - 0.1.2: add confing option 'is_skip'
+- 0.1.3: add failed log to exception msg
 
 ## Build
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/filter/split_column/SplitColumnFilterPlugin.java

@@ -107,23 +107,26 @@ public class SplitColumnFilterPlugin
                 int rowNum = 0;
                 while (reader.nextRecord()) {
                     rowNum++;
-                    String[] words = StringUtils.split(reader.getString(targetColumn),task.getDelimiter());
+                    String targetColumnValue = reader.getString(targetColumn);
+                    String[] words = StringUtils.split(targetColumnValue, task.getDelimiter());
                     SchemaConfig outputSchemaConfig = task.getOutputColumns();
                     // check split values
                     if (outputSchemaConfig.size() != words.length) {
                         Boolean isSkip = task.getIsSkip().get();
                         if (isSkip.booleanValue()) {
-                            String message = String.format("Skipped line %d: outputColumn has %d columns but value was separated in %d",
+                            String message = String.format("Skipped line %d: output_column has %d columns but value was separated in %d: \"%s\"",
                                 rowNum,
                                 outputSchemaConfig.size(),
-                                words.length
+                                words.length,
+                                targetColumnValue
                             );
                             log.warn(message);
                             continue;
                         } else {
-                            String message = String.format("outputColumn has %d columns but value was separated in %d",
+                            String message = String.format("output_column has %d columns but value was separated in %d: \"%s\"",
                                 outputSchemaConfig.size(),
-                                words.length
+                                words.length,
+                                targetColumnValue
                             );
                             throw new SplitColumnValidateException(message);
                         }

data/src/test/resources/example.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,a|1|1.1|True|2016-01-26
+2,14824,2015-01-27 19:01:23,20150127,b|2|2.2|False|2016-01-27
+3,27559,2015-01-28 02:20:02,20150128,c|3|3.3|False|2016-01-28
+4,11270,2015-01-29 11:54:36,20150129,d|4|4.4|True|2016-01-29

data/src/test/resources/example.yml

@@ -0,0 +1,26 @@
+in:
+  type: file
+  path_prefix: ./src/test/resources/example.csv
+  parser:
+    type: csv
+    delimiter: ','
+    skip_header_lines: 1
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+filters:
+  - type: split_column
+    delimiter: '|'
+    is_skip: true
+    target_key: comment
+    output_columns:
+      - {name: alph, type: string}
+      - {name: num, type: long}
+      - {name: dbl, type: double}
+      - {name: bool, type: boolean}
+      - {name: ts, type: timestamp, format: '%Y-%m-%d'}
+out:
+  type: stdout

data/src/test/resources/few.csv

@@ -0,0 +1,3 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,a|1|1.1|True
+2,14824,2015-01-27 19:01:23,20150127,b|2|2.2|False|2016-01-27

data/src/test/resources/few.yml

@@ -0,0 +1,26 @@
+in:
+  type: file
+  path_prefix: ./src/test/resources/few.csv
+  parser:
+    type: csv
+    delimiter: ','
+    skip_header_lines: 1
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+filters:
+  - type: split_column
+    delimiter: '|'
+    is_skip: true
+    target_key: comment
+    output_columns:
+      - {name: alph, type: string}
+      - {name: num, type: long}
+      - {name: dbl, type: double}
+      - {name: bool, type: boolean}
+      - {name: ts, type: timestamp, format: '%Y-%m-%d'}
+out:
+  type: stdout

data/src/test/resources/too_large.csv

@@ -0,0 +1,3 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,a|1|1.1|True|2016-01-26|a
+2,14824,2015-01-27 19:01:23,20150127,b|2|2.2|False|2016-01-27

data/src/test/resources/too_large.yml

@@ -0,0 +1,26 @@
+in:
+  type: file
+  path_prefix: ./src/test/resources/too_large.csv
+  parser:
+    type: csv
+    delimiter: ','
+    skip_header_lines: 1
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+filters:
+  - type: split_column
+    delimiter: '|'
+    is_skip: true
+    target_key: comment
+    output_columns:
+      - {name: alph, type: string}
+      - {name: num, type: long}
+      - {name: dbl, type: double}
+      - {name: bool, type: boolean}
+      - {name: ts, type: timestamp, format: '%Y-%m-%d'}
+out:
+  type: stdout

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-split_column
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - yskn67
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-18 00:00:00.000000000 Z
+date: 2017-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +58,13 @@ files:
 - lib/embulk/filter/split_column.rb
 - src/main/java/org/embulk/filter/split_column/SplitColumnFilterPlugin.java
 - src/test/java/org/embulk/filter/split_column/TestSplitColumnFilterPlugin.java
-- classpath/embulk-filter-split_column-0.1.2.jar
+- src/test/resources/example.csv
+- src/test/resources/example.yml
+- src/test/resources/few.csv
+- src/test/resources/few.yml
+- src/test/resources/too_large.csv
+- src/test/resources/too_large.yml
+- classpath/embulk-filter-split_column-0.1.3.jar
 homepage: https://github.com/yskn67/embulk-filter-split_column
 licenses:
 - MIT