embulk-filter-mask 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 131e2785761ffc21130a3351ed57d7e2a96df19d
-  data.tar.gz: 02d57d7419c31e43d91d19c23f36805ad9f4046f
+  metadata.gz: 420f82310881451f5b66848494db06f292c9f755
+  data.tar.gz: b3fb787c18bf556cf169d85343a6dacb4030f116
 SHA512:
-  metadata.gz: 95c7e11221c993510b6a5ab556f1460397d91d24b7b27740664ee10210465d20866de3743d81bdd0e3f35b024e4ded1135535887d67bdef69b8a6dacd3a386f5
-  data.tar.gz: e930e6d4510b46db97dd7343edeaa0c7079127d152bd07865d01aefcf388c55622d1eec4225525c9f904a39b0cec6df3235a2aea95d2d7784823782bd0c570af
+  metadata.gz: 501044a9ebb52794d3bda67aa9ef947157300be945e798ace60068c7162206c522dfa228eb936659493bf40135d37a2afb5f9fcf9dcf324a36c40b636a5cf1e1
+  data.tar.gz: b482c2be719bab2770ba7f34c618d0d2bd611f56103bcec9df102be970965d9defbfb2af6297671d1e78674a4ba5ca1eb39eda773c8c0fe5de3f341b3b6c21c8

data/.travis.yml

@@ -0,0 +1,12 @@
+language: java
+jdk:
+# Tentatively ignore these environments as we face SSL errors
+# https://github.com/gradle/gradle/issues/2421
+#
+#  - openjdk7
+#  - oraclejdk7
+  - oraclejdk8
+script:
+  - ./gradlew test
+after_success:
+  - ./gradlew jacocoTestReport coveralls

data/README.md

@@ -1,6 +1,8 @@
 # Mask filter plugin for Embulk
 
-mask columns with asterisks (still in initial development phase and missing basic functionalities to use in production )
+[![Coverage Status](https://coveralls.io/repos/github/beniyama/embulk-filter-mask/badge.svg)](https://coveralls.io/github/beniyama/embulk-filter-mask)
+
+Mask columns with asterisks in a variety of patterns (still in initial development phase and missing basic features to use in production).
 
 ## Overview
 
@@ -8,13 +10,23 @@ mask columns with asterisks (still in initial development phase and missing basi
 
 ## Configuration
 
+*Caution* : Now we use `type` to specify mask types such as `all` and `email`, instead of `pattern` which was used in version 0.1.1 or earlier.
+
 - **columns**: target columns which would be replaced with asterisks (string, required)
   - **name**: name of the column (string, required)
-  - **pattern**: mask pattern, `all` or `email` (string, default: `all`)
-  - **path**: JSON path, works if the column type is JSON (string, default: `$.`)
+  - **type**: mask type, `all`, `email`, `regex` or `substring` (string, default: `all`)
+  - **paths**: list of JSON path and type, works if the column type is JSON
+    - `[{key: $.json_path1}, {key: $.json_path2}]` would mask both `$.json_path1` and `$.json_path2` nodes
+    - Elements under the nodes would be converted to string and then masked (e.g., `[0,1,2]` -> `*******`)
+  - **length**: if specified, this filter replaces the column with fixed number of asterisks (integer, optional. supported only in `all`, `email`, `substring`.)
+  - **pattern**: Regex pattern such as "[0-9]+" (string, required for `regex` type)
+  - **start**: The beginning index for `substring` type. The value starts from 0 and inclusive (integer, default: 0)
+  - **end**: The ending index for `substring` type. The value is exclusive (integer, default: length of the target column)
 
 ## Example
 
+
+
 If you have below data in csv or other format file,
 
 |first_name | last_name | gender | age | contact |
@@ -33,7 +45,7 @@ filters:
     columns:
       - { name: last_name}
       - { name: age}
-      - { name: contact, pattern: email}
+      - { name: contact, type: email, length: 5}
 ```
 
 would produce
@@ -46,9 +58,29 @@ would produce
 | Christian | **** | male | ** | *****@example.com |
 | Amy |	***** | female | ** | *****@example.com |
 
+If you use `regex` and/or `substring` types,
+
+```yaml
+filters:
+  - type: mask
+    columns:
+      - { name: last_name, type: regex, pattern: "[a-z]"}
+      - { name: contact, type: substring, start: 5, length: 5}
+```
+
+would produce
+
+|first_name | last_name | gender | age | contact |
+|---|---|---|---|---|
+| B******* | Bell | male | 30 | bell.***** |
+| L**** | Duncan | male | 20 | lucas***** |
+| E******* |	May | female | 25 | eliza***** |
+| C******** | Reid | male | 15 | chris***** |
+| A** |	Avery | female | 40 | amy.a***** |
+
 JSON type column is also partially supported.
 
-If you have
+If you have a `user` column with this JSON data structure
 
 ```json
 {
@@ -57,7 +89,8 @@ If you have
     "last_name": "Bell"
   },
   "gender": "male",
-  "age": 30
+  "age": 30,
+  "email": "test_mail@example.com"
 }
 ```
 
@@ -67,8 +100,7 @@ below filter configuration
 filters:
   - type: mask
     columns:
-      - { name: full_name, path: $.first_name}
-      - { name: age, path: $.}      
+      - { name: user, paths: [{key: $.full_name.first_name}, {key: $.email, type: email}]}    
 ```
 
 would produce
@@ -80,7 +112,8 @@ would produce
     "last_name": "Bell"
   },
   "gender": "male",
-  "age": **
+  "age": 30,
+  "email": "*********@example.com"
 }
 ```
 

data/build.gradle

@@ -1,8 +1,10 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.kt3k.coveralls" version "2.8.1"
     id "java"
     id "checkstyle"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -13,17 +15,24 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.15"
-    provided "org.embulk:embulk-core:0.8.15"
+    compile  "org.embulk:embulk-core:0.8.29"
+    provided "org.embulk:embulk-core:0.8.29"
     compile "com.jayway.jsonpath:json-path:2.+"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.15:tests"
+    testCompile "org.embulk:embulk-core:0.8.29:tests"
+}
+
+jacocoTestReport {
+    reports {
+        xml.enabled = true // coveralls plugin depends on xml format report
+        html.enabled = true
+    }
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Tue Jul 12 16:30:09 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/src/main/java/org/embulk/filter/mask/MaskFilterPlugin.java

@@ -30,7 +30,7 @@ public class MaskFilterPlugin implements FilterPlugin {
         String getName();
 
         @Config("type")
-        @ConfigDefault("\"string\"")
+        @ConfigDefault("\"all\"")
         Optional<String> getType();
 
         @Config("pattern")
@@ -41,6 +41,14 @@ public class MaskFilterPlugin implements FilterPlugin {
         @ConfigDefault("null")
         Optional<Integer> getLength();
 
+        @Config("start")
+        @ConfigDefault("null")
+        Optional<Integer> getStart();
+
+        @Config("end")
+        @ConfigDefault("null")
+        Optional<Integer> getEnd();
+
         @Config("paths")
         @ConfigDefault("null")
         Optional<List<Map<String, String>>> getPaths();

data/src/main/java/org/embulk/filter/mask/MaskPageOutput.java

@@ -7,6 +7,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.spi.*;
 import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
 import org.embulk.filter.mask.MaskFilterPlugin.*;
 import org.msgpack.value.Value;
@@ -67,62 +68,87 @@ public class MaskPageOutput implements PageOutput {
                 continue;
             }
 
-            Object inputValue;
-            if (Types.STRING.equals(inputColumn.getType())) {
+            String name = inputColumn.getName();
+            Type type = inputColumn.getType();
+
+            if (Types.STRING.equals(type)) {
                 final String value = reader.getString(inputColumn);
-                inputValue = value;
-                builder.setString(inputColumn, value);
-            } else if (Types.BOOLEAN.equals(inputColumn.getType())) {
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setString(inputColumn, maskAsString(name, value));
+                } else {
+                    builder.setString(inputColumn, value);
+                }
+            } else if (Types.BOOLEAN.equals(type)) {
                 final boolean value = reader.getBoolean(inputColumn);
-                inputValue = value;
-                builder.setBoolean(inputColumn, value);
-            } else if (Types.DOUBLE.equals(inputColumn.getType())) {
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setString(inputColumn, maskAsString(name, value));
+                } else {
+                    builder.setBoolean(inputColumn, value);
+                }
+            } else if (Types.DOUBLE.equals(type)) {
                 final double value = reader.getDouble(inputColumn);
-                inputValue = value;
-                builder.setDouble(inputColumn, value);
-            } else if (Types.LONG.equals(inputColumn.getType())) {
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setString(inputColumn, maskAsString(name, value));
+                } else {
+                    builder.setDouble(inputColumn, value);
+                }
+            } else if (Types.LONG.equals(type)) {
                 final long value = reader.getLong(inputColumn);
-                inputValue = value;
-                builder.setLong(inputColumn, value);
-            } else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setString(inputColumn, maskAsString(name, value));
+                } else {
+                    builder.setLong(inputColumn, value);
+                }
+            } else if (Types.TIMESTAMP.equals(type)) {
                 final Timestamp value = reader.getTimestamp(inputColumn);
-                inputValue = value;
-                builder.setTimestamp(inputColumn, value);
-            } else if (Types.JSON.equals(inputColumn.getType())) {
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setString(inputColumn, maskAsString(name, value));
+                } else {
+                    builder.setTimestamp(inputColumn, value);
+                }
+            } else if (Types.JSON.equals(type)) {
                 final Value value = reader.getJson(inputColumn);
-                inputValue = value;
-                builder.setJson(inputColumn, value);
+                if (maskColumnMap.containsKey(name)) {
+                    builder.setJson(inputColumn, maskAsJson(name, value));
+                } else {
+                    builder.setJson(inputColumn, value);
+                }
             } else {
-                throw new DataException("Unexpected type:" + inputColumn.getType());
+                throw new DataException("Unexpected type:" + type);
             }
+        }
+    }
 
-            if (maskColumnMap.containsKey(inputColumn.getName())) {
-                MaskColumn maskColumn = maskColumnMap.get(inputColumn.getName());
-
-                if (Types.JSON.equals(inputColumn.getType())) {
-                    Value inputJson = (Value) inputValue;
-                    DocumentContext context = parseContext.parse(inputJson.toJson());
-                    List<Map<String, String>> paths = maskColumn.getPaths().or(new ArrayList<Map<String, String>>());
-
-                    for (Map<String, String> path : paths) {
-                        String key = path.get("key");
-                        String pattern = path.containsKey("pattern") ? path.get("pattern") : "all";
-                        int maskLength = path.containsKey("length") ? Integer.parseInt(path.get("length")) : 0;
-                        Object element = context.read(key);
-                        if (!key.equals("$") && element != null) {
-                            String maskedValue = mask(element, pattern, maskLength);
-                            String maskedJson = context.set(key, new TextNode(maskedValue).asText()).jsonString();
-                            builder.setJson(inputColumn, jsonParser.parse(maskedJson));
-                        }
-                    }
-                } else {
-                    String pattern = maskColumn.getPattern().get();
-                    int maskLength = maskColumn.getLength().or(0);
-                    String maskedString = mask(inputValue, pattern, maskLength);
-                    builder.setString(inputColumn, maskedString);
-                }
+    private String maskAsString(String name, Object value) {
+        MaskColumn maskColumn = maskColumnMap.get(name);
+        String type = maskColumn.getType().get();
+        String pattern = maskColumn.getPattern().or("");
+        Integer length = maskColumn.getLength().or(-1);
+        Integer start = maskColumn.getStart().or(-1);
+        Integer end = maskColumn.getEnd().or(-1);
+
+        return mask(type, value, pattern, length, start, end);
+    }
+
+    private Value maskAsJson(String name, Value value) {
+        MaskColumn maskColumn = maskColumnMap.get(name);
+        DocumentContext context = parseContext.parse(value.toJson());
+        List<Map<String, String>> paths = maskColumn.getPaths().or(new ArrayList<Map<String, String>>());
+
+        for (Map<String, String> path : paths) {
+            String key = path.get("key");
+            String type = path.containsKey("type") ? path.get("type") : "all";
+            String pattern = path.containsKey("pattern") ? path.get("pattern") : "";
+            Integer length = path.containsKey("length") ? Integer.parseInt(path.get("length")) : -1;
+            Integer start = path.containsKey("start") ? Integer.parseInt(path.get("start")) : -1;
+            Integer end = path.containsKey("end") ? Integer.parseInt(path.get("end")) : -1;
+            Object element = context.read(key);
+            if (!key.equals("$") && element != null) {
+                String maskedValue = mask(type, element, pattern, length, start, end);
+                context.set(key, new TextNode(maskedValue).asText()).jsonString();
             }
         }
+        return jsonParser.parse(context.jsonString());
     }
 
     @Override
@@ -135,25 +161,61 @@ public class MaskPageOutput implements PageOutput {
         builder.close();
     }
 
-    private String mask(Object value, String pattern, Integer length) {
+    private String mask(String type, Object value, String pattern, Integer length, Integer start, Integer end) {
         String maskedValue;
         String nakedValue = value.toString();
-        if (pattern.equals("email")) {
-            if (length > 0) {
-                String maskPattern = StringUtils.repeat("*", length) + "@$1";
-                maskedValue = nakedValue.replaceFirst("^.+?@(.+)$", maskPattern);
-            } else {
-                maskedValue = nakedValue.replaceAll(".(?=[^@]*@)", "*");
-            }
-        } else if (pattern.equals("all")) {
-            if (length > 0) {
-                maskedValue = StringUtils.repeat("*", length);
-            } else {
-                maskedValue = nakedValue.replaceAll(".", "*");
-            }
+        if (type.equals("regex")) {
+            maskedValue = maskRegex(nakedValue, pattern);
+        } else if (type.equals("substring")) {
+            maskedValue = maskSubstring(nakedValue, start, end, length);
+        } else if (type.equals("email")) {
+            maskedValue = maskEmail(nakedValue, length);
+        } else if (type.equals("all")) {
+            maskedValue = maskAll(nakedValue, length);
         } else {
             maskedValue = nakedValue;
         }
         return maskedValue;
     }
+
+    private String maskAll(Object value, Integer length) {
+        String maskedValue;
+        String nakedValue = value.toString();
+        if (length > 0) {
+            maskedValue = StringUtils.repeat("*", length);
+        } else {
+            maskedValue = nakedValue.replaceAll(".", "*");
+        }
+        return maskedValue;
+    }
+
+    private String maskEmail(Object value, Integer length) {
+        String maskedValue;
+        String nakedValue = value.toString();
+        if (length > 0) {
+            String maskPattern = StringUtils.repeat("*", length) + "@$1";
+            maskedValue = nakedValue.replaceFirst("^.+?@(.+)$", maskPattern);
+        } else {
+            maskedValue = nakedValue.replaceAll(".(?=[^@]*@)", "*");
+        }
+        return maskedValue;
+    }
+
+    private String maskRegex(Object value, String pattern) {
+        String nakedValue = value.toString();
+        return nakedValue.replaceAll(pattern, "*");
+    }
+
+    private String maskSubstring(Object value, Integer start, Integer end, Integer length) {
+        String nakedValue = value.toString();
+
+        if (nakedValue.length() <= start || (0 <= end && (end - 1) <= start)) return nakedValue;
+
+        start = start < 0 ? 0 : start;
+        end = (end < 0 || nakedValue.length() <= end) ? nakedValue.length() : end;
+        int repeat = length > 0 ? length : end - start;
+
+        StringBuffer buffer = new StringBuffer(nakedValue);
+        return buffer.replace(start, end, StringUtils.repeat("*", repeat)).toString();
+    }
 }

data/src/test/java/org/embulk/filter/mask/TestMaskFilterPlugin.java

@@ -262,8 +262,10 @@ public class TestMaskFilterPlugin {
                 "columns:\n" +
                 "  - { name: _c0}\n" +
                 "  - { name: _c1, paths: [{key: $.root.key1}]}\n" +
-                "  - { name: _c2, paths: [{key: $.root.key3, length: 2}, {key: $.root.key4, pattern: all}]}\n" +
-                "  - { name: _c3, paths: [{key: $.root.key1}, {key: $.root.key3.key7, pattern: email, length: 3}]}\n";
+                "  - { name: _c2, paths: [{key: $.root.key3, length: 2}, {key: $.root.key4, type: all}]}\n" +
+                "  - { name: _c3, paths: [{key: $.root.key1}, {key: $.root.key3.key7, type: email, length: 3}]}\n" +
+                "  - { name: _c4, paths: [{key: $.root.key1, type: regex, pattern: \"[0-9]\"}]}\n" +
+                "  - { name: _c5, paths: [{key: $.root.key1, type: substring, start: 2, end: 4, length: 5}]}\n";
 
         ConfigSource config = getConfigFromYaml(configYaml);
 
@@ -272,6 +274,8 @@ public class TestMaskFilterPlugin {
                 .add("_c1", JSON)
                 .add("_c2", JSON)
                 .add("_c3", JSON)
+                .add("_c4", JSON)
+                .add("_c5", JSON)
                 .build();
 
         final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
@@ -295,6 +299,8 @@ public class TestMaskFilterPlugin {
                 MockPageOutput mockPageOutput = new MockPageOutput();
                 try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
                     for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            jsonValue,
+                            jsonValue,
                             jsonValue,
                             jsonValue,
                             jsonValue,
@@ -309,11 +315,13 @@ public class TestMaskFilterPlugin {
                 assertEquals(1, records.size());
                 Object[] record = records.get(0);
 
-                assertEquals(4, record.length);
+                assertEquals(6, record.length);
                 assertEquals("{\"root\":{\"key1\":\"value1\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[0].toString());
                 assertEquals("{\"root\":{\"key1\":\"******\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[1].toString());
                 assertEquals("{\"root\":{\"key1\":\"value1\",\"key2\":2,\"key3\":\"**\",\"key4\":\"***********\"}}", record[2].toString());
                 assertEquals("{\"root\":{\"key1\":\"******\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"***@example.com\"},\"key4\":[0,1,2,3,4]}}", record[3].toString());
+                assertEquals("{\"root\":{\"key1\":\"value*\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[4].toString());
+                assertEquals("{\"root\":{\"key1\":\"va*****e1\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[5].toString());
             }
         });
     }
@@ -323,9 +331,9 @@ public class TestMaskFilterPlugin {
         String configYaml = "" +
                 "type: mask\n" +
                 "columns:\n" +
-                "  - { name: _c0, pattern: email}\n" +
-                "  - { name: _c1, pattern: email}\n" +
-                "  - { name: _c2, pattern: all}\n" +
+                "  - { name: _c0, type: email}\n" +
+                "  - { name: _c1, type: email}\n" +
+                "  - { name: _c2, type: all}\n" +
                 "  - { name: _c3}\n";
 
         ConfigSource config = getConfigFromYaml(configYaml);
@@ -372,4 +380,126 @@ public class TestMaskFilterPlugin {
             }
         });
     }
+
+    @Test
+    public void testRegexMaskType() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c1, type: regex, pattern: \"abc\" }\n" +
+                "  - { name: _c2, type: regex, pattern: \"(abc)\" }\n" +
+                "  - { name: _c3, type: regex, pattern: \"[0-9]+\" }\n" +
+                "  - { name: _c4, type: regex, pattern: \"[0-9]\" }\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", STRING)
+                .add("_c2", STRING)
+                .add("_c3", STRING)
+                .add("_c4", STRING)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String c0ColumnValue = "_c0_abcdefghi01234";
+                final String c1ColumnValue = "_c1_abcdefghi01234";
+                final String c2ColumnValue = "_c2_abcdefghi01234";
+                final String c3ColumnValue = "_c3_abcdefghi01234";
+                final String c4ColumnValue = "_c4_abcdefghi01234";
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            c0ColumnValue,
+                            c1ColumnValue,
+                            c2ColumnValue,
+                            c3ColumnValue,
+                            c4ColumnValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(5, record.length);
+                assertEquals("_c0_abcdefghi01234", record[0]);
+                assertEquals("_c1_*defghi01234", record[1]);
+                assertEquals("_c2_*defghi01234", record[2]);
+                assertEquals("_c*_abcdefghi*", record[3]);
+                assertEquals("_c*_abcdefghi*****", record[4]);
+            }
+        });
+    }
+
+    @Test
+    public void testSubstringMaskType() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c0, type: substring }\n" +
+                "  - { name: _c1, type: substring, start: 2, end: 5 }\n" +
+                "  - { name: _c2, type: substring, start: 6 }\n" +
+                "  - { name: _c3, type: substring, end: 4 }\n" +
+                "  - { name: _c4, type: substring, start: 3, length: 5 }\n" +
+                "  - { name: _c5, type: substring, start: 3, end: 2, length: 5 }\n"; // invalid configuration
+
+                ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", STRING)
+                .add("_c2", STRING)
+                .add("_c3", STRING)
+                .add("_c4", STRING)
+                .add("_c5", STRING)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String c0ColumnValue = "_c0_abcdefghi01234";
+                final String c1ColumnValue = "_c1_abcdefghi01234";
+                final String c2ColumnValue = "_c2_abcdefghi01234";
+                final String c3ColumnValue = "_c3_abcdefghi01234";
+                final String c4ColumnValue = "_c4_abcdefghi01234";
+                final String c5ColumnValue = "_c5_abcdefghi01234";
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            c0ColumnValue,
+                            c1ColumnValue,
+                            c2ColumnValue,
+                            c3ColumnValue,
+                            c4ColumnValue,
+                            c5ColumnValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(6, record.length);
+                assertEquals("******************", record[0]);
+                assertEquals("_c***bcdefghi01234", record[1]);
+                assertEquals("_c2_ab************", record[2]);
+                assertEquals("****abcdefghi01234", record[3]);
+                assertEquals("_c4*****", record[4]);
+                assertEquals("_c5_abcdefghi01234", record[5]);
+            }
+        });
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-mask
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Tetsuo Yamabe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-06 00:00:00.000000000 Z
+date: 2017-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -59,12 +60,12 @@ files:
 - src/main/java/org/embulk/filter/mask/MaskFilterPlugin.java
 - src/main/java/org/embulk/filter/mask/MaskPageOutput.java
 - src/test/java/org/embulk/filter/mask/TestMaskFilterPlugin.java
-- classpath/accessors-smart-1.1.jar
-- classpath/asm-5.0.3.jar
-- classpath/embulk-filter-mask-0.1.1.jar
-- classpath/json-path-2.2.0.jar
-- classpath/json-smart-2.2.1.jar
-- classpath/slf4j-api-1.7.16.jar
+- classpath/accessors-smart-1.2.jar
+- classpath/asm-5.0.4.jar
+- classpath/embulk-filter-mask-0.2.1.jar
+- classpath/json-path-2.4.0.jar
+- classpath/json-smart-2.3.jar
+- classpath/slf4j-api-1.7.25.jar
 homepage: https://github.com/beniyama/embulk-filter-mask
 licenses:
 - MIT