embulk-filter-mask 0.0.1 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63fab6bef7044ca3f9648b6dad5cbeb6e352c632
-  data.tar.gz: a8979001625a9530aaaa3f074b4889d73e5c8f54
+  metadata.gz: 131e2785761ffc21130a3351ed57d7e2a96df19d
+  data.tar.gz: 02d57d7419c31e43d91d19c23f36805ad9f4046f
 SHA512:
-  metadata.gz: 7edd005dc56ff2edca82a02bbbe5a93e1d639b3eeb655910b99bf7f0bb5cca784b211a39ad057adb59bd763588f32c3805f09ed22757ea222e25ad8849385819
-  data.tar.gz: 183574aeb6e1a426c14ab2032f961b167c6abca1b083c50e02c320cf94bbf6f3d709f347a77a46f415a7aefc605782c6ef056056d5391143db5384709d4cf8fd
+  metadata.gz: 95c7e11221c993510b6a5ab556f1460397d91d24b7b27740664ee10210465d20866de3743d81bdd0e3f35b024e4ded1135535887d67bdef69b8a6dacd3a386f5
+  data.tar.gz: e930e6d4510b46db97dd7343edeaa0c7079127d152bd07865d01aefcf388c55622d1eec4225525c9f904a39b0cec6df3235a2aea95d2d7784823782bd0c570af

data/build.gradle

@@ -13,17 +13,17 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.1.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.10"
-    provided "org.embulk:embulk-core:0.8.10"
+    compile  "org.embulk:embulk-core:0.8.15"
+    provided "org.embulk:embulk-core:0.8.15"
     compile "com.jayway.jsonpath:json-path:2.+"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.10:tests"
+    testCompile "org.embulk:embulk-core:0.8.15:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/filter/mask/MaskFilterPlugin.java

@@ -37,9 +37,13 @@ public class MaskFilterPlugin implements FilterPlugin {
         @ConfigDefault("\"all\"")
         Optional<String> getPattern();
 
-        @Config("path")
-        @ConfigDefault("\"$.\"")
-        Optional<String> getPath();
+        @Config("length")
+        @ConfigDefault("null")
+        Optional<Integer> getLength();
+
+        @Config("paths")
+        @ConfigDefault("null")
+        Optional<List<Map<String, String>>> getPaths();
     }
 
     @Override

data/src/main/java/org/embulk/filter/mask/MaskPageOutput.java

@@ -1,10 +1,8 @@
 package org.embulk.filter.mask;
 
 import com.fasterxml.jackson.databind.node.TextNode;
-import com.jayway.jsonpath.Configuration;
-import com.jayway.jsonpath.JsonPath;
-import com.jayway.jsonpath.Option;
-import com.jayway.jsonpath.ParseContext;
+import com.jayway.jsonpath.*;
+import org.apache.commons.lang3.StringUtils;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.*;
 import org.embulk.spi.json.JsonParser;
@@ -14,6 +12,7 @@ import org.embulk.filter.mask.MaskFilterPlugin.*;
 import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -99,17 +98,27 @@ public class MaskPageOutput implements PageOutput {
 
             if (maskColumnMap.containsKey(inputColumn.getName())) {
                 MaskColumn maskColumn = maskColumnMap.get(inputColumn.getName());
-                String targetValue = inputValue.toString();
-                String pattern = maskColumn.getPattern().get();
 
                 if (Types.JSON.equals(inputColumn.getType())) {
-                    String path = maskColumn.getPath().get();
-                    String element = parseContext.parse(targetValue).read(path);
-                    String maskedValue = mask(element, pattern);
-                    String maskedJson = parseContext.parse(targetValue).set(path, new TextNode(maskedValue).asText()).jsonString();
-                    builder.setJson(inputColumn, jsonParser.parse(maskedJson));
+                    Value inputJson = (Value) inputValue;
+                    DocumentContext context = parseContext.parse(inputJson.toJson());
+                    List<Map<String, String>> paths = maskColumn.getPaths().or(new ArrayList<Map<String, String>>());
+
+                    for (Map<String, String> path : paths) {
+                        String key = path.get("key");
+                        String pattern = path.containsKey("pattern") ? path.get("pattern") : "all";
+                        int maskLength = path.containsKey("length") ? Integer.parseInt(path.get("length")) : 0;
+                        Object element = context.read(key);
+                        if (!key.equals("$") && element != null) {
+                            String maskedValue = mask(element, pattern, maskLength);
+                            String maskedJson = context.set(key, new TextNode(maskedValue).asText()).jsonString();
+                            builder.setJson(inputColumn, jsonParser.parse(maskedJson));
+                        }
+                    }
                 } else {
-                    String maskedString = mask(targetValue, pattern);
+                    String pattern = maskColumn.getPattern().get();
+                    int maskLength = maskColumn.getLength().or(0);
+                    String maskedString = mask(inputValue, pattern, maskLength);
                     builder.setString(inputColumn, maskedString);
                 }
             }
@@ -126,16 +135,24 @@ public class MaskPageOutput implements PageOutput {
         builder.close();
     }
 
-    private String mask(String value, String pattern) {
+    private String mask(Object value, String pattern, Integer length) {
         String maskedValue;
+        String nakedValue = value.toString();
         if (pattern.equals("email")) {
-            Pattern regexPattern = Pattern.compile("^.+?@(.+)$");
-            Matcher matcher = regexPattern.matcher(value);
-            maskedValue = matcher.replaceFirst("*****@$1");
+            if (length > 0) {
+                String maskPattern = StringUtils.repeat("*", length) + "@$1";
+                maskedValue = nakedValue.replaceFirst("^.+?@(.+)$", maskPattern);
+            } else {
+                maskedValue = nakedValue.replaceAll(".(?=[^@]*@)", "*");
+            }
         } else if (pattern.equals("all")) {
-            maskedValue = value.replaceAll(".", "*");
+            if (length > 0) {
+                maskedValue = StringUtils.repeat("*", length);
+            } else {
+                maskedValue = nakedValue.replaceAll(".", "*");
+            }
         } else {
-            maskedValue = value;
+            maskedValue = nakedValue;
         }
         return maskedValue;
     }

data/src/test/java/org/embulk/filter/mask/TestMaskFilterPlugin.java

@@ -1,14 +1,31 @@
 package org.embulk.filter.mask;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
-import org.embulk.spi.Exec;
-import org.embulk.spi.MockFormatterPlugin;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.*;
+import org.embulk.spi.TestPageBuilderReader.*;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.util.Pages;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
+import org.msgpack.value.Value;
+
+import java.util.List;
+
+
+import static org.embulk.filter.mask.MaskFilterPlugin.PluginTask;
+import static org.embulk.filter.mask.MaskFilterPlugin.Control;
+import static org.embulk.spi.type.Types.*;
+import static org.junit.Assert.assertEquals;
+import static org.msgpack.value.ValueFactory.*;
 
 public class TestMaskFilterPlugin {
     @Rule
@@ -17,11 +34,47 @@ public class TestMaskFilterPlugin {
     @Rule
     public ExpectedException exception = ExpectedException.none();
 
+    private static Value s(String value) {
+        return newString(value);
+    }
+
+    private static Value i(int value) {
+        return newInteger(value);
+    }
+
+    private static Value f(double value) {
+        return newFloat(value);
+    }
+
+    private static Value b(boolean value) {
+        return newBoolean(value);
+    }
+
     private ConfigSource getConfigFromYaml(String yaml) {
         ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
         return loader.fromYamlString(yaml);
     }
 
+    private String getMaskedCharacters(Object value) {
+        String maskedValue = "";
+        for (int i = 0; i < value.toString().length(); i++) {
+            maskedValue += "*";
+        }
+        return maskedValue;
+    }
+
+    private String getMaskedEmail(String email) {
+        String maskedValue = "";
+        for (int i = 0; i < email.length(); i++) {
+            if (email.charAt(i) == '@') {
+                maskedValue += email.substring(i);
+                break;
+            }
+            maskedValue += "*";
+        }
+        return maskedValue;
+    }
+
     @Test
     public void testThrowExceptionAtMissingColumnsField() {
         String configYaml = "type: mask";
@@ -29,6 +82,294 @@ public class TestMaskFilterPlugin {
 
         exception.expect(ConfigException.class);
         exception.expectMessage("Field 'columns' is required but not set");
-        config.loadConfig(MockFormatterPlugin.PluginTask.class);
+        config.loadConfig(PluginTask.class);
+    }
+
+    @Test
+    public void testOnlyMaskTargetColumns() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c0}\n" +
+                "  - { name: _c2}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", STRING)
+                .add("_c2", STRING)
+                .add("_c3", STRING)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String c0ColumnValue = "_c0_THIS_MUST_BE_MASKED";
+                final String c1ColumnValue = "_c1_THIS_MUST_NOT_BE_MASKED";
+                final String c2ColumnValue = "_c2_THIS_MUST_BE_MASKED_ALSO";
+                final String c3ColumnValue = "_c3_THIS_MUST_NOT_BE_MASKED_ALSO";
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            c0ColumnValue,
+                            c1ColumnValue,
+                            c2ColumnValue,
+                            c3ColumnValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(4, record.length);
+                assertEquals(getMaskedCharacters(c0ColumnValue), record[0]);
+                assertEquals(c1ColumnValue, record[1]);
+                assertEquals(getMaskedCharacters(c2ColumnValue), record[2]);
+                assertEquals(c3ColumnValue, record[3]);
+            }
+        });
+    }
+
+    @Test
+    public void testPassVarietyOfTypes() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _dummy}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", BOOLEAN)
+                .add("_c2", DOUBLE)
+                .add("_c3", LONG)
+                .add("_c4", TIMESTAMP)
+                .add("_c5", JSON)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String c0ColumnValue = "_c0_STRING";
+                final Boolean c1ColumnValue = false;
+                final Double c2ColumnValue = 12345.6789;
+                final Long c3ColumnValue = Long.MAX_VALUE;
+                final Timestamp c4ColumnValue = Timestamp.ofEpochSecond(4);
+                final Value c5ColumnValue = newMapBuilder().put(s("_c5"), s("_v5")).build();
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            c0ColumnValue,
+                            c1ColumnValue,
+                            c2ColumnValue,
+                            c3ColumnValue,
+                            c4ColumnValue,
+                            c5ColumnValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(6, record.length);
+                assertEquals(c0ColumnValue, record[0]);
+                assertEquals(c1ColumnValue, record[1]);
+                assertEquals(c2ColumnValue, record[2]);
+                assertEquals(c3ColumnValue, record[3]);
+                assertEquals(c4ColumnValue, record[4]);
+                assertEquals(c5ColumnValue, record[5]);
+            }
+        });
+    }
+
+    @Test
+    public void testMaskVarietyOfTypes() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c0}\n" +
+                "  - { name: _c1}\n" +
+                "  - { name: _c2}\n" +
+                "  - { name: _c3}\n" +
+                "  - { name: _c4}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", BOOLEAN)
+                .add("_c2", DOUBLE)
+                .add("_c3", LONG)
+                .add("_c4", TIMESTAMP)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String c0ColumnValue = "_c0_STRING";
+                final Boolean c1ColumnValue = false;
+                final Double c2ColumnValue = 12345.6789;
+                final Long c3ColumnValue = Long.MAX_VALUE;
+                final Timestamp c4ColumnValue = Timestamp.ofEpochSecond(4);
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            c0ColumnValue,
+                            c1ColumnValue,
+                            c2ColumnValue,
+                            c3ColumnValue,
+                            c4ColumnValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(5, record.length);
+                assertEquals(getMaskedCharacters(c0ColumnValue), record[0]);
+                assertEquals(getMaskedCharacters(c1ColumnValue), record[1]);
+                assertEquals(getMaskedCharacters(c2ColumnValue), record[2]);
+                assertEquals(getMaskedCharacters(c3ColumnValue), record[3]);
+                assertEquals(getMaskedCharacters(c4ColumnValue), record[4]);
+            }
+        });
+    }
+
+    @Test
+    public void testMaskJson() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c0}\n" +
+                "  - { name: _c1, paths: [{key: $.root.key1}]}\n" +
+                "  - { name: _c2, paths: [{key: $.root.key3, length: 2}, {key: $.root.key4, pattern: all}]}\n" +
+                "  - { name: _c3, paths: [{key: $.root.key1}, {key: $.root.key3.key7, pattern: email, length: 3}]}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", JSON)
+                .add("_c1", JSON)
+                .add("_c2", JSON)
+                .add("_c3", JSON)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final Value jsonValue = newMapBuilder().put(
+                        s("root"),
+                        newMap(
+                                s("key1"), s("value1"),
+                                s("key2"), i(2),
+                                s("key3"), newMap(
+                                        s("key5"), s("value5"),
+                                        s("key6"), newArray(i(0), i(1), i(2), i(3), i(4)),
+                                        s("key7"), s("testme@example.com")
+                                ),
+                                s("key4"), newArray(i(0), i(1), i(2), i(3), i(4))
+                        )
+                ).build();
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            jsonValue,
+                            jsonValue,
+                            jsonValue,
+                            jsonValue
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(4, record.length);
+                assertEquals("{\"root\":{\"key1\":\"value1\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[0].toString());
+                assertEquals("{\"root\":{\"key1\":\"******\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"testme@example.com\"},\"key4\":[0,1,2,3,4]}}", record[1].toString());
+                assertEquals("{\"root\":{\"key1\":\"value1\",\"key2\":2,\"key3\":\"**\",\"key4\":\"***********\"}}", record[2].toString());
+                assertEquals("{\"root\":{\"key1\":\"******\",\"key2\":2,\"key3\":{\"key5\":\"value5\",\"key6\":[0,1,2,3,4],\"key7\":\"***@example.com\"},\"key4\":[0,1,2,3,4]}}", record[3].toString());
+            }
+        });
+    }
+
+    @Test
+    public void testMaskEmail() {
+        String configYaml = "" +
+                "type: mask\n" +
+                "columns:\n" +
+                "  - { name: _c0, pattern: email}\n" +
+                "  - { name: _c1, pattern: email}\n" +
+                "  - { name: _c2, pattern: all}\n" +
+                "  - { name: _c3}\n";
+
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        final Schema inputSchema = Schema.builder()
+                .add("_c0", STRING)
+                .add("_c1", STRING)
+                .add("_c2", STRING)
+                .add("_c3", STRING)
+                .add("_c4", STRING)
+                .build();
+
+        final MaskFilterPlugin maskFilterPlugin = new MaskFilterPlugin();
+        maskFilterPlugin.transaction(config, inputSchema, new Control() {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema) {
+                final String email1 = "dummy_test-me.1234@dummy-mail1.com";
+                final String email2 = "!#$%&'*+-/=?^_`.{|}~@dummy-mail2.com";
+
+                MockPageOutput mockPageOutput = new MockPageOutput();
+                try (PageOutput pageOutput = maskFilterPlugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), inputSchema,
+                            email1,
+                            email2,
+                            email1,
+                            email1,
+                            email1
+                    )) {
+                        pageOutput.add(page);
+                    }
+                    pageOutput.finish();
+                }
+                List<Object[]> records = Pages.toObjects(outputSchema, mockPageOutput.pages);
+
+                assertEquals(1, records.size());
+                Object[] record = records.get(0);
+
+                assertEquals(5, record.length);
+                assertEquals(getMaskedEmail(email1), record[0]);
+                assertEquals(getMaskedEmail(email2), record[1]);
+                assertEquals(getMaskedCharacters(email1), record[2]);
+                assertEquals(getMaskedCharacters(email1), record[3]);
+                assertEquals(email1, record[4]);
+            }
+        });
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-mask
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.1
 platform: ruby
 authors:
 - Tetsuo Yamabe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-22 00:00:00.000000000 Z
+date: 2017-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,7 +61,7 @@ files:
 - src/test/java/org/embulk/filter/mask/TestMaskFilterPlugin.java
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
-- classpath/embulk-filter-mask-0.0.1.jar
+- classpath/embulk-filter-mask-0.1.1.jar
 - classpath/json-path-2.2.0.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/slf4j-api-1.7.16.jar