embulk-filter-json_csv2arrayofobjects 0.1.0

data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/JsonCsv2arrayofobjectsFilterPlugin.java

@@ -0,0 +1,137 @@
+package org.embulk.filter.json_csv2arrayofobjects;
+
+import com.google.common.base.Optional;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Type;
+
+import java.util.List;
+
+public class JsonCsv2arrayofobjectsFilterPlugin
+        implements FilterPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("column")
+        public String getColumn();
+
+        @Config("key")
+        public String getKey();
+
+        @Config("delimiter")
+        @ConfigDefault("\",\"")
+        public Optional<String> getDelimiter();
+
+        @Config("sub_delimiter")
+        @ConfigDefault("\"-\"")
+        public Optional<String> getSubDelimiter();
+
+        @Config("sequence_name")
+        @ConfigDefault("null")
+        public Optional<String> getSequenceName();
+
+        @Config("output_keys")
+        public List<JsonKeyTask> getOutputKeys();
+    }
+
+    public interface JsonKeyTask
+            extends Task
+    {
+        @Config("name")
+        public String getName();
+
+        @Config("type")
+        public Type getType();
+    }
+
+    public void validate(PluginTask task, Schema inputSchema)
+    {
+        // throws exception when the column does not exist
+        Column column = inputSchema.lookupColumn(task.getColumn());
+        Type colType = column.getType();
+        // delimiter and sub_delimtier should not be equal
+        String delimiter = task.getDelimiter().get();
+        String subDelimiter = task.getSubDelimiter().get();
+        if (delimiter.equals(subDelimiter)) {
+            String errMsg = "delimiter and sub_delimiter should not be equal";
+            throw new ConfigException(errMsg);
+        }
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema inputSchema,
+            FilterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        validate(task, inputSchema);
+        Schema outputSchema = inputSchema;
+        control.run(task.dump(), outputSchema);
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, Schema inputSchema,
+            Schema outputSchema, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        PageBuilder pageBuilder = new PageBuilder(
+            Exec.getBufferAllocator(), outputSchema, output);
+        PageReader pageReader = new PageReader(inputSchema);
+        Filter filter = new Filter(task);
+        ColumnVisitorImpl visitor = new ColumnVisitorImpl(
+            pageReader, pageBuilder, filter, task);
+        return new PageOutputImpl(
+            pageReader, pageBuilder, outputSchema, visitor);
+    }
+
+    public static class PageOutputImpl implements PageOutput
+    {
+        private PageReader pageReader;
+        private PageBuilder pageBuilder;
+        private Schema outputSchema;
+        private ColumnVisitorImpl visitor;
+
+        PageOutputImpl(PageReader pageReader, PageBuilder pageBuilder, Schema outputSchema, ColumnVisitorImpl visitor)
+        {
+            this.pageReader = pageReader;
+            this.pageBuilder = pageBuilder;
+            this.outputSchema = outputSchema;
+            this.visitor = visitor;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            pageReader.setPage(page);
+            while (pageReader.nextRecord()) {
+                outputSchema.visitColumns(visitor);
+                pageBuilder.addRecord();
+            }
+        }
+
+        @Override
+        public void finish()
+        {
+            pageBuilder.finish();
+        }
+
+        @Override
+        public void close()
+        {
+            pageBuilder.close();
+        }
+    };
+}

data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/StringCast.java

@@ -0,0 +1,101 @@
+package org.embulk.filter.json_csv2arrayofobjects;
+
+import com.google.common.collect.ImmutableSet;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.DataException;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.JsonType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
+
+public class StringCast
+{
+    public static final ImmutableSet<String> TRUE_STRINGS =
+        ImmutableSet.of(
+                "true", "True", "TRUE",
+                "yes", "Yes", "YES",
+                "t", "T", "y", "Y",
+                "on", "On", "ON",
+                "1");
+
+    public static final ImmutableSet<String> FALSE_STRINGS =
+        ImmutableSet.of(
+                "false", "False", "FALSE",
+                "no", "No", "NO",
+                "f", "F", "n", "N",
+                "off", "Off", "OFF",
+                "0");
+
+    private StringCast() {}
+
+    public static String buildErrorMessage(String as, String value)
+    {
+        return String.format("Cannot cast String to %s: \"%s\"", as, value);
+    }
+
+    public static boolean asBoolean(String value)
+    {
+        if (TRUE_STRINGS.contains(value)) {
+            return true;
+        }
+        else if (FALSE_STRINGS.contains(value)) {
+            return false;
+        }
+        else {
+            throw new DataException(buildErrorMessage("boolean", value));
+        }
+    }
+
+    public static double asDouble(String value)
+    {
+        try {
+            return Double.parseDouble(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("double", value), ex);
+        }
+    }
+
+    public static long asLong(String value)
+    {
+        try {
+            return Long.parseLong(value);
+        }
+        catch (NumberFormatException ex) {
+            throw new DataException(buildErrorMessage("long", value), ex);
+        }
+    }
+
+    public static String asString(String value)
+    {
+        return value;
+    }
+
+    public static Object cast(String value, Type outputType)
+    {
+        if (outputType instanceof BooleanType) {
+            return asBoolean(value);
+        }
+        else if (outputType instanceof DoubleType) {
+            return asDouble(value);
+        }
+        else if (outputType instanceof LongType) {
+            return asLong(value);
+        }
+        else if (outputType instanceof StringType) {
+            return asString(value);
+        }
+        else if (outputType instanceof JsonType) {
+            throw new ConfigException("Casting to json is not supported.");
+        }
+        else if (outputType instanceof TimestampType) {
+            throw new ConfigException("Casting to timestamp is not supported.");
+        }
+        else {
+            throw new ConfigException(String.format("Invalid type: %s", outputType));
+        }
+    }
+}

data/src/test/java/org/embulk/filter/json_csv2arrayofobjects/TestFilter.java

@@ -0,0 +1,238 @@
+package org.embulk.filter.json_csv2arrayofobjects;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.filter.json_csv2arrayofobjects.JsonCsv2arrayofobjectsFilterPlugin.PluginTask;
+import org.embulk.spi.DataException;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.embulk.filter.json_csv2arrayofobjects.TestJsonCsv2arrayofobjectsFilterPlugin.taskFromYamlString;
+import static org.junit.Assert.assertEquals;
+
+public class TestFilter
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void oneOutputKey()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = null;
+        String got = null;
+        String expected = null;
+        // element size > 1
+        inputValue = "{\"key\": \"a,b,c\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[{\"name\":\"a\"},{\"name\":\"b\"},{\"name\":\"c\"}]}";
+        assertEquals(expected, got);
+        // element size = 1
+        inputValue = "{\"key\": \"a\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[{\"name\":\"a\"}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void multipleOutputKeys()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: long}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = null;
+        String got = null;
+        String expected = null;
+        // element size > 1
+        inputValue = "{\"key\": \"a-1,b-2,c-3\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[{\"number\":1,\"name\":\"a\"},{\"number\":2,\"name\":\"b\"},{\"number\":3,\"name\":\"c\"}]}";
+        assertEquals(expected, got);
+        // element size = 1
+        inputValue = "{\"key\": \"a-1\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[{\"number\":1,\"name\":\"a\"}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void nestedJson()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key1.key2",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: double}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key1\": {\"key2\": \"a-1.5\"}}";
+        String got = filter.doFilter(inputValue);
+        String expected = "{\"key1\":{\"key2\":[{\"number\":1.5,\"name\":\"a\"}]}}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void valueIsNull()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: double}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key\": null}";
+        String got = filter.doFilter(inputValue);
+        String expected = "{\"key\":[]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void skipEmptyElement()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: long}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = null;
+        String got = null;
+        String expected = null;
+        // value is empty
+        inputValue = "{\"key\": \"\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[]}";
+        assertEquals(expected, got);
+        // value contains empty element
+        inputValue = "{\"key\": \",a-1,,,b-2,\"}";
+        got = filter.doFilter(inputValue);
+        expected = "{\"key\":[{\"number\":1,\"name\":\"a\"},{\"number\":2,\"name\":\"b\"}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void explicitDelimiter()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "delimiter: \":\"",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: tf, type: boolean}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key\": \"a-t:b-f\"}";
+        String got = filter.doFilter(inputValue);
+        String expected = "{\"key\":[{\"tf\":true,\"name\":\"a\"},{\"tf\":false,\"name\":\"b\"}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void explicitSubDelimiter()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "sub_delimiter: \"_\"",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: long}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key\": \"a_1,b_2\"}";
+        String got = filter.doFilter(inputValue);
+        String expected = "{\"key\":[{\"number\":1,\"name\":\"a\"},{\"number\":2,\"name\":\"b\"}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test
+    public void explicitSequenceName()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "sequence_name: seq",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: long}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key\": \"a-1,b-2\"}";
+        filter.doFilter(inputValue);
+        String got = filter.doFilter(inputValue);
+        String expected = "{\"key\":[{\"number\":1,\"name\":\"a\",\"seq\":0},{\"number\":2,\"name\":\"b\",\"seq\":1}]}";
+        assertEquals(expected, got);
+    }
+
+    @Test(expected = DataException.class)
+    public void inputValueIsNull()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: double}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = null;
+        filter.doFilter(inputValue);
+    }
+
+    @Test(expected = DataException.class)
+    public void inputValueIsEmpty()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: double}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = null;
+        filter.doFilter(inputValue);
+    }
+
+    @Test(expected = DataException.class)
+    public void invalidKey()
+    {
+        PluginTask task = taskFromYamlString(
+            "type: json_csv2arrayofobjects",
+            "column: json_payload",
+            "key: key1.key2",
+            "output_keys:",
+            "  - {name: name, type: string}",
+            "  - {name: number, type: long}"
+        );
+        Filter filter = new Filter(task);
+        String inputValue = "{\"key\": \"a-1,b-2,c-3\"}";
+        filter.doFilter(inputValue);
+    }
+}