embulk-input-singer_tap 0.1.0

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/input/singer_tap.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  "singer_tap", "org.embulk.input.singer_tap.SingerTapInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/singer_tap/ArrayParser.java

@@ -0,0 +1,31 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ArrayParser implements  RecordParser {
+    private RecordParser elementParser;
+
+    public ArrayParser(JsonNode node) throws Exception {
+        elementParser = ParserGenerator.generateParser(node);
+    }
+
+    public Value parse(JsonNode node) {
+        ArrayNode arrNode = (ArrayNode) node;
+        List<Value> list = new ArrayList<>();
+        for (JsonNode element : arrNode) {
+            list.add(elementParser.parse(element));
+        }
+
+        return ValueFactory.newArray(list);
+    }
+
+    public Type embulkType() { return Types.JSON; }
+}

data/src/main/java/org/embulk/input/singer_tap/BooleanParser.java

@@ -0,0 +1,16 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class BooleanParser implements RecordParser {
+    @Override
+    public Value parse(JsonNode node) {
+        return ValueFactory.newBoolean(node.asBoolean());
+    }
+
+    public Type embulkType() { return Types.BOOLEAN; }
+}

data/src/main/java/org/embulk/input/singer_tap/IntegerParser.java

@@ -0,0 +1,16 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class IntegerParser implements RecordParser {
+    @Override
+    public Value parse(JsonNode node) {
+        return ValueFactory.newInteger(node.asLong());
+    }
+
+    public Type embulkType() { return Types.LONG; }
+}

data/src/main/java/org/embulk/input/singer_tap/NullableArrayParser.java

@@ -0,0 +1,19 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableArrayParser extends ArrayParser {
+    public NullableArrayParser(JsonNode node) throws Exception {
+        super(node);
+    }
+
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return super.parse(node);
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/NullableBooleanParser.java

@@ -0,0 +1,15 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableBooleanParser extends BooleanParser {
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return ValueFactory.newBoolean(node.asBoolean());
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/NullableIntegerParser.java

@@ -0,0 +1,15 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableIntegerParser extends IntegerParser {
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return ValueFactory.newInteger(node.asLong());
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/NullableNumberParser.java

@@ -0,0 +1,15 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableNumberParser extends NumberParser {
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return ValueFactory.newFloat(node.asDouble());
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/NullableObjectParser.java

@@ -0,0 +1,21 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableObjectParser extends ObjectParser {
+    public NullableObjectParser(ObjectNode node) throws Exception {
+        super(node);
+    }
+
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return super.parse(node);
+    }
+
+}

data/src/main/java/org/embulk/input/singer_tap/NullableStringParser.java

@@ -0,0 +1,15 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NullableStringParser extends StringParser {
+    @Override
+    public Value parse(JsonNode node) {
+        if (node.isNull()) {
+            return ValueFactory.newNil();
+        }
+        return ValueFactory.newString(node.asText());
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/NumberParser.java

@@ -0,0 +1,16 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class NumberParser implements RecordParser {
+    @Override
+    public Value parse(JsonNode node) {
+        return ValueFactory.newFloat(node.asDouble());
+    }
+
+    public Type embulkType() { return Types.DOUBLE; }
+}

data/src/main/java/org/embulk/input/singer_tap/ObjectParser.java

@@ -0,0 +1,52 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+public class ObjectParser implements RecordParser {
+    private Map<String, RecordParser> fieldParser;
+
+    public ObjectParser(ObjectNode node) throws Exception {
+        fieldParser = new LinkedHashMap<>();
+        for (Iterator<Map.Entry<String, JsonNode>> it = node.fields(); it.hasNext(); ) {
+            Map.Entry<String, JsonNode> e = it.next();
+            fieldParser.put(e.getKey(), ParserGenerator.generateParser(e.getValue()));
+        }
+    }
+
+    @Override
+    public Value parse(JsonNode node) {
+        ObjectNode objNode = (ObjectNode) node;
+        Map<Value, Value> map = new HashMap<>();
+
+        for (Map.Entry<String, RecordParser> entry : fieldParser.entrySet()) {
+            String key = entry.getKey();
+            JsonNode valNode = objNode.get(key);
+            Value val;
+            if (valNode == null) {
+                val = ValueFactory.newNil();
+            }
+            else {
+                val = fieldParser.get(key).parse(valNode);
+            }
+            map.put(ValueFactory.newString(key), val);
+        }
+
+        return ValueFactory.newMap(map);
+    }
+
+    public Map<String, RecordParser> properties() {
+        return fieldParser;
+    }
+
+    public Type embulkType() { return Types.JSON; }
+}

data/src/main/java/org/embulk/input/singer_tap/ParserGenerator.java

@@ -0,0 +1,79 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ParserGenerator {
+    public static RecordParser generateParser(JsonNode node) throws Exception {
+        String type;
+        Boolean nullable = false;
+        JsonNode type_node = node.get("type");
+        if (type_node.isTextual()) {
+            type = type_node.asText();
+        }
+        else if (type_node.isArray()) {
+            ArrayNode types = (ArrayNode) type_node;
+            int len = types.size();
+            if (len == 2) {
+                String type_0 = types.get(0).asText();
+                String type_1 = types.get(1).asText();
+                if (type_0.equals("null")) {
+                    nullable = true;
+                    type = type_1;
+                }
+                else if (type_1.equals("null")) {
+                    nullable = true;
+                    type = type_0;
+                }
+                else {
+                    type = "undefined";
+                }
+            }
+            else if (len == 1){
+                type = types.get(0).asText();
+            }
+            else {
+                throw new Exception("number of possible types must be 1 or 2!");
+            }
+        }
+        else {
+            throw new Exception("invalid type!");
+        }
+
+        if (nullable) {
+            switch (type) {
+                case "string":
+                    return new NullableStringParser();
+                case "number":
+                    return new NumberParser();
+                case "integer":
+                    return new NullableIntegerParser();
+                case "boolean":
+                    return new NullableBooleanParser();
+                case "object":
+                    return new NullableObjectParser((ObjectNode) (node.get("properties")));
+                case "array":
+                    return new NullableArrayParser(node.get("items"));
+                default:
+                    throw new Exception("Undefined type");
+            }
+        }
+        switch (type) {
+            case "string":
+                return new StringParser();
+            case "number":
+                return new NumberParser();
+            case "integer":
+                return new IntegerParser();
+            case "boolean":
+                return new BooleanParser();
+            case "object":
+                return new ObjectParser((ObjectNode) (node.get("properties")));
+            case "array":
+                return new ArrayParser(node.get("items"));
+            default:
+                throw new Exception("Undefined type");
+        }
+    }
+}

data/src/main/java/org/embulk/input/singer_tap/RecordParser.java

@@ -0,0 +1,11 @@
+package org.embulk.input.singer_tap;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.spi.type.Type;
+import org.msgpack.value.Value;
+
+
+public interface RecordParser {
+    Value parse(JsonNode node);
+    Type embulkType();
+}

data/src/main/java/org/embulk/input/singer_tap/SingerTapInputPlugin.java

@@ -0,0 +1,265 @@
+package org.embulk.input.singer_tap;
+
+import java.io.*;
+import java.util.*;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.embulk.config.*;
+import org.embulk.spi.*;
+import org.msgpack.value.Value;
+
+
+public class SingerTapInputPlugin
+        implements InputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("tap_command")
+        public String getTapCommand();
+
+        @Config("config")
+        public String getConfig();
+
+        @Config("catalog")
+        @ConfigDefault("null")
+        public Optional<String> getCatalog();
+
+        @Config("properties")
+        @ConfigDefault("null")
+        public Optional<String> getProperties();
+
+        @Config("input_state")
+        @ConfigDefault("null")
+        public Optional<String> getInputState();
+
+        @Config("output_state")
+        @ConfigDefault("null")
+        public Optional<String> getOutputState();
+
+        String getSchemaFileName();
+        void setSchemaFileName(String schemaFile);
+
+        List<String> getCommandLine();
+        void setCommandLine(List<String> commandLine);
+
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        String command = task.getTapCommand() + " --config " + task.getConfig();
+
+        File schemaFile;
+        if (task.getCatalog().isPresent()) {
+            if (task.getProperties().isPresent()) {
+                throw new ConfigException("only one of 'catalog' or 'properties' parameter is needed");
+            }
+            String schemaFileName = task.getCatalog().get();
+            schemaFile = new File(schemaFileName);
+            task.setSchemaFileName(schemaFileName);
+            command = command + " --catalog " + schemaFileName;
+        }
+        else {
+            if (!task.getProperties().isPresent()) {
+                throw new ConfigException("'catalog' or 'properties' parameter is needed");
+            }
+            String schemaFileName = task.getProperties().get();
+            schemaFile = new File(schemaFileName);
+            task.setSchemaFileName(schemaFileName);
+            command = command + " --properties " + schemaFileName;
+        }
+        Schema schema = generateSchema(schemaFile);
+
+        if (task.getInputState().isPresent()) {
+            String stateFileName = task.getInputState().get();
+            command = command + " --state " + stateFileName;
+        }
+
+        List<String> cmdline = new ArrayList<>(Arrays.asList("sh", "-c"));
+        cmdline.add(command);
+        task.setCommandLine(cmdline);
+
+        int taskCount = 1;  // number of run() method calls
+
+        return resume(task.dump(), schema, taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        control.run(taskSource, schema, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        BufferAllocator allocator = task.getBufferAllocator();
+
+        RecordParser parser;
+        try {
+            JsonNode schemaNode = getSchema(new File(task.getSchemaFileName()));
+            parser = ParserGenerator.generateParser(schemaNode);
+        }
+        catch (Exception e) {
+            throw new ConfigException(e.getMessage());
+        }
+
+        List<String> cmdline = task.getCommandLine();
+        ProcessBuilder pb = new ProcessBuilder(cmdline);
+        pb.redirectError(ProcessBuilder.Redirect.INHERIT);
+        String state = "";
+        try {
+            Process process = pb.start();
+            InputStream stream = process.getInputStream();
+            try (BufferedReader br = new BufferedReader(new InputStreamReader(stream))) {
+                String line;
+                JsonNode root;
+                PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);;
+                ObjectMapper recordMapper = new ObjectMapper();
+                while ((line = br.readLine()) != null) {
+                    root = recordMapper.readTree(line);
+                    String type = root.get("type").asText();
+                    if (type.equals("RECORD")) {
+                        JsonNode recordNode = root.get("record");
+                        Value record = parser.parse(recordNode);
+                        if (!(record.isMapValue())) {
+                            throw new DataException("invalid record");
+                        }
+                        Map<String, Value> rec = new HashMap<>();
+                        for (Map.Entry<Value, Value> entry : record.asMapValue().entrySet()) {
+                            rec.put(entry.getKey().asStringValue().asString(), entry.getValue());
+                        }
+                        for (Column column : pageBuilder.getSchema().getColumns()) {
+                            setColumn(column, rec, pageBuilder);
+                        }
+                        pageBuilder.addRecord();
+                    }
+                    else if (type.equals("SCHEMA")) {
+
+                    }
+                    else if (type.equals("STATE")) {
+                        state = root.get("value").toString();
+                    }
+                    else {
+                        throw new DataException("invalid message type: " + type);
+                    }
+                }
+                pageBuilder.finish();
+            }
+        }
+        catch (IOException e) {
+            throw new DataException(e.getMessage());
+        }
+        finally {
+            if (task.getOutputState().isPresent()) {
+                String statePath = task.getOutputState().get();
+                try (FileOutputStream writer = new FileOutputStream(statePath)) {
+                    writer.write(state.getBytes());
+                }
+                catch (Exception e) {
+                    throw new DataException(e.getMessage());
+                }
+            }
+        }
+
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return Exec.newConfigDiff();
+    }
+
+    private JsonNode getSchema(File catalog) {
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            JsonNode streams = mapper.readTree(catalog).get("streams");
+            JsonNode schemaNode = null;
+            for (JsonNode stream : streams) {
+                JsonNode schema = stream.get("schema");
+                JsonNode selected = schema.get("selected");
+                if (selected != null && selected.asBoolean()) {
+                    schemaNode = schema;
+                    break;
+                }
+            }
+            if (schemaNode == null) {
+                throw new ConfigException("schema not selected");
+            }
+            return schemaNode;
+        }
+        catch (Exception e) {
+            throw new ConfigException(e.getMessage());
+        }
+    }
+
+    private Schema generateSchema(File catalog) throws ConfigException {
+        Schema.Builder builder = Schema.builder();
+        try {
+            JsonNode schemaNode = getSchema(catalog);
+            RecordParser parser = ParserGenerator.generateParser(schemaNode);
+            if (!(parser instanceof ObjectParser)) {
+                throw new DataException("invalid schema");
+            }
+            ObjectParser toplevelParser = (ObjectParser) parser;
+            for (Map.Entry<String, RecordParser> entry : toplevelParser.properties().entrySet()) {
+                String colName = entry.getKey();
+                RecordParser colParser = entry.getValue();
+                builder.add(colName, colParser.embulkType());
+            }
+        }
+        catch (Exception e) {
+            throw new ConfigException(e.getMessage());
+        }
+        return builder.build();
+    }
+
+    private void setColumn(Column column, Map<String, Value> mapValue, PageBuilder pageBuilder) {
+        String key = column.getName();
+        Value val = mapValue.get(key);
+        if (val.isStringValue()) {
+            pageBuilder.setString(column, val.asStringValue().asString());
+        }
+        else if (val.isIntegerValue()) {
+            pageBuilder.setLong(column, val.asIntegerValue().asLong());
+        }
+        else if (val.isFloatValue()) {
+            pageBuilder.setDouble(column, val.asFloatValue().toDouble());
+        }
+        else if (val.isBooleanValue()) {
+            pageBuilder.setBoolean(column, val.asBooleanValue().getBoolean());
+        }
+        else if (val.isMapValue() || val.isArrayValue()) {
+            pageBuilder.setJson(column, val);
+        }
+        else if (val.isNilValue()) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            throw new DataException("invalid type of record for column: " + key + ": "+ val.getValueType());
+        }
+    }
+}