embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -0,0 +1,141 @@
+package org.embulk.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Properties;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import org.yaml.snakeyaml.Yaml;
+
+public class ConfigLoader
+{
+    private final ModelManager model;
+
+    @Inject
+    public ConfigLoader(ModelManager model)
+    {
+        this.model = model;
+    }
+
+    public ConfigSource newConfigSource()
+    {
+        return new DataSourceImpl(model);
+    }
+
+    public ConfigSource fromJsonString(String string)
+    {
+        JsonNode node;
+        try {
+            node = new ObjectMapper().readTree(string);
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    public ConfigSource fromJsonFile(File file) throws IOException
+    {
+        try (FileInputStream is = new FileInputStream(file)) {
+            return fromJson(is);
+        }
+    }
+
+    public ConfigSource fromJson(InputStream stream) throws IOException
+    {
+        JsonNode node = new ObjectMapper().readTree(stream);
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    public ConfigSource fromYamlString(String string)
+    {
+        JsonNode node = objectToJson(new Yaml().load(string));
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    public ConfigSource fromYamlFile(File file) throws IOException
+    {
+        try (FileInputStream stream = new FileInputStream(file)) {
+            return fromYaml(stream);
+        }
+    }
+
+    public ConfigSource fromYaml(InputStream stream) throws IOException
+    {
+        JsonNode node = objectToJson(new Yaml().load(stream));
+        validateJsonNode(node);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    private static void validateJsonNode(JsonNode node)
+    {
+        if (!node.isObject()) {
+            throw new RuntimeJsonMappingException("Expected object to load ConfigSource but got "+node);
+        }
+    }
+
+    @Deprecated
+    public ConfigSource fromJson(JsonParser parser) throws IOException
+    {
+        // TODO check parsed.isObject()
+        ObjectNode source = (ObjectNode) new ObjectMapper().readTree(parser);
+        return new DataSourceImpl(model, source);
+    }
+
+    public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
+    {
+        ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+        for (String propName : props.stringPropertyNames()) {
+            builder.put(propName, props.getProperty(propName));
+        }
+        return fromPropertiesYamlLiteral(builder.build(), keyPrefix);
+    }
+
+    public ConfigSource fromPropertiesYamlLiteral(Map<String, String> props, String keyPrefix)
+    {
+        ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
+        DataSource ds = new DataSourceImpl(model, source);
+        Yaml yaml = new Yaml();
+        for (Map.Entry<String, String> pair : props.entrySet()) {
+            if (!pair.getKey().startsWith(keyPrefix)) {
+                continue;
+            }
+            String keyName = pair.getKey().substring(keyPrefix.length());
+            Object parsedValue = yaml.load(pair.getValue());  // TODO exception handling
+            JsonNode node = objectToJson(parsedValue);
+
+            // handle "." as a map acccessor. for example:
+            // in.parser.type=csv => {"in": {"parser": {"type": "csv"}}}
+            // TODO handle "[]" as array index
+            String[] fragments = keyName.split("\\.");
+            DataSource key = ds;
+            for (int i=0; i < fragments.length - 1; i++) {
+                key = key.getNestedOrSetEmpty(fragments[i]);  // TODO exception handling
+            }
+            key.set(fragments[fragments.length - 1], node);
+        }
+        return new DataSourceImpl(model, source);
+    }
+
+    private JsonNode objectToJson(Object object)
+    {
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.readTree(objectMapper.writeValueAsString(object));
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java

@@ -0,0 +1,31 @@
+package org.embulk.config;
+
+public interface ConfigSource
+        extends DataSource
+{
+    <T> T loadConfig(Class<T> taskType);
+
+    @Override
+    ConfigSource getNested(String attrName);
+
+    @Override
+    ConfigSource getNestedOrSetEmpty(String attrName);
+
+    @Override
+    ConfigSource set(String attrName, Object v);
+
+    @Override
+    ConfigSource setNested(String attrName, DataSource v);
+
+    @Override
+    ConfigSource setAll(DataSource other);
+
+    @Override
+    ConfigSource remove(String attrName);
+
+    @Override
+    ConfigSource deepCopy();
+
+    @Override
+    ConfigSource merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/DataSource.java

@@ -0,0 +1,39 @@
+package org.embulk.config;
+
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public interface DataSource
+{
+    List<String> getAttributeNames();
+
+    Iterable<Map.Entry<String, JsonNode>> getAttributes();
+
+    boolean isEmpty();
+
+    boolean has(String attrName);
+
+    <E> E get(Class<E> type, String attrName);
+
+    <E> E get(Class<E> type, String attrName, E defaultValue);
+
+    DataSource getNested(String attrName);
+
+    DataSource getNestedOrSetEmpty(String attrName);
+
+    DataSource set(String attrName, Object v);
+
+    DataSource setNested(String attrName, DataSource v);
+
+    DataSource setAll(DataSource other);
+
+    DataSource remove(String attrName);
+
+    DataSource deepCopy();
+
+    DataSource merge(DataSource other);
+
+    ObjectNode getObjectNode();
+}

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -0,0 +1,231 @@
+package org.embulk.config;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Iterator;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+
+public class DataSourceImpl
+        implements ConfigSource, TaskSource, TaskReport, /* Deprecated */ CommitReport, ConfigDiff
+{
+    protected final ObjectNode data;
+    protected final ModelManager model;
+
+    public DataSourceImpl(ModelManager model)
+    {
+        this(model, new ObjectNode(JsonNodeFactory.instance));
+    }
+
+    // visible for DataSourceSerDe, ConfigSourceLoader and TaskInvocationHandler.dump
+    public DataSourceImpl(ModelManager model, ObjectNode data)
+    {
+        this.data = data;
+        this.model = model;
+    }
+
+    protected DataSourceImpl newInstance(ModelManager model, ObjectNode data)
+    {
+        return new DataSourceImpl(model, (ObjectNode) data);
+    }
+
+    // visible for DataSourceSerDe.DataSourceSerializer
+    @Override
+    public ObjectNode getObjectNode()
+    {
+        return data;
+    }
+
+    @Override
+    public List<String> getAttributeNames()
+    {
+        return ImmutableList.copyOf(data.fieldNames());
+    }
+
+    @Override
+    public Iterable<Map.Entry<String, JsonNode>> getAttributes()
+    {
+        return new Iterable<Map.Entry<String,JsonNode>>() {
+            public Iterator<Map.Entry<String, JsonNode>> iterator()
+            {
+                return data.fields();
+            }
+        };
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return !data.fieldNames().hasNext();
+    }
+
+    @Override
+    public boolean has(String attrName)
+    {
+        return data.has(attrName);
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            throw new ConfigException("Attribute "+attrName+" is required but not set");
+        }
+        return model.readObject(type, json.traverse());
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName, E defaultValue)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            return defaultValue;
+        }
+        return model.readObject(type, json.traverse());
+    }
+
+    @Override
+    public DataSourceImpl getNested(String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            throw new ConfigException("Attribute "+attrName+" is required but not set");
+        }
+        if (!json.isObject()) {
+            throw new ConfigException("Attribute "+attrName+" must be an object");
+        }
+        return newInstance(model, (ObjectNode) json);
+    }
+
+    @Override
+    public DataSourceImpl getNestedOrSetEmpty(String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            json = data.objectNode();
+            data.set(attrName, json);
+        } else if (!json.isObject()) {
+            throw new ConfigException("Attribute "+attrName+" must be an object");
+        }
+        return newInstance(model, (ObjectNode) json);
+    }
+
+    @Override
+    public DataSourceImpl set(String attrName, Object v)
+    {
+        if (v == null) {
+            remove(attrName);
+        } else {
+            data.set(attrName, model.writeObjectAsJsonNode(v));
+        }
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl setNested(String attrName, DataSource v)
+    {
+        data.set(attrName, v.getObjectNode());
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl setAll(DataSource other)
+    {
+        for (Map.Entry<String, JsonNode> field : other.getAttributes()) {
+            data.set(field.getKey(), field.getValue());
+        }
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl remove(String attrName)
+    {
+        data.remove(attrName);
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl deepCopy()
+    {
+        return newInstance(model, data.deepCopy());
+    }
+
+    @Override
+    public DataSourceImpl merge(DataSource other)
+    {
+        mergeJsonObject(data, other.deepCopy().getObjectNode());
+        return this;
+    }
+
+    private static void mergeJsonObject(ObjectNode src, ObjectNode other)
+    {
+        Iterator<Map.Entry<String, JsonNode>> ite = other.fields();
+        while (ite.hasNext()) {
+            Map.Entry<String, JsonNode> pair = ite.next();
+            JsonNode s = src.get(pair.getKey());
+            JsonNode v = pair.getValue();
+            if (v.isObject() && s != null && s.isObject()) {
+                mergeJsonObject((ObjectNode) s, (ObjectNode) v);
+            } else if (v.isArray() && s != null && s.isArray()) {
+                mergeJsonArray((ArrayNode) s, (ArrayNode) v);
+            } else {
+                src.replace(pair.getKey(), v);
+            }
+        }
+    }
+
+    private static void mergeJsonArray(ArrayNode src, ArrayNode other)
+    {
+        for (int i=0; i < other.size(); i++) {
+            JsonNode s = src.get(i);
+            JsonNode v = other.get(i);
+            if (s == null) {
+                src.add(v);
+            } else if (v.isObject() && s.isObject()) {
+                mergeJsonObject((ObjectNode) s, (ObjectNode) v);
+            } else if (v.isArray() && s.isArray()) {
+                mergeJsonArray((ArrayNode) s, (ArrayNode) v);
+            } else {
+                src.remove(i);
+                src.insert(i, v);
+            }
+        }
+    }
+
+    @Override
+    public <T> T loadTask(Class<T> taskType)
+    {
+        return model.readObject(taskType, data.traverse());
+    }
+
+    @Override
+    public <T> T loadConfig(Class<T> taskType)
+    {
+        return model.readObjectWithConfigSerDe(taskType, data.traverse());
+    }
+
+    @Override
+    public String toString()
+    {
+        return data.toString();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof DataSource)) {
+            return false;
+        }
+        return data.equals(((DataSource) other).getObjectNode());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return data.hashCode();
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java

@@ -0,0 +1,84 @@
+package org.embulk.config;
+
+import java.io.IOException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class DataSourceSerDe
+{
+    public static class SerDeModule
+            extends SimpleModule
+    {
+        @SuppressWarnings("deprecation")
+        public SerDeModule(final ModelManager model)
+        {
+            // DataSourceImpl
+            addSerializer(DataSourceImpl.class, new DataSourceSerializer<DataSourceImpl>());
+            addDeserializer(DataSourceImpl.class, new DataSourceDeserializer<DataSourceImpl>(model));
+
+            // ConfigSource
+            addSerializer(ConfigSource.class, new DataSourceSerializer<ConfigSource>());
+            addDeserializer(ConfigSource.class, new DataSourceDeserializer<ConfigSource>(model));
+
+            // TaskSource
+            addSerializer(TaskSource.class, new DataSourceSerializer<TaskSource>());
+            addDeserializer(TaskSource.class, new DataSourceDeserializer<TaskSource>(model));
+
+            // TaskReport
+            addSerializer(TaskReport.class, new DataSourceSerializer<TaskReport>());
+            addDeserializer(TaskReport.class, new DataSourceDeserializer<TaskReport>(model));
+
+            // CommitReport (Deprecated)
+            addSerializer(CommitReport.class, new DataSourceSerializer<CommitReport>());
+            addDeserializer(CommitReport.class, new DataSourceDeserializer<CommitReport>(model));
+
+            // ConfigDiff
+            addSerializer(ConfigDiff.class, new DataSourceSerializer<ConfigDiff>());
+            addDeserializer(ConfigDiff.class, new DataSourceDeserializer<ConfigDiff>(model));
+        }
+    }
+
+    private static class DataSourceDeserializer <T extends DataSource>  // TODO T extends DataSource super DataSourceImpl
+            extends JsonDeserializer<T>
+    {
+        private final ModelManager model;
+        private final ObjectMapper treeObjectMapper;
+
+        DataSourceDeserializer(ModelManager model)
+        {
+            this.model = model;
+            this.treeObjectMapper = new ObjectMapper();
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
+        {
+            JsonNode json = treeObjectMapper.readTree(jp);
+            if (!json.isObject()) {
+                throw new JsonMappingException("Expected object to deserialize DataSource", jp.getCurrentLocation());
+            }
+            return (T) new DataSourceImpl(model, (ObjectNode) json);
+        }
+    }
+
+    private static class DataSourceSerializer <T extends DataSource>
+            extends JsonSerializer<T>
+    {
+        @Override
+        public void serialize(T value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            value.getObjectNode().serialize(jgen, provider);
+        }
+    }
+}