embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/config/DataSourceSerDe.java

@@ -0,0 +1,80 @@
+package org.embulk.config;
+
+import java.io.IOException;
+import com.google.inject.Inject;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class DataSourceSerDe
+{
+    public static class SerDeModule
+            extends SimpleModule
+    {
+        public SerDeModule(final ModelManager model)
+        {
+            // DataSourceImpl
+            addSerializer(DataSourceImpl.class, new DataSourceSerializer<DataSourceImpl>());
+            addDeserializer(DataSourceImpl.class, new DataSourceDeserializer<DataSourceImpl>(model));
+
+            // ConfigSource
+            addSerializer(ConfigSource.class, new DataSourceSerializer<ConfigSource>());
+            addDeserializer(ConfigSource.class, new DataSourceDeserializer<ConfigSource>(model));
+
+            // TaskSource
+            addSerializer(TaskSource.class, new DataSourceSerializer<TaskSource>());
+            addDeserializer(TaskSource.class, new DataSourceDeserializer<TaskSource>(model));
+
+            // CommitReport
+            addSerializer(CommitReport.class, new DataSourceSerializer<CommitReport>());
+            addDeserializer(CommitReport.class, new DataSourceDeserializer<CommitReport>(model));
+
+            // NextConfig
+            addSerializer(NextConfig.class, new DataSourceSerializer<NextConfig>());
+            addDeserializer(NextConfig.class, new DataSourceDeserializer<NextConfig>(model));
+        }
+    }
+
+    private static class DataSourceDeserializer <T extends DataSource>  // TODO T extends DataSource super DataSourceImpl
+            extends JsonDeserializer<T>
+    {
+        private final ModelManager model;
+        private final ObjectMapper treeObjectMapper;
+
+        DataSourceDeserializer(ModelManager model)
+        {
+            this.model = model;
+            this.treeObjectMapper = new ObjectMapper();
+        }
+
+        @Override
+        public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
+        {
+            JsonNode json = treeObjectMapper.readTree(jp);
+            if (!json.isObject()) {
+                throw new JsonMappingException("Expected object to deserialize DataSource", jp.getCurrentLocation());
+            }
+            return (T) new DataSourceImpl(model, (ObjectNode) json);
+        }
+    }
+
+    private static class DataSourceSerializer <T extends DataSource>
+            extends JsonSerializer<T>
+    {
+        @Override
+        public void serialize(T value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            value.getObjectNode().serialize(jgen, provider);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java

@@ -0,0 +1,20 @@
+package org.embulk.config;
+
+import java.lang.reflect.Type;
+import com.fasterxml.jackson.core.type.TypeReference;
+
+class GenericTypeReference
+        extends TypeReference<Object>
+{
+    private final Type type;
+
+    public GenericTypeReference(Type type)
+    {
+        this.type = type;
+    }
+
+    public Type getType()
+    {
+        return type;
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/ModelManager.java

@@ -0,0 +1,125 @@
+package org.embulk.config;
+
+import javax.validation.Validation;
+import org.apache.bval.jsr303.ApacheValidationProvider;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+public class ModelManager
+{
+    private final Injector injector;
+    private final ObjectMapper objectMapper;
+    private final ObjectMapper configObjectMapper;  // configObjectMapper uses different TaskDeserializer
+    private final TaskValidator taskValidator;
+
+    @Inject
+    public ModelManager(Injector injector, ObjectMapper objectMapper)
+    {
+        this.injector = injector;
+        this.objectMapper = objectMapper;
+        this.configObjectMapper = objectMapper.copy();
+        this.taskValidator = new TaskValidator(
+                Validation.byProvider(ApacheValidationProvider.class).configure().buildValidatorFactory().getValidator());
+
+        objectMapper.registerModule(new TaskSerDe.TaskSerializerModule(objectMapper));
+        objectMapper.registerModule(new TaskSerDe.TaskDeserializerModule(objectMapper, this));
+        objectMapper.registerModule(new DataSourceSerDe.SerDeModule(this));
+        configObjectMapper.registerModule(new TaskSerDe.TaskSerializerModule(configObjectMapper));
+        configObjectMapper.registerModule(new TaskSerDe.ConfigTaskDeserializerModule(configObjectMapper, this));
+        configObjectMapper.registerModule(new DataSourceSerDe.SerDeModule(this));
+    }
+
+    public <T> T readObject(Class<T> valueType, String json)
+    {
+        try {
+            return objectMapper.readValue(json, valueType);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public <T> T readObject(Class<T> valueType, JsonParser parser)
+    {
+        try {
+            return objectMapper.readValue(parser, valueType);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public <T> T readObjectWithConfigSerDe(Class<T> valueType, String json)
+    {
+        T t;
+        try {
+            t = configObjectMapper.readValue(json, valueType);
+        } catch (Exception ex) {
+            Throwables.propagateIfInstanceOf(ex, ConfigException.class);
+            throw new ConfigException(ex);
+        }
+        validate(t);
+        return t;
+    }
+
+    public <T> T readObjectWithConfigSerDe(Class<T> valueType, JsonParser parser)
+    {
+        T t;
+        try {
+            t = configObjectMapper.readValue(parser, valueType);
+        } catch (Exception ex) {
+            Throwables.propagateIfInstanceOf(ex, ConfigException.class);
+            throw new ConfigException(ex);
+        }
+        validate(t);
+        return t;
+    }
+
+    public String writeObject(Object object)
+    {
+        try {
+            return objectMapper.writeValueAsString(object);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public void validate(Object object)
+    {
+        taskValidator.validateModel(object);
+    }
+
+    // visible for DataSource.set
+    JsonNode writeObjectAsJsonNode(Object v)
+    {
+        String json = writeObject(v);
+        try {
+            return objectMapper.readValue(json, JsonNode.class);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    // visible for TaskInvocationHandler.invokeDump
+    ObjectNode writeObjectAsObjectNode(Object v)
+    {
+        String json = writeObject(v);
+        try {
+            return objectMapper.readValue(json, ObjectNode.class);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    // visible for TaskSerDe.set
+    // TODO create annotation calss and get its instance at the 2nd argument
+    <T> T getInjectedInstance(Class<T> type)
+    {
+        return injector.getInstance(type);
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/NextConfig.java

@@ -0,0 +1,26 @@
+package org.embulk.config;
+
+public interface NextConfig
+        extends DataSource
+{
+    @Override
+    public NextConfig getNested(String attrName);
+
+    @Override
+    public NextConfig getNestedOrSetEmpty(String attrName);
+
+    @Override
+    public NextConfig set(String attrName, Object v);
+
+    @Override
+    public NextConfig setNested(String attrName, DataSource v);
+
+    @Override
+    public NextConfig setAll(DataSource other);
+
+    @Override
+    public NextConfig deepCopy();
+
+    @Override
+    public NextConfig merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/Task.java

@@ -0,0 +1,10 @@
+package org.embulk.config;
+
+public interface Task
+{
+    public void validate();
+    // TODO public void validateConfig();
+    // ConfigSource.loadTask calls validateConfig and it validates only fields that have Config annotation
+
+    public TaskSource dump();
+}

data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java

@@ -0,0 +1,180 @@
+package org.embulk.config;
+
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationHandler;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import com.google.common.collect.ImmutableMap;
+
+class TaskInvocationHandler
+        implements InvocationHandler
+{
+    private final ModelManager model;
+    private final Class<?> iface;
+    private final Map<String, Object> objects;
+    private final Set<String> injectedFields;
+
+    public TaskInvocationHandler(ModelManager model, Class<?> iface, Map<String, Object> objects, Set<String> injectedFields)
+    {
+        this.model = model;
+        this.iface = iface;
+        this.objects = objects;
+        this.injectedFields = injectedFields;
+    }
+
+    /**
+     * fieldName = Method of the getter
+     */
+    public static Map<String, Method> fieldGetters(Class<?> iface)
+    {
+        ImmutableMap.Builder<String, Method> builder = ImmutableMap.builder();
+        for (Method method : iface.getMethods()) {
+            String methodName = method.getName();
+            String fieldName = getterFieldNameOrNull(methodName);
+            if (fieldName != null && hasExpectedArgumentLength(method, 0)) {
+                builder.put(fieldName, method);
+            }
+        }
+        return builder.build();
+    }
+
+    // visible for ModelManager.AccessorSerializer
+    Map<String, Object> getObjects()
+    {
+        return objects;
+    }
+
+    // visible for ModelManager.AccessorSerializer
+    Set<String> getInjectedFields()
+    {
+        return injectedFields;
+    }
+
+    protected Object invokeGetter(Method method, String fieldName)
+    {
+        return objects.get(fieldName);
+    }
+
+    protected void invokeSetter(Method method, String fieldName, Object value)
+    {
+        if (value == null) {
+            objects.remove(fieldName);
+        } else {
+            objects.put(fieldName, value);
+        }
+    }
+
+    private Map<String, Object> getSerializableFields()
+    {
+        Map<String, Object> data = new HashMap<String, Object>(objects);
+        for (String injected : injectedFields) {
+            data.remove(injected);
+        }
+        return data;
+    }
+
+    protected TaskSource invokeDump()
+    {
+        return new DataSourceImpl(model, model.writeObjectAsObjectNode(getSerializableFields()));
+    }
+
+    protected String invokeToString()
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append(iface.getName());
+        sb.append(getSerializableFields());
+        return sb.toString();
+    }
+
+    protected int invokeHashCode()
+    {
+        return objects.hashCode();
+    }
+
+    protected boolean invokeEquals(Object other)
+    {
+        return (other instanceof TaskInvocationHandler) &&
+            objects.equals(((TaskInvocationHandler) other).objects);
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args)
+    {
+        String methodName = method.getName();
+
+        switch(methodName) {
+        case "validate":
+            checkArgumentLength(method, 0, methodName);
+            model.validate(proxy);
+            return proxy;
+
+        case "dump":
+            checkArgumentLength(method, 0, methodName);
+            return invokeDump();
+
+        case "toString":
+            checkArgumentLength(method, 0, methodName);
+            return invokeToString();
+
+        case "hashCode":
+            checkArgumentLength(method, 0, methodName);
+            return invokeHashCode();
+
+        case "equals":
+            checkArgumentLength(method, 1, methodName);
+            if (args[0] instanceof Proxy) {
+                Object otherHandler = Proxy.getInvocationHandler(args[0]);
+                return invokeEquals(otherHandler);
+            }
+            return false;
+
+        default:
+            {
+                String fieldName;
+                fieldName = getterFieldNameOrNull(methodName);
+                if (fieldName != null) {
+                    checkArgumentLength(method, 0, methodName);
+                    return invokeGetter(method, fieldName);
+                }
+                fieldName = setterFieldNameOrNull(methodName);
+                if (fieldName != null) {
+                    checkArgumentLength(method, 1, methodName);
+                    invokeSetter(method, fieldName, args[0]);
+                    return this;
+                }
+            }
+        }
+
+        throw new IllegalArgumentException(String.format("Undefined method '%s'", methodName));
+    }
+
+    private static String getterFieldNameOrNull(String methodName)
+    {
+        if (methodName.startsWith("get")) {
+            return methodName.substring(3);
+        }
+        return null;
+    }
+
+    private static String setterFieldNameOrNull(String methodName)
+    {
+        if (methodName.startsWith("set")) {
+            return methodName.substring(3);
+        }
+        return null;
+    }
+
+    protected static boolean hasExpectedArgumentLength(Method method, int expected)
+    {
+        return method.getParameterTypes().length == expected;
+    }
+
+    protected static void checkArgumentLength(Method method, int expected, String methodName)
+    {
+        if (!hasExpectedArgumentLength(method, expected)) {
+            throw new IllegalArgumentException(
+                    String.format("Method '%s' expected %d argument but got %d arguments", methodName, expected, method.getParameterTypes().length));
+        }
+    }
+}