embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/config/GenericTypeReference.java

@@ -0,0 +1,20 @@
+package org.embulk.config;
+
+import java.lang.reflect.Type;
+import com.fasterxml.jackson.core.type.TypeReference;
+
+class GenericTypeReference
+        extends TypeReference<Object>
+{
+    private final Type type;
+
+    public GenericTypeReference(Type type)
+    {
+        this.type = type;
+    }
+
+    public Type getType()
+    {
+        return type;
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/ModelManager.java

@@ -0,0 +1,123 @@
+package org.embulk.config;
+
+import javax.validation.Validation;
+import org.apache.bval.jsr303.ApacheValidationProvider;
+import com.google.inject.Inject;
+import com.google.inject.Injector;
+import com.google.common.base.Throwables;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class ModelManager
+{
+    private final Injector injector;
+    private final ObjectMapper objectMapper;
+    private final ObjectMapper configObjectMapper;  // configObjectMapper uses different TaskDeserializer
+    private final TaskValidator taskValidator;
+
+    @Inject
+    public ModelManager(Injector injector, ObjectMapper objectMapper)
+    {
+        this.injector = injector;
+        this.objectMapper = objectMapper;
+        this.configObjectMapper = objectMapper.copy();
+        this.taskValidator = new TaskValidator(
+                Validation.byProvider(ApacheValidationProvider.class).configure().buildValidatorFactory().getValidator());
+
+        objectMapper.registerModule(new TaskSerDe.TaskSerializerModule(objectMapper));
+        objectMapper.registerModule(new TaskSerDe.TaskDeserializerModule(objectMapper, this));
+        objectMapper.registerModule(new DataSourceSerDe.SerDeModule(this));
+        configObjectMapper.registerModule(new TaskSerDe.TaskSerializerModule(configObjectMapper));
+        configObjectMapper.registerModule(new TaskSerDe.ConfigTaskDeserializerModule(configObjectMapper, this));
+        configObjectMapper.registerModule(new DataSourceSerDe.SerDeModule(this));
+    }
+
+    public <T> T readObject(Class<T> valueType, String json)
+    {
+        try {
+            return objectMapper.readValue(json, valueType);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public <T> T readObject(Class<T> valueType, JsonParser parser)
+    {
+        try {
+            return objectMapper.readValue(parser, valueType);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public <T> T readObjectWithConfigSerDe(Class<T> valueType, String json)
+    {
+        T t;
+        try {
+            t = configObjectMapper.readValue(json, valueType);
+        } catch (Exception ex) {
+            Throwables.propagateIfInstanceOf(ex, ConfigException.class);
+            throw new ConfigException(ex);
+        }
+        validate(t);
+        return t;
+    }
+
+    public <T> T readObjectWithConfigSerDe(Class<T> valueType, JsonParser parser)
+    {
+        T t;
+        try {
+            t = configObjectMapper.readValue(parser, valueType);
+        } catch (Exception ex) {
+            Throwables.propagateIfInstanceOf(ex, ConfigException.class);
+            throw new ConfigException(ex);
+        }
+        validate(t);
+        return t;
+    }
+
+    public String writeObject(Object object)
+    {
+        try {
+            return objectMapper.writeValueAsString(object);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    public void validate(Object object)
+    {
+        taskValidator.validateModel(object);
+    }
+
+    // visible for DataSource.set
+    JsonNode writeObjectAsJsonNode(Object v)
+    {
+        String json = writeObject(v);
+        try {
+            return objectMapper.readValue(json, JsonNode.class);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    // visible for TaskInvocationHandler.invokeDump
+    ObjectNode writeObjectAsObjectNode(Object v)
+    {
+        String json = writeObject(v);
+        try {
+            return objectMapper.readValue(json, ObjectNode.class);
+        } catch (Exception ex) {
+            throw Throwables.propagate(ex);
+        }
+    }
+
+    // visible for TaskSerDe.set
+    // TODO create annotation calss and get its instance at the 2nd argument
+    <T> T getInjectedInstance(Class<T> type)
+    {
+        return injector.getInstance(type);
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/Task.java

@@ -0,0 +1,10 @@
+package org.embulk.config;
+
+public interface Task
+{
+    void validate();
+    // TODO public void validateConfig();
+    // ConfigSource.loadTask calls validateConfig and it validates only fields that have Config annotation
+
+    TaskSource dump();
+}

data/embulk-core/src/main/java/org/embulk/config/TaskInvocationHandler.java

@@ -0,0 +1,180 @@
+package org.embulk.config;
+
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationHandler;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
+import com.google.common.collect.ImmutableMap;
+
+class TaskInvocationHandler
+        implements InvocationHandler
+{
+    private final ModelManager model;
+    private final Class<?> iface;
+    private final Map<String, Object> objects;
+    private final Set<String> injectedFields;
+
+    public TaskInvocationHandler(ModelManager model, Class<?> iface, Map<String, Object> objects, Set<String> injectedFields)
+    {
+        this.model = model;
+        this.iface = iface;
+        this.objects = objects;
+        this.injectedFields = injectedFields;
+    }
+
+    /**
+     * fieldName = Method of the getter
+     */
+    public static Map<String, Method> fieldGetters(Class<?> iface)
+    {
+        ImmutableMap.Builder<String, Method> builder = ImmutableMap.builder();
+        for (Method method : iface.getMethods()) {
+            String methodName = method.getName();
+            String fieldName = getterFieldNameOrNull(methodName);
+            if (fieldName != null && hasExpectedArgumentLength(method, 0)) {
+                builder.put(fieldName, method);
+            }
+        }
+        return builder.build();
+    }
+
+    // visible for ModelManager.AccessorSerializer
+    Map<String, Object> getObjects()
+    {
+        return objects;
+    }
+
+    // visible for ModelManager.AccessorSerializer
+    Set<String> getInjectedFields()
+    {
+        return injectedFields;
+    }
+
+    protected Object invokeGetter(Method method, String fieldName)
+    {
+        return objects.get(fieldName);
+    }
+
+    protected void invokeSetter(Method method, String fieldName, Object value)
+    {
+        if (value == null) {
+            objects.remove(fieldName);
+        } else {
+            objects.put(fieldName, value);
+        }
+    }
+
+    private Map<String, Object> getSerializableFields()
+    {
+        Map<String, Object> data = new HashMap<String, Object>(objects);
+        for (String injected : injectedFields) {
+            data.remove(injected);
+        }
+        return data;
+    }
+
+    protected TaskSource invokeDump()
+    {
+        return new DataSourceImpl(model, model.writeObjectAsObjectNode(getSerializableFields()));
+    }
+
+    protected String invokeToString()
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append(iface.getName());
+        sb.append(getSerializableFields());
+        return sb.toString();
+    }
+
+    protected int invokeHashCode()
+    {
+        return objects.hashCode();
+    }
+
+    protected boolean invokeEquals(Object other)
+    {
+        return (other instanceof TaskInvocationHandler) &&
+            objects.equals(((TaskInvocationHandler) other).objects);
+    }
+
+    public Object invoke(Object proxy, Method method, Object[] args)
+    {
+        String methodName = method.getName();
+
+        switch(methodName) {
+        case "validate":
+            checkArgumentLength(method, 0, methodName);
+            model.validate(proxy);
+            return proxy;
+
+        case "dump":
+            checkArgumentLength(method, 0, methodName);
+            return invokeDump();
+
+        case "toString":
+            checkArgumentLength(method, 0, methodName);
+            return invokeToString();
+
+        case "hashCode":
+            checkArgumentLength(method, 0, methodName);
+            return invokeHashCode();
+
+        case "equals":
+            checkArgumentLength(method, 1, methodName);
+            if (args[0] instanceof Proxy) {
+                Object otherHandler = Proxy.getInvocationHandler(args[0]);
+                return invokeEquals(otherHandler);
+            }
+            return false;
+
+        default:
+            {
+                String fieldName;
+                fieldName = getterFieldNameOrNull(methodName);
+                if (fieldName != null) {
+                    checkArgumentLength(method, 0, methodName);
+                    return invokeGetter(method, fieldName);
+                }
+                fieldName = setterFieldNameOrNull(methodName);
+                if (fieldName != null) {
+                    checkArgumentLength(method, 1, methodName);
+                    invokeSetter(method, fieldName, args[0]);
+                    return this;
+                }
+            }
+        }
+
+        throw new IllegalArgumentException(String.format("Undefined method '%s'", methodName));
+    }
+
+    private static String getterFieldNameOrNull(String methodName)
+    {
+        if (methodName.startsWith("get")) {
+            return methodName.substring(3);
+        }
+        return null;
+    }
+
+    private static String setterFieldNameOrNull(String methodName)
+    {
+        if (methodName.startsWith("set")) {
+            return methodName.substring(3);
+        }
+        return null;
+    }
+
+    protected static boolean hasExpectedArgumentLength(Method method, int expected)
+    {
+        return method.getParameterTypes().length == expected;
+    }
+
+    protected static void checkArgumentLength(Method method, int expected, String methodName)
+    {
+        if (!hasExpectedArgumentLength(method, expected)) {
+            throw new IllegalArgumentException(
+                    String.format("Method '%s' expected %d argument but got %d arguments", methodName, expected, method.getParameterTypes().length));
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/TaskReport.java

@@ -0,0 +1,29 @@
+package org.embulk.config;
+
+public interface TaskReport
+        extends DataSource
+{
+    @Override
+    TaskReport getNested(String attrName);
+
+    @Override
+    TaskReport getNestedOrSetEmpty(String attrName);
+
+    @Override
+    TaskReport set(String attrName, Object v);
+
+    @Override
+    TaskReport setNested(String attrName, DataSource v);
+
+    @Override
+    TaskReport setAll(DataSource other);
+
+    @Override
+    TaskReport remove(String attrName);
+
+    @Override
+    TaskReport deepCopy();
+
+    @Override
+    TaskReport merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java

@@ -0,0 +1,345 @@
+package org.embulk.config;
+
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Method;
+import java.lang.reflect.Type;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+import java.util.concurrent.ConcurrentHashMap;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableMap;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.deser.Deserializers;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.BeanDescription;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.DeserializationConfig;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+class TaskSerDe
+{
+    public static class TaskSerializer
+            extends JsonSerializer<Task>
+    {
+        private final ObjectMapper nestedObjectMapper;
+
+        public TaskSerializer(ObjectMapper nestedObjectMapper)
+        {
+            this.nestedObjectMapper = nestedObjectMapper;
+        }
+
+        @Override
+        public void serialize(Task value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            if (value instanceof Proxy) {
+                Object handler = Proxy.getInvocationHandler(value);
+                if (handler instanceof TaskInvocationHandler) {
+                    TaskInvocationHandler h = (TaskInvocationHandler) handler;
+                    Map<String, Object> objects = h.getObjects();
+                    jgen.writeStartObject();
+                    for (Map.Entry<String, Object> pair : objects.entrySet()) {
+                        if (h.getInjectedFields().contains(pair.getKey())) {
+                            continue;
+                        }
+                        jgen.writeFieldName(pair.getKey());
+                        nestedObjectMapper.writeValue(jgen, pair.getValue());
+                    }
+                    jgen.writeEndObject();
+                    return;
+                }
+            }
+            // TODO exception class & message
+            throw new UnsupportedOperationException("Serializing Task is not supported");
+        }
+    }
+
+    public static class TaskDeserializer <T>
+            extends JsonDeserializer<T>
+    {
+        private final ObjectMapper nestedObjectMapper;
+        private final ModelManager model;
+        private final Class<?> iface;
+        private final Map<String, FieldEntry> mappings;
+        private final List<InjectEntry> injects;
+
+        public TaskDeserializer(ObjectMapper nestedObjectMapper, ModelManager model, Class<T> iface)
+        {
+            this.nestedObjectMapper = nestedObjectMapper;
+            this.model = model;
+            this.iface = iface;
+            this.mappings = getterMappings(iface);
+            this.injects = injectEntries(iface);
+        }
+
+        protected Map<String, FieldEntry> getterMappings(Class<?> iface)
+        {
+            ImmutableMap.Builder<String, FieldEntry> builder = ImmutableMap.builder();
+            for (Map.Entry<String, Method> getter : TaskInvocationHandler.fieldGetters(iface).entrySet()) {
+                Method getterMethod = getter.getValue();
+                String fieldName = getter.getKey();
+
+                if (getterMethod.getAnnotation(ConfigInject.class) != null) {
+                    // InjectEntry
+                    continue;
+                }
+
+                Type fieldType = getterMethod.getGenericReturnType();
+
+                Optional<String> jsonKey = getJsonKey(getterMethod, fieldName);
+                if (!jsonKey.isPresent()) {
+                    // skip this field
+                    continue;
+                }
+                Optional<String> defaultJsonString = getDefaultJsonString(getterMethod);
+                builder.put(jsonKey.get(), new FieldEntry(fieldName, fieldType, defaultJsonString));
+            }
+            return builder.build();
+        }
+
+        protected List<InjectEntry> injectEntries(Class<?> iface)
+        {
+            ImmutableList.Builder<InjectEntry> builder = ImmutableList.builder();
+            for (Map.Entry<String, Method> getter : TaskInvocationHandler.fieldGetters(iface).entrySet()) {
+                Method getterMethod = getter.getValue();
+                String fieldName = getter.getKey();
+                ConfigInject inject = getterMethod.getAnnotation(ConfigInject.class);
+                if (inject != null) {
+                    // InjectEntry
+                    builder.add(new InjectEntry(fieldName, getterMethod.getReturnType()));
+                }
+            }
+            return builder.build();
+        }
+
+        protected Optional<String> getJsonKey(Method getterMethod, String fieldName)
+        {
+            return Optional.of(fieldName);
+        }
+
+        protected Optional<String> getDefaultJsonString(Method getterMethod)
+        {
+            return Optional.absent();
+        }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        public T deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
+        {
+            Map<String, Object> objects = new ConcurrentHashMap<String, Object>();
+            HashMap<String, FieldEntry> unusedMappings = new HashMap<>(mappings);
+
+            JsonToken current;
+            current = jp.getCurrentToken();
+            if (current == JsonToken.START_OBJECT) {
+                current = jp.nextToken();
+            }
+            for (; current != JsonToken.END_OBJECT; current = jp.nextToken()) {
+                String key = jp.getCurrentName();
+                current = jp.nextToken();
+                FieldEntry field = mappings.get(key);
+                if (field == null) {
+                    jp.skipChildren();
+                } else {
+                    Object value = nestedObjectMapper.readValue(jp, new GenericTypeReference(field.getType()));
+                    if (value == null) {
+                        throw new JsonMappingException("Setting null to a task field is not allowed. Use Optional<T> (com.google.common.base.Optional) to represent null.");
+                    }
+                    objects.put(field.getName(), value);
+                    unusedMappings.remove(key);
+                }
+            }
+
+            // set default values
+            for (Map.Entry<String, FieldEntry> unused : unusedMappings.entrySet()) {
+                FieldEntry field = unused.getValue();
+                if (field.getDefaultJsonString().isPresent()) {
+                    Object value = nestedObjectMapper.readValue(field.getDefaultJsonString().get(), new GenericTypeReference(field.getType()));
+                    if (value == null) {
+                        throw new JsonMappingException("Setting null to a task field is not allowed. Use Optional<T> (com.google.common.base.Optional) to represent null.");
+                    }
+                    objects.put(field.getName(), value);
+                } else {
+                    // required field
+                    throw new JsonMappingException("Field '"+unused.getKey()+"' is required but not set", jp.getCurrentLocation());
+                }
+            }
+
+            // inject
+            ImmutableSet.Builder<String> injectedFields = ImmutableSet.builder();
+            for (InjectEntry inject : injects) {
+                objects.put(inject.getName(), model.getInjectedInstance(inject.getType()));
+                injectedFields.add(inject.getName());
+            }
+
+            return (T) Proxy.newProxyInstance(
+                    iface.getClassLoader(), new Class<?>[] { iface },
+                    new TaskInvocationHandler(model, iface, objects, injectedFields.build()));
+        }
+
+        private static class FieldEntry
+        {
+            private final String name;
+            private final Type type;
+            private final Optional<String> defaultJsonString;
+
+            public FieldEntry(String name, Type type, Optional<String> defaultJsonString)
+            {
+                this.name = name;
+                this.type = type;
+                this.defaultJsonString = defaultJsonString;
+            }
+
+            public String getName()
+            {
+                return name;
+            }
+
+            public Type getType()
+            {
+                return type;
+            }
+
+            public Optional<String> getDefaultJsonString()
+            {
+                return defaultJsonString;
+            }
+        }
+
+        private static class InjectEntry
+        {
+            private final String name;
+            private Class<?> type;
+
+            public InjectEntry(String name, Class<?> type)
+            {
+                this.name = name;
+                this.type = type;
+            }
+
+            public String getName()
+            {
+                return name;
+            }
+
+            public Class<?> getType()
+            {
+                return type;
+            }
+        }
+    }
+
+    public static class TaskSerializerModule
+            extends SimpleModule
+    {
+        public TaskSerializerModule(ObjectMapper nestedObjectMapper)
+        {
+            super();
+            addSerializer(Task.class, new TaskSerializer(nestedObjectMapper));
+        }
+    }
+
+    public static class ConfigTaskDeserializer <T>
+            extends TaskDeserializer<T>
+    {
+        public ConfigTaskDeserializer(ObjectMapper nestedObjectMapper, ModelManager model, Class<T> iface)
+        {
+            super(nestedObjectMapper, model, iface);
+        }
+
+        @Override
+        protected Optional<String> getJsonKey(Method getterMethod, String fieldName)
+        {
+            Config a = getterMethod.getAnnotation(Config.class);
+            if (a != null) {
+                return Optional.of(a.value());
+            } else {
+                return Optional.absent();  // skip this field
+            }
+        }
+
+        @Override
+        public Optional<String> getDefaultJsonString(Method getterMethod)
+        {
+            ConfigDefault a = getterMethod.getAnnotation(ConfigDefault.class);
+            if (a != null && !a.value().isEmpty()) {
+                return Optional.of(a.value());
+            }
+            return super.getDefaultJsonString(getterMethod);
+        }
+    }
+
+    public static class TaskDeserializerModule
+            extends Module // can't use just SimpleModule, due to generic types
+    {
+        protected final ObjectMapper nestedObjectMapper;
+        protected final ModelManager model;
+
+        public TaskDeserializerModule(ObjectMapper nestedObjectMapper, ModelManager model)
+        {
+            this.nestedObjectMapper = nestedObjectMapper;
+            this.model = model;
+        }
+
+        @Override
+        public String getModuleName() { return "embulk.config.TaskSerDe"; }
+
+        @Override
+        public Version version() { return Version.unknownVersion(); }
+
+        @Override
+        public void setupModule(SetupContext context)
+        {
+            context.addDeserializers(new Deserializers.Base() {
+                @Override
+                public JsonDeserializer<?> findBeanDeserializer(JavaType type, DeserializationConfig config,
+                        BeanDescription beanDesc) throws JsonMappingException
+                {
+                    Class<?> raw = type.getRawClass();
+                    if (Task.class.isAssignableFrom(raw)) {
+                        return newTaskDeserializer(raw);
+                    }
+                    return super.findBeanDeserializer(type, config, beanDesc);
+                }
+            });
+        }
+
+        @SuppressWarnings("unchecked")
+        protected JsonDeserializer<?> newTaskDeserializer(Class<?> raw)
+        {
+            return new TaskDeserializer(nestedObjectMapper, model, raw);
+        }
+    }
+
+    public static class ConfigTaskDeserializerModule
+            extends TaskDeserializerModule
+    {
+        public ConfigTaskDeserializerModule(ObjectMapper nestedObjectMapper, ModelManager model)
+        {
+            super(nestedObjectMapper, model);
+        }
+
+        @Override
+        public String getModuleName() { return "embulk.config.ConfigTaskSerDe"; }
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected JsonDeserializer<?> newTaskDeserializer(Class<?> raw)
+        {
+            return new ConfigTaskDeserializer(nestedObjectMapper, model, raw);
+        }
+    }
+}