embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java

@@ -0,0 +1,113 @@
+package org.embulk.spi.unit;
+
+import java.io.File;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class LocalFileSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(LocalFile.class, new LocalFileSerializer());
+        module.addDeserializer(LocalFile.class, new LocalFileDeserializer());
+        mapper.registerModule(module);
+    }
+
+    private static class LocalFileSerializer
+            extends JsonSerializer<LocalFile>
+    {
+        @Override
+        public void serialize(LocalFile value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeStartObject();
+            jgen.writeFieldName("base64");
+            jgen.writeBinary(value.getContent());
+            jgen.writeEndObject();
+        }
+    }
+
+    private static class LocalFileDeserializer
+            extends JsonDeserializer<LocalFile>
+    {
+        private final File tempDir;
+
+        public LocalFileDeserializer()
+        {
+            this(new File(System.getProperty("java.io.tmpdir")));
+        }
+
+        public LocalFileDeserializer(File tempDir)
+        {
+            this.tempDir = tempDir;
+        }
+
+        @Override
+        public LocalFile deserialize(JsonParser jp, DeserializationContext ctxt) throws IOException
+        {
+            JsonToken t = jp.getCurrentToken();
+            if (t == JsonToken.START_OBJECT) {
+                t = jp.nextToken();
+            }
+
+            switch(t) {
+            case VALUE_NULL:
+                return null;
+
+            case FIELD_NAME:
+                {
+                    LocalFile result;
+
+                    String keyName = jp.getCurrentName();
+                    if ("content".equals(keyName)) {
+                        jp.nextToken();
+                        result = LocalFile.ofContent(jp.getValueAsString());
+                    } else if ("base64".equals(keyName)) {
+                        jp.nextToken();
+                        ByteArrayOutputStream out = new ByteArrayOutputStream();
+                        jp.readBinaryValue(ctxt.getBase64Variant(), out);
+                        result = LocalFile.ofContent(out.toByteArray());
+                    } else {
+                        throw ctxt.mappingException("Unknown key '"+keyName+"' to deserialize LocalFile");
+                    }
+
+                    t = jp.nextToken();
+                    if (t != JsonToken.END_OBJECT) {
+                        throw ctxt.mappingException("Unexpected extra map keys to LocalFile");
+                    }
+                    return result;
+                }
+
+            case END_OBJECT:
+            case START_ARRAY:
+            case END_ARRAY:
+                throw ctxt.mappingException("Attempted unexpected map or array to LocalFile");
+
+            case VALUE_EMBEDDED_OBJECT:
+                {
+                    Object obj = jp.getEmbeddedObject();
+                    if (obj == null) {
+                        return null;
+                    }
+                    if (LocalFile.class.isAssignableFrom(obj.getClass())) {
+                        return (LocalFile) obj;
+                    }
+                    throw ctxt.mappingException("Don't know how to convert embedded Object of type "+obj.getClass().getName()+" into LocalFile");
+                }
+
+            default:
+                return LocalFile.of(jp.getValueAsString());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/ToString.java

@@ -0,0 +1,54 @@
+package org.embulk.spi.unit;
+
+import com.google.common.base.Optional;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.node.NullNode;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class ToString
+{
+    private final String string;
+
+    public ToString(String string)
+    {
+        this.string = string;
+    }
+
+    @JsonCreator
+    public ToString(Optional<JsonNode> option) throws JsonMappingException
+    {
+        JsonNode node = option.or(NullNode.getInstance());
+        if (node.isTextual()) {
+            this.string = node.textValue();
+        } else if (node.isValueNode()) {
+            this.string = node.toString();
+        } else {
+            throw new JsonMappingException(String.format("Arrays and objects are invalid: '%s'", node));
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (!(obj instanceof ToString)) {
+            return false;
+        }
+        ToString o = (ToString) obj;
+        return string.equals(o.string);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return string.hashCode();
+    }
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        return string;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java

@@ -0,0 +1,34 @@
+package org.embulk.spi.unit;
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Properties;
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+public class ToStringMap
+        extends HashMap<String, String>
+{
+    @JsonCreator
+    public ToStringMap(Map<String, ToString> map)
+    {
+        super(Maps.transformValues(map, new Function<ToString, String>() {
+            public String apply(ToString value)
+            {
+                if (value == null) {
+                    return "null";
+                } else {
+                    return value.toString();
+                }
+            }
+        }));
+    }
+
+    public Properties toProperties()
+    {
+        Properties props = new Properties();
+        props.putAll(this);
+        return props;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java

@@ -0,0 +1,55 @@
+package org.embulk.spi.util;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class CharsetSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(Charset.class, new CharsetSerializer());
+        module.addDeserializer(Charset.class, new CharsetDeserializer());
+        mapper.registerModule(module);
+    }
+
+    public static class CharsetSerializer
+            extends JsonSerializer<Charset>
+    {
+        @Override
+        public void serialize(Charset value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeString(value.name());
+        }
+    }
+
+    public static class CharsetDeserializer
+            extends FromStringDeserializer<Charset>
+    {
+        public CharsetDeserializer()
+        {
+            super(Charset.class);
+        }
+
+        @Override
+        protected Charset _deserialize(String value, DeserializationContext context)
+                throws JsonMappingException
+        {
+            try {
+                return Charset.forName(value);
+            } catch (UnsupportedOperationException ex) {
+                // TODO include link to a document to the message for the list of supported time zones
+                throw new JsonMappingException(String.format("Unknown charset '%s'", value));
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java

@@ -0,0 +1,81 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.DecoderPlugin;
+
+public abstract class Decoders
+{
+    private Decoders() { }
+
+    public static List<DecoderPlugin> newDecoderPlugins(ExecSession exec, List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<DecoderPlugin> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(exec.newPlugin(DecoderPlugin.class, config.get(PluginType.class, "type")));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources);
+    }
+
+    public static void transaction(List<DecoderPlugin> plugins, List<ConfigSource> configs,
+            Decoders.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction();
+    }
+
+    public static FileInput open(List<DecoderPlugin> plugins, List<TaskSource> taskSources,
+            FileInput input)
+    {
+        FileInput in = input;
+        int pos = 0;
+        while (pos < plugins.size()) {
+            in = plugins.get(pos).open(taskSources.get(pos), in);
+            pos++;
+        }
+        return in;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<DecoderPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Decoders.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private int pos;
+
+        RecursiveControl(List<DecoderPlugin> plugins, List<ConfigSource> configs,
+                Decoders.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+        }
+
+        public void transaction()
+        {
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), new DecoderPlugin.Control() {
+                    public void run(TaskSource taskSource)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction();
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java

@@ -0,0 +1,10 @@
+package org.embulk.spi.util;
+
+public class DynamicColumnNotFoundException
+        extends RuntimeException
+{
+    public DynamicColumnNotFoundException(String message)
+    {
+        super(message);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java

@@ -0,0 +1,18 @@
+package org.embulk.spi.util;
+
+import org.embulk.spi.time.Timestamp;
+
+public interface DynamicColumnSetter
+{
+    void setNull();
+
+    void set(boolean value);
+
+    void set(long value);
+
+    void set(double value);
+
+    void set(String value);
+
+    void set(Timestamp value);
+}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java

@@ -0,0 +1,94 @@
+package org.embulk.spi.util;
+
+import org.joda.time.DateTimeZone;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.util.dynamic.BooleanColumnSetter;
+import org.embulk.spi.util.dynamic.LongColumnSetter;
+import org.embulk.spi.util.dynamic.DoubleColumnSetter;
+import org.embulk.spi.util.dynamic.StringColumnSetter;
+import org.embulk.spi.util.dynamic.TimestampColumnSetter;
+import org.embulk.spi.util.dynamic.DefaultValueSetter;
+import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampFormat;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.config.ConfigException;
+
+public class DynamicColumnSetterFactory
+{
+    private final DefaultValueSetter defaultValue;
+    private final DynamicPageBuilder.BuilderTask task;
+
+    DynamicColumnSetterFactory(
+            DynamicPageBuilder.BuilderTask task,
+            DefaultValueSetter defaultValue)
+    {
+        this.defaultValue = defaultValue;
+        this.task = task;
+    }
+
+    public static DefaultValueSetter nullDefaultValue()
+    {
+        return new NullDefaultValueSetter();
+    }
+
+    public DynamicColumnSetter newColumnSetter(PageBuilder pageBuilder, Column column)
+    {
+        Type type = column.getType();
+        if (type instanceof BooleanType) {
+            return new BooleanColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof LongType) {
+            return new LongColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof DoubleType) {
+            return new DoubleColumnSetter(pageBuilder, column, defaultValue);
+        } else if (type instanceof StringType) {
+            TimestampFormatter formatter = new TimestampFormatter(task.getJRuby(),
+                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+            return new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
+        } else if (type instanceof TimestampType) {
+            // TODO use flexible time format like Ruby's Time.parse
+            TimestampParser parser = new TimestampParser(task.getJRuby(),
+                    getTimestampFormat(column).getFormat(), getTimeZone(column));
+            return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
+        }
+        throw new ConfigException("Unknown column type: "+type);
+    }
+
+    private TimestampFormat getTimestampFormat(Column column)
+    {
+        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
+        if (option != null) {
+            return option.getTimestampFormat();
+        } else {
+            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%6N");
+        }
+    }
+
+    private DateTimeZone getTimeZone(Column column)
+    {
+        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
+        if (option != null) {
+            return option.getTimeZone().or(task.getDefaultTimeZone());
+        } else {
+            return task.getDefaultTimeZone();
+        }
+    }
+
+    private DynamicPageBuilder.ColumnOption getColumnOption(Column column)
+    {
+        ConfigSource option = task.getColumnOptions().get(column.getName());
+        if (option != null) {
+            return option.loadConfig(DynamicPageBuilder.ColumnOption.class);
+        } else {
+            return null;
+        }
+    }
+}