embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -0,0 +1,98 @@
+package org.embulk.spi.time;
+
+import java.util.Set;
+import java.util.Date;
+import java.text.SimpleDateFormat;
+import java.text.ParsePosition;
+import com.google.common.collect.ImmutableSet;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+
+public class TimestampFormat
+{
+    private final String format;
+
+    @JsonCreator
+    public TimestampFormat(String format)
+    {
+        this.format = format;
+    }
+
+    @JsonValue
+    public String getFormat()
+    {
+        return format;
+    }
+
+    public TimestampFormatter newFormatter(TimestampFormatter.FormatterTask task)
+    {
+        return new TimestampFormatter(format, task);
+    }
+
+    public TimestampParser newParser(TimestampParser.ParserTask task)
+    {
+        return new TimestampParser(format, task);
+    }
+
+    private static Set<String> availableTimeZoneNames = ImmutableSet.copyOf(DateTimeZone.getAvailableIDs());
+
+    public static DateTimeZone parseDateTimeZone(String s)
+    {
+        if(s.startsWith("+") || s.startsWith("-")) {
+            return DateTimeZone.forID("GMT"+s);
+
+        } else {
+            try {
+                int rawOffset = (int) DateTimeFormat.forPattern("z").parseMillis(s);
+                if(rawOffset == 0) {
+                    return DateTimeZone.UTC;
+                }
+                int offset = rawOffset / -1000;
+                int h = offset / 3600;
+                int m = offset % 3600;
+                return DateTimeZone.forOffsetHoursMinutes(h, m);
+            } catch (IllegalArgumentException ex) {
+                // parseMillis failed
+            }
+
+            // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+            // we want to only return timezone if exact match, otherwise exception
+            if (availableTimeZoneNames.contains(s)) {
+                //return TimeZone.getTimeZone(s);
+                return DateTimeZone.forID(s);
+            }
+            return null;
+        }
+    }
+
+    //// Java standard TimeZone
+    //static TimeZone parseDateTimeZone(String s)
+    //{
+    //    if(s.startsWith("+") || s.startsWith("-")) {
+    //        return TimeZone.getTimeZone("GMT"+s);
+    //
+    //    } else {
+    //        ParsePosition pp = new ParsePosition(0);
+    //        Date off = new SimpleDateFormat("z").parse(s, pp);
+    //        if(off != null && pp.getErrorIndex() == -1) {
+    //            int rawOffset = (int) off.getTime();
+    //            if(rawOffset == 0) {
+    //                return TimeZone.UTC;
+    //            }
+    //            int offset = rawOffset / -1000;
+    //            int h = offset / 3600;
+    //            int m = offset % 3600;
+    //            return DateTimeZone.getTimeZone(String.format("GMT%+02d%02d", h, m));
+    //        }
+    //
+    //        // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+    //        // we want to only return timezone if exact match, otherwise exception
+    //        if (availableTimeZoneNames.contains(s)) {
+    //            return TimeZone.getTimeZone(s);
+    //        }
+    //        return null;
+    //    }
+    //}
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -0,0 +1,55 @@
+package org.embulk.spi.time;
+
+import java.util.Locale;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.util.RubyDateFormat;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.util.LineEncoder;
+
+public class TimestampFormatter
+{
+    public interface FormatterTask
+            extends Task
+    {
+        @Config("timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getTimeZone();
+
+        @JacksonInject
+        public ScriptingContainer getJRuby();
+    }
+
+    private final RubyDateFormat dateFormat;
+    private final DateTimeZone timeZone;
+
+    public TimestampFormatter(String format, FormatterTask task)
+    {
+        this(task.getJRuby(), format, task.getTimeZone());
+    }
+
+    public TimestampFormatter(ScriptingContainer jruby, String format, DateTimeZone timeZone)
+    {
+        this.timeZone = timeZone;
+        this.dateFormat = new RubyDateFormat(format, Locale.ENGLISH, true);
+    }
+
+    public void format(Timestamp value, LineEncoder encoder)
+    {
+        // TODO optimize by directly appending to internal buffer
+        encoder.addText(format(value));
+    }
+
+    public String format(Timestamp value)
+    {
+        // TODO optimize by using reused StringBuilder
+        dateFormat.setDateTime(new DateTime(value.toEpochMilli(), timeZone));
+        dateFormat.setNSec(value.getNano());
+        return dateFormat.format(null);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java

@@ -0,0 +1,6 @@
+package org.embulk.spi.time;
+
+public class TimestampParseException
+        extends Exception
+{
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -0,0 +1,60 @@
+package org.embulk.spi.time;
+
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
+
+public class TimestampParser
+{
+    public interface ParserTask
+            extends Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @JacksonInject
+        public ScriptingContainer getJRuby();
+    }
+
+    private final JRubyTimeParserHelper helper;
+    private final DateTimeZone defaultTimeZone;
+
+    public TimestampParser(String format, ParserTask task)
+    {
+        this(task.getJRuby(), format, task.getDefaultTimeZone());
+    }
+
+    // TODO this is still private because this might need current time
+    private TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
+    {
+        JRubyTimeParserHelperFactory helperFactory = (JRubyTimeParserHelperFactory) jruby.runScriptlet("Embulk::Java::TimeParserHelper::Factory.new");
+        // TODO get default current time from ExecTask.getExecTimestamp
+        this.helper = (JRubyTimeParserHelper) helperFactory.newInstance(format, 1970, 1, 1, 0, 0, 0, 0);  // TODO default time zone
+        this.defaultTimeZone = defaultTimeZone;
+    }
+
+    public Timestamp parse(String text) throws TimestampParseException
+    {
+        long localMillis = helper.strptime(text);
+        String zone = helper.getZone();
+
+        DateTimeZone timeZone = defaultTimeZone;
+        if (zone != null) {
+            // TODO cache parsed zone?
+            timeZone = parseDateTimeZone(zone);
+            if (timeZone == null) {
+                throw new TimestampParseException();
+            }
+        }
+
+        long milli = timeZone.convertLocalToUTC(localMillis, false);
+
+        return Timestamp.ofEpochMilli(milli);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java

@@ -0,0 +1,50 @@
+package org.embulk.spi.time;
+
+import java.io.IOException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class TimestampSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(Timestamp.class, new DateTimeZoneSerializer());
+        module.addDeserializer(Timestamp.class, new DateTimeZoneDeserializer());
+        mapper.registerModule(module);
+    }
+
+    public static class DateTimeZoneSerializer
+            extends JsonSerializer<Timestamp>
+    {
+        @Override
+        public void serialize(Timestamp value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeString(value.toString());
+        }
+    }
+
+    public static class DateTimeZoneDeserializer
+            extends FromStringDeserializer<Timestamp>
+    {
+        public DateTimeZoneDeserializer()
+        {
+            super(Timestamp.class);
+        }
+
+        @Override
+        protected Timestamp _deserialize(String value, DeserializationContext context)
+                throws JsonMappingException
+        {
+            return Timestamp.fromString(value);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java

@@ -0,0 +1,55 @@
+package org.embulk.spi.type;
+
+public abstract class AbstractType
+        implements Type
+{
+    private final String name;
+    private final Class<?> javaType;
+    private byte fixedStorageSize;
+
+    protected AbstractType(String name, Class<?> javaType, int fixedStorageSize)
+    {
+        this.name = name;
+        this.javaType = javaType;
+        this.fixedStorageSize = (byte) fixedStorageSize;
+    }
+
+    @Override
+    public String getName()
+    {
+        return name;
+    }
+
+    @Override
+    public Class<?> getJavaType()
+    {
+        return javaType;
+    }
+
+    @Override
+    public byte getFixedStorageSize()
+    {
+        return fixedStorageSize;
+    }
+
+    @Override
+    public boolean equals(Object o)
+    {
+        if (o == null) {
+            return false;
+        }
+        return o.getClass().isAssignableFrom(getClass());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return getClass().hashCode();
+    }
+
+    @Override
+    public String toString()
+    {
+        return name;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class BooleanType
+        extends AbstractType
+{
+    static final BooleanType BOOLEAN = new BooleanType();
+
+    private BooleanType()
+    {
+        super("boolean", boolean.class, 1);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class DoubleType
+        extends AbstractType
+{
+    static final DoubleType DOUBLE = new DoubleType();
+
+    private DoubleType()
+    {
+        super("double", double.class, 8);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class LongType
+        extends AbstractType
+{
+    static final LongType LONG = new LongType();
+
+    private LongType()
+    {
+        super("long", long.class, 8);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class StringType
+        extends AbstractType
+{
+    static final StringType STRING = new StringType();
+
+    private StringType()
+    {
+        super("string", String.class, 4);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java

@@ -0,0 +1,39 @@
+package org.embulk.spi.type;
+
+import org.embulk.spi.time.Timestamp;
+
+public class TimestampType
+        extends AbstractType
+{
+    static final TimestampType TIMESTAMP = new TimestampType();
+
+    private static final String DEFAULT_FORMAT = "%Y-%m-%d %H-%M-%S,%N %z";
+
+    private final String format;
+
+    private TimestampType()
+    {
+        this(null);
+    }
+
+    private TimestampType(String format)
+    {
+        super("timestamp", Timestamp.class, 12);  // long msec + int nsec
+        this.format = format;
+    }
+
+    public TimestampType withFormat(String format)
+    {
+        // TODO is this correct design...?
+        return new TimestampType(format);
+    }
+
+    public String getFormat()
+    {
+        if (format == null) {
+            return DEFAULT_FORMAT;
+        } else {
+            return format;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/Type.java

@@ -0,0 +1,15 @@
+package org.embulk.spi.type;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+@JsonDeserialize(using=TypeDeserializer.class)
+public interface Type
+{
+    @JsonValue
+    public String getName();
+
+    public Class<?> getJavaType();
+
+    public byte getFixedStorageSize();
+}