embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java

@@ -0,0 +1,159 @@
+package org.embulk.spi.time;
+
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.DateTime;
+import org.jruby.Ruby;
+import org.jruby.RubyTime;
+
+public class Timestamp
+        implements Comparable<Timestamp>
+{
+    private final static DateTimeFormatter TO_STRING_FORMATTER_SECONDS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss z").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_MILLIS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS z").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_CUSTOM = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZoneUTC();
+    private static final Pattern FROM_STRING_PATTERN = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})(?:\\.(\\d{1,9}))? UTC");
+
+    private final long seconds;
+    private final int nano;
+
+    private Timestamp(long seconds, int nano)
+    {
+        this.seconds = seconds;
+        this.nano = nano;
+    }
+
+    public static Timestamp ofEpochSecond(long epochSecond)
+    {
+        return new Timestamp(epochSecond, 0);
+    }
+
+    public static Timestamp ofEpochSecond(long epochSecond, long nanoAdjustment)
+    {
+        return new Timestamp(epochSecond + nanoAdjustment / 1000000000, (int) (nanoAdjustment % 1000000000));
+    }
+
+    public static Timestamp ofEpochMilli(long epochMilli)
+    {
+        return new Timestamp(epochMilli / 1000, (int) (epochMilli % 1000 * 1000000));
+    }
+
+    public long getEpochSecond()
+    {
+        return seconds;
+    }
+
+    public int getNano()
+    {
+        return nano;
+    }
+
+    public long toEpochMilli()
+    {
+        return seconds * 1000 + nano / 1000000;
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (this == other) {
+            return true;
+        }
+        if (!(other instanceof Timestamp)) {
+            return false;
+        }
+        Timestamp o = (Timestamp) other;
+        return this.seconds == o.seconds && this.nano == o.nano;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int h = (int) (seconds ^ (seconds >>> 32));
+        h += 17 * nano;
+        return h;
+    }
+
+    @Override
+    public int compareTo(Timestamp t)
+    {
+        if (seconds < t.seconds) {
+            return -1;
+        } else if (seconds == t.seconds) {
+            return nano == t.nano ? 0 : (nano < t.nano ? -1 : 1);
+        } else {
+            return 1;
+        }
+    }
+
+    public RubyTime getRubyTime(Ruby runtime)
+    {
+        RubyTime time = new RubyTime(runtime, runtime.getClass("Time"), new DateTime(toEpochMilli())).gmtime();
+        time.setNSec(nano % 1000000);
+        return time;
+    }
+
+    public static Timestamp fromRubyTime(RubyTime time)
+    {
+        long msec = time.getDateTime().getMillis();
+        long sec = msec / 1000;
+        long nsec = time.getNSec() + (msec % 1000) * 1000000;
+        return Timestamp.ofEpochSecond(sec, nsec);
+    }
+
+    @Override
+    public String toString()
+    {
+        if (nano == 0) {
+            return TO_STRING_FORMATTER_SECONDS.print(getEpochSecond() * 1000);
+
+        } else if (nano % 1000000 == 0) {
+            return TO_STRING_FORMATTER_MILLIS.print(toEpochMilli());
+
+        } else {
+            StringBuffer sb = new StringBuffer();
+            TO_STRING_FORMATTER_CUSTOM.printTo(sb, getEpochSecond() * 1000);
+            sb.append(".");
+
+            String digits;
+            int zeroDigits;
+            if (nano % 1000 == 0) {
+                digits = Integer.toString(nano / 1000);
+                zeroDigits = 6 - digits.length();
+            } else {
+                digits = Integer.toString(nano);
+                zeroDigits = 9 - digits.length();
+            }
+            sb.append(digits);
+            for (; zeroDigits > 0; zeroDigits--) {
+                sb.append('0');
+            }
+
+            sb.append(" UTC");
+            return sb.toString();
+        }
+    }
+
+    static Timestamp fromString(String text)
+    {
+        // TODO exception handling
+        Matcher m = FROM_STRING_PATTERN.matcher(text);
+        if (!m.matches()) {
+            throw new IllegalArgumentException(String.format("Invalid timestamp format '%s'", text));
+        }
+
+        long seconds = TO_STRING_FORMATTER_CUSTOM.parseDateTime(m.group(1)).getMillis() / 1000;
+
+        int nano;
+        String frac = m.group(2);
+        if (frac == null) {
+            nano = 0;
+        } else {
+            nano = Integer.parseInt(frac) * (int) Math.pow(10, 9 - frac.length());
+        }
+
+        return new Timestamp(seconds, nano);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -0,0 +1,100 @@
+package org.embulk.spi.time;
+
+import java.util.Set;
+import com.google.common.collect.ImmutableSet;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormat;
+
+public class TimestampFormat
+{
+    private final String format;
+
+    @JsonCreator
+    public TimestampFormat(String format)
+    {
+        this.format = format;
+    }
+
+    @JsonValue
+    public String getFormat()
+    {
+        return format;
+    }
+
+    @Deprecated
+    public TimestampFormatter newFormatter(TimestampFormatter.FormatterTask task)
+    {
+        return new TimestampFormatter(format, task);
+    }
+
+    @Deprecated
+    public TimestampParser newParser(TimestampParser.ParserTask task)
+    {
+        return new TimestampParser(format, task);
+    }
+
+    private static Set<String> availableTimeZoneNames = ImmutableSet.copyOf(DateTimeZone.getAvailableIDs());
+
+    public static DateTimeZone parseDateTimeZone(String s)
+    {
+        if(s.startsWith("+") || s.startsWith("-")) {
+            return DateTimeZone.forID(s);
+
+        } else if (s.equals("Z")) {
+            return DateTimeZone.UTC;
+
+        } else {
+            try {
+                int rawOffset = (int) DateTimeFormat.forPattern("z").parseMillis(s);
+                if(rawOffset == 0) {
+                    return DateTimeZone.UTC;
+                }
+                int offset = rawOffset / -1000;
+                int h = offset / 3600;
+                int m = offset % 3600;
+                return DateTimeZone.forOffsetHoursMinutes(h, m);
+            } catch (IllegalArgumentException ex) {
+                // parseMillis failed
+            }
+
+            // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+            // we want to only return timezone if exact match, otherwise exception
+            if (availableTimeZoneNames.contains(s)) {
+                //return TimeZone.getTimeZone(s);
+                return DateTimeZone.forID(s);
+            }
+            return null;
+        }
+    }
+
+    //// Java standard TimeZone
+    //static TimeZone parseDateTimeZone(String s)
+    //{
+    //    if(s.startsWith("+") || s.startsWith("-")) {
+    //        return TimeZone.getTimeZone("GMT"+s);
+    //
+    //    } else {
+    //        ParsePosition pp = new ParsePosition(0);
+    //        Date off = new SimpleDateFormat("z").parse(s, pp);
+    //        if(off != null && pp.getErrorIndex() == -1) {
+    //            int rawOffset = (int) off.getTime();
+    //            if(rawOffset == 0) {
+    //                return TimeZone.UTC;
+    //            }
+    //            int offset = rawOffset / -1000;
+    //            int h = offset / 3600;
+    //            int m = offset % 3600;
+    //            return DateTimeZone.getTimeZone(String.format("GMT%+02d%02d", h, m));
+    //        }
+    //
+    //        // TimeZone.getTimeZone returns GMT zone if given timezone id is not found
+    //        // we want to only return timezone if exact match, otherwise exception
+    //        if (availableTimeZoneNames.contains(s)) {
+    //            return TimeZone.getTimeZone(s);
+    //        }
+    //        return null;
+    //    }
+    //}
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -0,0 +1,97 @@
+package org.embulk.spi.time;
+
+import java.util.Locale;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import com.google.common.base.Optional;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.util.RubyDateFormat;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.util.LineEncoder;
+
+public class TimestampFormatter
+{
+    @Deprecated
+    public interface FormatterTask
+            extends org.embulk.config.Task
+    {
+        @Config("timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getTimeZone();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N %z\"")
+        public String getDefaultTimestampFormat();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface TimestampColumnOption
+    {
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+
+        @Config("format")
+        @ConfigDefault("null")
+        public Optional<String> getFormat();
+    }
+
+    private final RubyDateFormat dateFormat;
+    private final DateTimeZone timeZone;
+
+    @Deprecated
+    public TimestampFormatter(String format, FormatterTask task)
+    {
+        this(task.getJRuby(), format, task.getTimeZone());
+    }
+
+    public TimestampFormatter(Task task, Optional<? extends TimestampColumnOption> columnOption)
+    {
+        this(task.getJRuby(),
+                columnOption.isPresent() ?
+                    columnOption.get().getFormat().or(task.getDefaultTimestampFormat())
+                    : task.getDefaultTimestampFormat(),
+                columnOption.isPresent() ?
+                    columnOption.get().getTimeZone().or(task.getDefaultTimeZone())
+                    : task.getDefaultTimeZone());
+    }
+
+    public TimestampFormatter(ScriptingContainer jruby, String format, DateTimeZone timeZone)
+    {
+        this.timeZone = timeZone;
+        this.dateFormat = new RubyDateFormat(format, Locale.ENGLISH, true);
+    }
+
+    public DateTimeZone getTimeZone()
+    {
+        return timeZone;
+    }
+
+    public void format(Timestamp value, LineEncoder encoder)
+    {
+        // TODO optimize by directly appending to internal buffer
+        encoder.addText(format(value));
+    }
+
+    public String format(Timestamp value)
+    {
+        // TODO optimize by using reused StringBuilder
+        dateFormat.setDateTime(new DateTime(value.getEpochSecond()*1000, timeZone));
+        dateFormat.setNSec(value.getNano());
+        return dateFormat.format(null);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java

@@ -0,0 +1,10 @@
+package org.embulk.spi.time;
+
+public class TimestampParseException
+        extends Exception
+{
+    public TimestampParseException(String message)
+    {
+        super(message);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -0,0 +1,104 @@
+package org.embulk.spi.time;
+
+import org.joda.time.DateTimeZone;
+import com.google.common.base.Optional;
+import org.jruby.embed.ScriptingContainer;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
+
+public class TimestampParser
+{
+    @Deprecated
+    public interface ParserTask
+            extends org.embulk.config.Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"")
+        public String getDefaultTimestampFormat();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface TimestampColumnOption
+    {
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+
+        @Config("format")
+        @ConfigDefault("null")
+        public Optional<String> getFormat();
+    }
+
+    private final JRubyTimeParserHelper helper;
+    private final DateTimeZone defaultTimeZone;
+
+    @Deprecated
+    public TimestampParser(String format, ParserTask task)
+    {
+        this(task.getJRuby(), format, task.getDefaultTimeZone());
+    }
+
+    TimestampParser(Task task)
+    {
+        this(task.getJRuby(), task.getDefaultTimestampFormat(), task.getDefaultTimeZone());
+    }
+
+    public TimestampParser(Task task, TimestampColumnOption columnOption)
+    {
+        this(task.getJRuby(),
+                columnOption.getFormat().or(task.getDefaultTimestampFormat()),
+                columnOption.getTimeZone().or(task.getDefaultTimeZone()));
+    }
+
+    public TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
+    {
+        JRubyTimeParserHelperFactory helperFactory = (JRubyTimeParserHelperFactory) jruby.runScriptlet("Embulk::Java::TimeParserHelper::Factory.new");
+        // TODO get default current time from ExecTask.getExecTimestamp
+        this.helper = (JRubyTimeParserHelper) helperFactory.newInstance(format, 1970, 1, 1, 0, 0, 0, 0);  // TODO default time zone
+        this.defaultTimeZone = defaultTimeZone;
+    }
+
+    public DateTimeZone getDefaultTimeZone()
+    {
+        return defaultTimeZone;
+    }
+
+    public Timestamp parse(String text) throws TimestampParseException
+    {
+        long localUsec = helper.strptimeUsec(text);
+        String zone = helper.getZone();
+
+        DateTimeZone timeZone = defaultTimeZone;
+        if (zone != null) {
+            // TODO cache parsed zone?
+            timeZone = parseDateTimeZone(zone);
+            if (timeZone == null) {
+                throw new TimestampParseException("Invalid time zone name '" + text + "'");
+            }
+        }
+
+        long localSec = localUsec / 1000000;
+        long usec = localUsec % 1000000;
+        long sec = timeZone.convertLocalToUTC(localSec*1000, false) / 1000;
+
+        return Timestamp.ofEpochSecond(sec, usec * 1000);
+    }
+}