embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java

@@ -0,0 +1,17 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface ParserPlugin
+{
+    public interface Control
+    {
+        public void run(TaskSource taskSource, Schema schema);
+    }
+
+    public void transaction(ConfigSource config, ParserPlugin.Control control);
+
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output);
+}

data/embulk-core/src/main/java/org/embulk/spi/Schema.java

@@ -0,0 +1,101 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.google.common.base.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.embulk.spi.type.Type;
+
+public class Schema
+{
+    private final List<Column> columns;
+
+    @JsonCreator
+    public Schema(List<Column> columns)
+    {
+        this.columns = columns;
+    }
+
+    @JsonValue
+    public List<Column> getColumns()
+    {
+        return columns;
+    }
+
+    public int getColumnCount()
+    {
+        return columns.size();
+    }
+
+    public Column getColumn(int index)
+    {
+        return columns.get(index);
+    }
+
+    public String getColumnName(int index)
+    {
+        return getColumn(index).getName();
+    }
+
+    public Type getColumnType(int index)
+    {
+        return getColumn(index).getType();
+    }
+
+    public void visitColumns(SchemaVisitor visitor)
+    {
+        for (Column column : columns) {
+            column.visit(visitor);
+        }
+    }
+
+    public boolean isEmpty()
+    {
+        return columns.isEmpty();
+    }
+
+    public int size()
+    {
+        return columns.size();
+    }
+
+    public int getFixedStorageSize()
+    {
+        int total = 0;
+        for (Column column : columns) {
+            total += column.getType().getFixedStorageSize();
+        }
+        return total;
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof Schema)) {
+            return false;
+        }
+        Schema other = (Schema) obj;
+        return Objects.equal(columns, other.columns);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(columns);
+    }
+
+    @Override
+    public String toString()
+    {
+        StringBuilder sbuf = new StringBuilder();
+        sbuf.append("Schema{\n");
+        for (Column c : columns) {
+            sbuf.append(String.format(" %4d: %s %s%n", c.getIndex(), c.getName(), c.getType()));
+        }
+        sbuf.append("}");
+        return sbuf.toString();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java

@@ -0,0 +1,52 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class SchemaConfig
+{
+    private final List<ColumnConfig> columns;
+
+    @JsonCreator
+    public SchemaConfig(List<ColumnConfig> columns)
+    {
+        this.columns = columns;
+    }
+
+    @JsonValue
+    public List<ColumnConfig> getColumns()
+    {
+        return columns;
+    }
+
+    public Schema toSchema()
+    {
+        ImmutableList.Builder<Column> builder = ImmutableList.builder();
+        for (int i=0; i < columns.size(); i++) {
+            builder.add(columns.get(i).toColumn(i));
+        }
+        return new Schema(builder.build());
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof SchemaConfig)) {
+            return false;
+        }
+        SchemaConfig other = (SchemaConfig) obj;
+        return Objects.equal(columns, other.columns);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(columns);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/SchemaVisitor.java

@@ -0,0 +1,14 @@
+package org.embulk.spi;
+
+public interface SchemaVisitor
+{
+    public void booleanColumn(Column column);
+
+    public void longColumn(Column column);
+
+    public void doubleColumn(Column column);
+
+    public void stringColumn(Column column);
+
+    public void timestampColumn(Column column);
+}

data/embulk-core/src/main/java/org/embulk/spi/Transactional.java

@@ -0,0 +1,10 @@
+package org.embulk.spi;
+
+import org.embulk.config.CommitReport;
+
+public interface Transactional
+{
+    public void abort();
+
+    public CommitReport commit();
+}

data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java

@@ -0,0 +1,17 @@
+package org.embulk.spi;
+
+import org.embulk.config.CommitReport;
+
+public interface TransactionalFileInput
+        extends Transactional, FileInput
+{
+    public Buffer poll();
+
+    public boolean nextFile();
+
+    public void close();
+
+    public void abort();
+
+    public CommitReport commit();
+}

data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java

@@ -0,0 +1,19 @@
+package org.embulk.spi;
+
+import org.embulk.config.CommitReport;
+
+public interface TransactionalFileOutput
+        extends Transactional, FileOutput
+{
+    public void nextFile();
+
+    public void add(Buffer buffer);
+
+    public void finish();
+
+    public void close();
+
+    public void abort();
+
+    public CommitReport commit();
+}

data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java

@@ -0,0 +1,17 @@
+package org.embulk.spi;
+
+import org.embulk.config.CommitReport;
+
+public interface TransactionalPageOutput
+        extends Transactional, PageOutput
+{
+    public void add(Page page);
+
+    public void finish();
+
+    public void close();
+
+    public void abort();
+
+    public CommitReport commit();
+}

data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java

@@ -0,0 +1,57 @@
+package org.embulk.spi.time;
+
+import java.io.IOException;
+import org.joda.time.DateTimeZone;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.Module;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+import org.embulk.config.ModelManager;
+
+public class DateTimeZoneSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(DateTimeZone.class, new DateTimeZoneSerializer());
+        module.addDeserializer(DateTimeZone.class, new DateTimeZoneDeserializer());
+        mapper.registerModule(module);
+    }
+
+    public static class DateTimeZoneSerializer
+            extends JsonSerializer<DateTimeZone>
+    {
+        @Override
+        public void serialize(DateTimeZone value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeString(value.getID());
+        }
+    }
+
+    public static class DateTimeZoneDeserializer
+            extends FromStringDeserializer<DateTimeZone>
+    {
+        public DateTimeZoneDeserializer()
+        {
+            super(DateTimeZone.class);
+        }
+
+        @Override
+        protected DateTimeZone _deserialize(String value, DeserializationContext context)
+                throws JsonMappingException
+        {
+            DateTimeZone parsed = TimestampFormat.parseDateTimeZone(value);
+            if (parsed == null) {
+                // TODO include link to a document to the message for the list of supported time zones
+                throw new JsonMappingException(String.format("Unknown time zone name '%s'", value));
+            }
+            return parsed;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java

@@ -0,0 +1,8 @@
+package org.embulk.spi.time;
+
+public interface JRubyTimeParserHelper
+{
+    public long strptime(String text) throws TimestampParseException;
+
+    public String getZone();
+}

data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java

@@ -0,0 +1,6 @@
+package org.embulk.spi.time;
+
+public interface JRubyTimeParserHelperFactory
+{
+    public JRubyTimeParserHelper newInstance(String formatString, int year, int mon, int day, int hour, int min, int sec, int usec);
+}

data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java

@@ -0,0 +1,159 @@
+package org.embulk.spi.time;
+
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.DateTime;
+import org.jruby.Ruby;
+import org.jruby.RubyTime;
+
+public class Timestamp
+        implements Comparable<Timestamp>
+{
+    private final static DateTimeFormatter TO_STRING_FORMATTER_SECONDS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss z").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_MILLIS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS z").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_CUSTOM = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZoneUTC();
+    private static final Pattern FROM_STRING_PATTERN = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})(?:\\.(\\d{1,9}))? UTC");
+
+    private final long seconds;
+    private final int nano;
+
+    private Timestamp(long seconds, int nano)
+    {
+        this.seconds = seconds;
+        this.nano = nano;
+    }
+
+    public static Timestamp ofEpochSecond(long epochSecond)
+    {
+        return new Timestamp(epochSecond, 0);
+    }
+
+    public static Timestamp ofEpochSecond(long epochSecond, long nanoAdjustment)
+    {
+        return new Timestamp(epochSecond + nanoAdjustment / 1000000000, (int) (nanoAdjustment % 1000000000));
+    }
+
+    public static Timestamp ofEpochMilli(long epochMilli)
+    {
+        return new Timestamp(epochMilli / 1000, (int) (epochMilli % 1000 * 1000000));
+    }
+
+    public long getEpochSecond()
+    {
+        return seconds;
+    }
+
+    public int getNano()
+    {
+        return nano;
+    }
+
+    public long toEpochMilli()
+    {
+        return seconds * 1000 + nano / 1000000;
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (this == other) {
+            return true;
+        }
+        if (!(other instanceof Timestamp)) {
+            return false;
+        }
+        Timestamp o = (Timestamp) other;
+        return this.seconds == o.seconds && this.nano == o.nano;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        int h = (int) (seconds ^ (seconds >>> 32));
+        h += 17 * nano;
+        return h;
+    }
+
+    @Override
+    public int compareTo(Timestamp t)
+    {
+        if (seconds < t.seconds) {
+            return -1;
+        } else if (seconds == t.seconds) {
+            return nano == t.nano ? 0 : (nano < t.nano ? -1 : 1);
+        } else {
+            return 1;
+        }
+    }
+
+    public RubyTime getRubyTime(Ruby runtime)
+    {
+        RubyTime time = new RubyTime(runtime, runtime.getClass("Time"), new DateTime(toEpochMilli())).gmtime();
+        time.setNSec(nano % 1000000);
+        return time;
+    }
+
+    public Timestamp fromRubyTime(RubyTime time)
+    {
+        long msec = time.getDateTime().getMillis();
+        long sec = msec / 1000;
+        long nsec = time.getNSec() + (msec % 1000) * 1000000;
+        return Timestamp.ofEpochSecond(sec, nsec);
+    }
+
+    @Override
+    public String toString()
+    {
+        if (nano == 0) {
+            return TO_STRING_FORMATTER_SECONDS.print(getEpochSecond() * 1000);
+
+        } else if (nano % 1000000 == 0) {
+            return TO_STRING_FORMATTER_MILLIS.print(toEpochMilli());
+
+        } else {
+            StringBuffer sb = new StringBuffer();
+            TO_STRING_FORMATTER_CUSTOM.printTo(sb, getEpochSecond() * 1000);
+            sb.append(".");
+
+            String digits;
+            int zeroDigits;
+            if (nano % 1000 == 0) {
+                digits = Integer.toString(nano / 1000);
+                zeroDigits = 6 - digits.length();
+            } else {
+                digits = Integer.toString(nano);
+                zeroDigits = 9 - digits.length();
+            }
+            sb.append(digits);
+            for (; zeroDigits > 0; zeroDigits--) {
+                sb.append('0');
+            }
+
+            sb.append(" UTC");
+            return sb.toString();
+        }
+    }
+
+    static Timestamp fromString(String text)
+    {
+        // TODO exception handling
+        Matcher m = FROM_STRING_PATTERN.matcher(text);
+        if (!m.matches()) {
+            throw new IllegalArgumentException(String.format("Invalid timestamp format '%s'", text));
+        }
+
+        long seconds = TO_STRING_FORMATTER_CUSTOM.parseDateTime(m.group(1)).getMillis() / 1000;
+
+        int nano;
+        String frac = m.group(2);
+        if (frac == null) {
+            nano = 0;
+        } else {
+            nano = Integer.parseInt(frac) * (int) Math.pow(10, 9 - frac.length());
+        }
+
+        return new Timestamp(seconds, nano);
+    }
+}