embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java

@@ -0,0 +1,49 @@
+package org.embulk.spi.time;
+
+import java.io.IOException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class TimestampSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(Timestamp.class, new DateTimeZoneSerializer());
+        module.addDeserializer(Timestamp.class, new DateTimeZoneDeserializer());
+        mapper.registerModule(module);
+    }
+
+    public static class DateTimeZoneSerializer
+            extends JsonSerializer<Timestamp>
+    {
+        @Override
+        public void serialize(Timestamp value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeString(value.toString());
+        }
+    }
+
+    public static class DateTimeZoneDeserializer
+            extends FromStringDeserializer<Timestamp>
+    {
+        public DateTimeZoneDeserializer()
+        {
+            super(Timestamp.class);
+        }
+
+        @Override
+        protected Timestamp _deserialize(String value, DeserializationContext context)
+                throws JsonMappingException
+        {
+            return Timestamp.fromString(value);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/AbstractType.java

@@ -0,0 +1,58 @@
+package org.embulk.spi.type;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public abstract class AbstractType
+        implements Type
+{
+    private final String name;
+    private final Class<?> javaType;
+    private byte fixedStorageSize;
+
+    protected AbstractType(String name, Class<?> javaType, int fixedStorageSize)
+    {
+        this.name = name;
+        this.javaType = javaType;
+        this.fixedStorageSize = (byte) fixedStorageSize;
+    }
+
+    @Override
+    public String getName()
+    {
+        return name;
+    }
+
+    @Override
+    public Class<?> getJavaType()
+    {
+        return javaType;
+    }
+
+    @Override
+    public byte getFixedStorageSize()
+    {
+        return fixedStorageSize;
+    }
+
+    @SuppressFBWarnings(value = "EQ_UNUSUAL")
+    @Override
+    public boolean equals(Object o)
+    {
+        if (o == null) {
+            return false;
+        }
+        return o.getClass().isAssignableFrom(getClass());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return getClass().hashCode();
+    }
+
+    @Override
+    public String toString()
+    {
+        return name;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/BooleanType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class BooleanType
+        extends AbstractType
+{
+    static final BooleanType BOOLEAN = new BooleanType();
+
+    private BooleanType()
+    {
+        super("boolean", boolean.class, 1);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/DoubleType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class DoubleType
+        extends AbstractType
+{
+    static final DoubleType DOUBLE = new DoubleType();
+
+    private DoubleType()
+    {
+        super("double", double.class, 8);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/LongType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class LongType
+        extends AbstractType
+{
+    static final LongType LONG = new LongType();
+
+    private LongType()
+    {
+        super("long", long.class, 8);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/StringType.java

@@ -0,0 +1,12 @@
+package org.embulk.spi.type;
+
+public class StringType
+        extends AbstractType
+{
+    static final StringType STRING = new StringType();
+
+    private StringType()
+    {
+        super("string", String.class, 4);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java

@@ -0,0 +1,41 @@
+package org.embulk.spi.type;
+
+import org.embulk.spi.time.Timestamp;
+
+public class TimestampType
+        extends AbstractType
+{
+    static final TimestampType TIMESTAMP = new TimestampType();
+
+    private static final String DEFAULT_FORMAT = "%Y-%m-%d %H:%M:%S.%6N %z";
+
+    private final String format;
+
+    private TimestampType()
+    {
+        this(null);
+    }
+
+    private TimestampType(String format)
+    {
+        super("timestamp", Timestamp.class, 12);  // long msec + int nsec
+        this.format = format;
+    }
+
+    @Deprecated
+    public TimestampType withFormat(String format)
+    {
+        // TODO is this correct design...?
+        return new TimestampType(format);
+    }
+
+    @Deprecated
+    public String getFormat()
+    {
+        if (format == null) {
+            return DEFAULT_FORMAT;
+        } else {
+            return format;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/Type.java

@@ -0,0 +1,15 @@
+package org.embulk.spi.type;
+
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
+@JsonDeserialize(using=TypeDeserializer.class)
+public interface Type
+{
+    @JsonValue
+    String getName();
+
+    Class<?> getJavaType();
+
+    byte getFixedStorageSize();
+}

data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java

@@ -0,0 +1,44 @@
+package org.embulk.spi.type;
+
+import java.util.Map;
+import java.io.IOException;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+
+public class TypeDeserializer
+        extends FromStringDeserializer<Type>
+{
+    private static final Map<String, Type> stringToTypeMap;
+
+    static {
+        ImmutableMap.Builder<String, Type> builder = ImmutableMap.builder();
+        builder.put(BooleanType.BOOLEAN.getName(), BooleanType.BOOLEAN);
+        builder.put(LongType.LONG.getName(), LongType.LONG);
+        builder.put(DoubleType.DOUBLE.getName(), DoubleType.DOUBLE);
+        builder.put(StringType.STRING.getName(), StringType.STRING);
+        builder.put(TimestampType.TIMESTAMP.getName(), TimestampType.TIMESTAMP);
+        stringToTypeMap = builder.build();
+    }
+
+    public TypeDeserializer()
+    {
+        super(Type.class);
+    }
+
+    @Override
+    protected Type _deserialize(String value, DeserializationContext context)
+            throws IOException
+    {
+        Type t = stringToTypeMap.get(value);
+        if (t == null) {
+            throw new JsonMappingException(
+                    String.format("Unknown type name '%s'. Supported types are: %s",
+                        value,
+                        Joiner.on(", ").join(stringToTypeMap.keySet())));
+        }
+        return t;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/Types.java

@@ -0,0 +1,14 @@
+package org.embulk.spi.type;
+
+public class Types
+{
+    public static final BooleanType BOOLEAN = BooleanType.BOOLEAN;
+
+    public static final LongType LONG = LongType.LONG;
+
+    public static final DoubleType DOUBLE = DoubleType.DOUBLE;
+
+    public static final StringType STRING = StringType.STRING;
+
+    public static final TimestampType TIMESTAMP = TimestampType.TIMESTAMP;
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/ByteSize.java

@@ -0,0 +1,156 @@
+package org.embulk.spi.unit;
+
+import java.util.Objects;
+import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import com.google.common.base.Preconditions;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+public class ByteSize
+        implements Comparable<ByteSize>
+{
+    private static final Pattern PATTERN = Pattern.compile("\\A(\\d+(?:\\.\\d+)?)\\s?([a-zA-Z]*)\\z");
+
+    private final long bytes;
+    private final Unit displayUnit;
+
+    public ByteSize(double size, Unit unit)
+    {
+        Preconditions.checkArgument(!Double.isInfinite(size), "size is infinite");
+        Preconditions.checkArgument(!Double.isNaN(size), "size is not a number");
+        Preconditions.checkArgument(size >= 0, "size is negative");
+        Preconditions.checkNotNull(unit, "unit is null");
+        Preconditions.checkArgument(size * unit.getFactor() <= (double) Long.MAX_VALUE, "size is large than (2^63)-1 in bytes");
+        this.bytes = (long) (size * unit.getFactor());
+        this.displayUnit = unit;
+    }
+
+    @JsonCreator
+    public ByteSize(long bytes)
+    {
+        Preconditions.checkArgument(bytes >= 0, "size is negative");
+        this.bytes = bytes;
+        this.displayUnit = Unit.BYTES;
+    }
+
+    public long getBytes()
+    {
+        return bytes;
+    }
+
+    public int getBytesInt()
+    {
+        if (bytes > Integer.MAX_VALUE) {
+            throw new RuntimeException("Byte size is too large (must be smaller than 2GB)");
+        }
+        return (int) bytes;
+    }
+
+    public long roundTo(Unit unit)
+    {
+        return (long) Math.floor(getValue(unit) + 0.5);
+    }
+
+    public double getValue(Unit unit)
+    {
+        return bytes / (double) unit.getFactor();
+    }
+
+    @JsonCreator
+    public static ByteSize parseByteSize(String size)
+    {
+        Preconditions.checkNotNull(size, "size is null");
+        Preconditions.checkArgument(!size.isEmpty(), "size is empty");
+
+        Matcher matcher = PATTERN.matcher(size);
+        if (!matcher.matches()) {
+            throw new IllegalArgumentException("Invalid byte size string '" + size + "'");
+        }
+
+        double value = Double.parseDouble(matcher.group(1));  // NumberFormatException extends IllegalArgumentException.
+
+        String unitString = matcher.group(2);
+        if (unitString.isEmpty()) {
+            return new ByteSize(value, Unit.BYTES);
+        } else {
+            String upperUnitString = unitString.toUpperCase(Locale.ENGLISH);
+            for (Unit unit : Unit.values()) {
+                if (unit.getUnitString().toUpperCase(Locale.ENGLISH).equals(upperUnitString)) {
+                    return new ByteSize(value, unit);
+                }
+            }
+        }
+
+        throw new IllegalArgumentException("Unknown unit '" + unitString + "'");
+    }
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        double value = getValue(displayUnit);
+        String integer = String.format(Locale.ENGLISH, "%d", (long) value);
+        String decimal = String.format(Locale.ENGLISH, "%.2f", value);
+        if (decimal.equals(integer + ".00")) {
+            return integer + displayUnit.getUnitString();
+        } else {
+            return decimal + displayUnit.getUnitString();
+        }
+    }
+
+    @Override
+    public int compareTo(ByteSize o)
+    {
+        return Long.compare(bytes, o.bytes);
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ByteSize)) {
+            return false;
+        }
+        ByteSize o = (ByteSize) obj;
+        return this.bytes == o.bytes;
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(bytes);
+    }
+
+    public enum Unit
+    {
+        BYTES(1L, "B"),
+        KB(1L << 10, "KB"),
+        MB(1L << 20, "MB"),
+        GB(1L << 30, "GB"),
+        TB(1L << 40, "TB"),
+        PT(1L << 50, "PB");
+
+        private final long factor;
+        private final String unitString;
+
+        Unit(long factor, String unitString)
+        {
+            this.factor = factor;
+            this.unitString = unitString;
+        }
+
+        long getFactor()
+        {
+            return factor;
+        }
+
+        String getUnitString()
+        {
+            return unitString;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFile.java

@@ -0,0 +1,106 @@
+package org.embulk.spi.unit;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.Files;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import org.embulk.spi.Exec;
+import org.embulk.spi.TempFileSpace;
+import org.embulk.spi.TempFileException;
+
+public class LocalFile
+{
+    public static LocalFile of(File path) throws IOException
+    {
+        return of(path.toPath());
+    }
+
+    public static LocalFile of(Path path) throws IOException
+    {
+        return new LocalFile(path, Files.readAllBytes(path));
+    }
+
+    public static LocalFile of(String path) throws IOException
+    {
+        return of(Paths.get(path));
+    }
+
+    public static LocalFile ofContent(byte[] content)
+    {
+        return new LocalFile(content);
+    }
+
+    public static LocalFile ofContent(String content)
+    {
+        return new LocalFile(content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    private Path path;
+    private final byte[] content;
+
+    private LocalFile(Path path, byte[] content)
+    {
+        this.path = path;
+        this.content = content;
+    }
+
+    private LocalFile(byte[] content)
+    {
+        this.path = null;
+        this.content = content;
+    }
+
+    public File getFile()
+    {
+        return getPath(Exec.getTempFileSpace()).toFile();
+    }
+
+    public File getFile(TempFileSpace space)
+    {
+        return getPath(space).toFile();
+    }
+
+    public Path getPath()
+    {
+        return getPath(Exec.getTempFileSpace());
+    }
+
+    public synchronized Path getPath(TempFileSpace tempFileSpace)
+    {
+        if (path == null) {
+            Path temp = tempFileSpace.createTempFile().toPath();
+            try {
+                Files.write(temp, content);
+            } catch (IOException ex) {
+                throw new TempFileException(ex);
+            }
+            this.path = temp;
+        }
+        return path;
+    }
+
+    public byte[] getContent()
+    {
+        return content;
+    }
+
+    public String getContentAsString()
+    {
+        return new String(content);
+    }
+
+    public String getContentAsString(Charset charset)
+    {
+        return new String(content, charset);
+    }
+
+    public InputStream newContentInputStream()
+    {
+        return new ByteArrayInputStream(content);
+    }
+}