embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java

@@ -0,0 +1,47 @@
+package org.embulk.spi.type;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.io.IOException;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import org.embulk.config.ModelManager;
+
+public class TypeDeserializer
+        extends FromStringDeserializer<Type>
+{
+    private static final Map<String, Type> stringToTypeMap;
+
+    static {
+        ImmutableMap.Builder<String, Type> builder = ImmutableMap.builder();
+        builder.put(BooleanType.BOOLEAN.getName(), BooleanType.BOOLEAN);
+        builder.put(LongType.LONG.getName(), LongType.LONG);
+        builder.put(DoubleType.DOUBLE.getName(), DoubleType.DOUBLE);
+        builder.put(StringType.STRING.getName(), StringType.STRING);
+        builder.put(TimestampType.TIMESTAMP.getName(), TimestampType.TIMESTAMP);
+        stringToTypeMap = builder.build();
+    }
+
+    public TypeDeserializer()
+    {
+        super(Type.class);
+    }
+
+    @Override
+    protected Type _deserialize(String value, DeserializationContext context)
+            throws IOException
+    {
+        Type t = stringToTypeMap.get(value);
+        if (t == null) {
+            throw new JsonMappingException(
+                    String.format("Unknown type name '%s'. Supported types are: %s",
+                        value,
+                        Joiner.on(", ").join(stringToTypeMap.keySet())));
+        }
+        return t;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/type/Types.java

@@ -0,0 +1,14 @@
+package org.embulk.spi.type;
+
+public class Types
+{
+    public static final BooleanType BOOLEAN = BooleanType.BOOLEAN;
+
+    public static final LongType LONG = LongType.LONG;
+
+    public static final DoubleType DOUBLE = DoubleType.DOUBLE;
+
+    public static final StringType STRING = StringType.STRING;
+
+    public static final TimestampType TIMESTAMP = TimestampType.TIMESTAMP;
+}

data/embulk-core/src/main/java/org/embulk/spi/util/CharsetSerDe.java

@@ -0,0 +1,55 @@
+package org.embulk.spi.util;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.fasterxml.jackson.module.guice.ObjectMapperModule;
+
+public class CharsetSerDe
+{
+    public static void configure(ObjectMapperModule mapper)
+    {
+        SimpleModule module = new SimpleModule();
+        module.addSerializer(Charset.class, new CharsetSerializer());
+        module.addDeserializer(Charset.class, new CharsetDeserializer());
+        mapper.registerModule(module);
+    }
+
+    public static class CharsetSerializer
+            extends JsonSerializer<Charset>
+    {
+        @Override
+        public void serialize(Charset value, JsonGenerator jgen, SerializerProvider provider)
+                throws IOException
+        {
+            jgen.writeString(value.name());
+        }
+    }
+
+    public static class CharsetDeserializer
+            extends FromStringDeserializer<Charset>
+    {
+        public CharsetDeserializer()
+        {
+            super(Charset.class);
+        }
+
+        @Override
+        protected Charset _deserialize(String value, DeserializationContext context)
+                throws JsonMappingException
+        {
+            try {
+                return Charset.forName(value);
+            } catch (UnsupportedOperationException ex) {
+                // TODO include link to a document to the message for the list of supported time zones
+                throw new JsonMappingException(String.format("Unknown charset '%s'", value));
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Decoders.java

@@ -0,0 +1,81 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.DecoderPlugin;
+
+public abstract class Decoders
+{
+    private Decoders() { }
+
+    public static List<DecoderPlugin> newDecoderPlugins(ExecSession exec, List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<DecoderPlugin> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(exec.newPlugin(DecoderPlugin.class, config.get(PluginType.class, "type")));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources);
+    }
+
+    public static void transaction(List<DecoderPlugin> plugins, List<ConfigSource> configs,
+            Decoders.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction();
+    }
+
+    public static FileInput open(List<DecoderPlugin> plugins, List<TaskSource> taskSources,
+            FileInput input)
+    {
+        FileInput in = input;
+        int pos = 0;
+        while (pos < plugins.size()) {
+            in = plugins.get(pos).open(taskSources.get(pos), in);
+            pos++;
+        }
+        return in;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<DecoderPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Decoders.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private int pos;
+
+        RecursiveControl(List<DecoderPlugin> plugins, List<ConfigSource> configs,
+                Decoders.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+        }
+
+        public void transaction()
+        {
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), new DecoderPlugin.Control() {
+                    public void run(TaskSource taskSource)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction();
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java

@@ -0,0 +1,81 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.EncoderPlugin;
+
+public abstract class Encoders
+{
+    private Encoders() { }
+
+    public static List<EncoderPlugin> newEncoderPlugins(ExecSession exec, List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<EncoderPlugin> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(exec.newPlugin(EncoderPlugin.class, config.get(PluginType.class, "type")));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources);
+    }
+
+    public static void transaction(List<EncoderPlugin> plugins, List<ConfigSource> configs,
+            Encoders.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction();
+    }
+
+    public static FileOutput open(List<EncoderPlugin> plugins, List<TaskSource> taskSources,
+            FileOutput output)
+    {
+        FileOutput out = output;
+        int pos = 0;
+        while (pos < plugins.size()) {
+            out = plugins.get(pos).open(taskSources.get(pos), out);
+            pos++;
+        }
+        return out;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<EncoderPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Encoders.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private int pos;
+
+        RecursiveControl(List<EncoderPlugin> plugins, List<ConfigSource> configs,
+                Encoders.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+        }
+
+        public void transaction()
+        {
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), new EncoderPlugin.Control() {
+                    public void run(TaskSource taskSource)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction();
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java

@@ -0,0 +1,110 @@
+package org.embulk.spi.util;
+
+import java.io.InputStream;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+
+public class FileInputInputStream
+        extends InputStream
+{
+    private final FileInput in;
+    private int pos;
+    private Buffer buffer = Buffer.EMPTY;
+
+    public FileInputInputStream(FileInput in)
+    {
+        this.in = in;
+    }
+
+    public boolean markSupported()
+    {
+        return false;
+    }
+
+    public boolean nextFile()
+    {
+        releaseBuffer();
+        return in.nextFile();
+    }
+
+    @Override
+    public int available()
+    {
+        return buffer.limit() - pos;
+    }
+
+    @Override
+    public int read()
+    {
+        while (pos >= buffer.limit()) {
+            if (!nextBuffer()) {
+                return -1;
+            }
+        }
+        byte b = buffer.array()[buffer.offset() + pos];
+        pos++;
+        if (pos >= buffer.limit()) {
+            releaseBuffer();
+        }
+        return b & 0xff;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len)
+    {
+        while (pos >= buffer.limit()) {
+            if (!nextBuffer()) {
+                return -1;
+            }
+        }
+        int remaining = buffer.limit() - pos;
+        boolean allConsumed;
+        if (remaining <= len) {
+            allConsumed = true;
+            len = remaining;
+        } else {
+            allConsumed = false;
+        }
+        if (b != null) {
+            // b == null if skip
+            buffer.getBytes(pos, b, off, len);
+        }
+        if (allConsumed) {
+            releaseBuffer();
+        } else {
+            pos += len;
+        }
+        return len;
+    }
+
+    @Override
+    public long skip(long len)
+    {
+        return read(null, 0, (int) Math.min(len, Integer.MAX_VALUE));
+    }
+
+    private boolean nextBuffer()
+    {
+        releaseBuffer();
+        Buffer b = in.poll();
+        if (b == null) {
+            return false;
+        }
+        buffer = b;
+        return true;
+    }
+
+    private void releaseBuffer()
+    {
+        buffer.release();
+        buffer = Buffer.EMPTY;
+        pos = 0;
+    }
+
+    @Override
+    public void close()
+    {
+        releaseBuffer();
+        in.close();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java

@@ -0,0 +1,94 @@
+package org.embulk.spi.util;
+
+import java.io.OutputStream;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.FileOutput;
+
+public class FileOutputOutputStream
+        extends OutputStream
+{
+    private final FileOutput out;
+    private final BufferAllocator allocator;
+    private int pos;
+    private Buffer buffer;
+
+    public FileOutputOutputStream(FileOutput out, BufferAllocator allocator)
+    {
+        this.out = out;
+        this.allocator = allocator;
+        this.buffer = allocator.allocate();
+    }
+
+    public void nextFile()
+    {
+        out.nextFile();
+    }
+
+    @Override
+    public void write(int b)
+    {
+        buffer.array()[buffer.offset() + pos] = (byte) b;
+        pos++;
+        if (pos >= buffer.capacity()) {
+            flush();
+        }
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len)
+    {
+        while (true) {
+            int available = buffer.capacity() - pos;
+            if (available < len) {
+                buffer.setBytes(pos, b, off, available);
+                pos += available;
+                len -= available;
+                off += available;
+                flush();
+            } else {
+                buffer.setBytes(pos, b, off, len);
+                pos += len;
+                if (available <= len) {
+                    flush();
+                }
+                break;
+            }
+        }
+    }
+
+    private boolean doFlush()
+    {
+        if (pos > 0) {
+            buffer.limit(pos);
+            out.add(buffer);
+            buffer = Buffer.EMPTY;
+            pos = 0;
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public void flush()
+    {
+        if (doFlush()) {
+            buffer = allocator.allocate();
+        }
+    }
+
+    public void finish()
+    {
+        doFlush();
+        out.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        out.close();
+        buffer.release();
+        buffer = Buffer.EMPTY;
+        pos = 0;
+    }
+}