embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/BufferAllocator.java

@@ -0,0 +1,8 @@
+package org.embulk.spi;
+
+public interface BufferAllocator
+{
+    public Buffer allocate();
+
+    public Buffer allocate(int minimumCapacity);
+}

data/embulk-core/src/main/java/org/embulk/spi/Column.java

@@ -0,0 +1,92 @@
+package org.embulk.spi;
+
+import com.google.common.base.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.StringType;
+
+public class Column
+{
+    private final int index;
+    private final String name;
+    private final Type type;
+
+    @JsonCreator
+    public Column(
+            @JsonProperty("index") int index,
+            @JsonProperty("name") String name,
+            @JsonProperty("type") Type type)
+    {
+        this.index = index;
+        this.name = name;
+        this.type = type;
+    }
+
+    @JsonProperty("index")
+    public int getIndex()
+    {
+        return index;
+    }
+
+    @JsonProperty("name")
+    public String getName()
+    {
+        return name;
+    }
+
+    @JsonProperty("type")
+    public Type getType()
+    {
+        return type;
+    }
+
+    public void visit(SchemaVisitor visitor)
+    {
+        if (type instanceof BooleanType) {
+            visitor.booleanColumn(this);
+        } else if (type instanceof LongType) {
+            visitor.longColumn(this);
+        } else if (type instanceof DoubleType) {
+            visitor.doubleColumn(this);
+        } else if (type instanceof StringType) {
+            visitor.stringColumn(this);
+        } else if (type instanceof TimestampType) {
+            visitor.timestampColumn(this);
+        } else {
+            assert(false);
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof Column)) {
+            return false;
+        }
+        Column other = (Column) obj;
+        return Objects.equal(index, other.index) &&
+            Objects.equal(name, other.name) &&
+            Objects.equal(type, other.type);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(index, name, type);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("Column{index:%d, name:%s, type:%s}",
+                getIndex(), getName(), getType().getName());
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -0,0 +1,79 @@
+package org.embulk.spi;
+
+import com.google.common.base.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.TimestampType;
+
+public class ColumnConfig
+{
+    private final String name;
+    private final Type type;
+    private final String format;
+
+    @JsonCreator
+    public ColumnConfig(
+            @JsonProperty("name") String name,
+            @JsonProperty("type") Type type,
+            @JsonProperty("format") String format)
+    {
+        this.name = name;
+        this.type = type;
+        this.format = format;
+    }
+
+    @JsonProperty("name")
+    public String getName()
+    {
+        return name;
+    }
+
+    @JsonProperty("type")
+    public Type getType()
+    {
+        return type;
+    }
+
+    @JsonProperty("format")
+    public String getFormat()
+    {
+        return format;
+    }
+
+    public Column toColumn(int index)
+    {
+        if (type instanceof TimestampType && format != null) {
+            return new Column(index, name, ((TimestampType) type).withFormat(format));
+        } else {
+            return new Column(index, name, type);
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ColumnConfig)) {
+            return false;
+        }
+        ColumnConfig other = (ColumnConfig) obj;
+        return Objects.equal(this.name, other.name) &&
+            Objects.equal(type, other.type);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(name, type);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("ColumnConfig[%s, %s]",
+                getName(), getType());
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java

@@ -0,0 +1,16 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface DecoderPlugin
+{
+    public interface Control
+    {
+        public void run(TaskSource taskSource);
+    }
+
+    public void transaction(ConfigSource config, DecoderPlugin.Control control);
+
+    public FileInput open(TaskSource taskSource, FileInput input);
+}

data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java

@@ -0,0 +1,16 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface EncoderPlugin
+{
+    public interface Control
+    {
+        public void run(TaskSource taskSource);
+    }
+
+    public void transaction(ConfigSource config, EncoderPlugin.Control control);
+
+    public FileOutput open(TaskSource taskSource, FileOutput fileOutput);
+}

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -0,0 +1,76 @@
+package org.embulk.spi;
+
+import org.slf4j.Logger;
+import org.embulk.config.Task;
+import org.embulk.config.CommitReport;
+import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.plugin.PluginType;
+
+public class Exec
+{
+    private static final InheritableThreadLocal<ExecSession> session = new InheritableThreadLocal<ExecSession>();
+
+    private Exec() { }
+
+    public static <T> T doWith(ExecSession session, ExecAction<T> action) throws Exception
+    {
+        Exec.session.set(session);
+        try {
+            return action.run();
+        } finally {
+            Exec.session.set(null);
+        }
+    }
+
+    public static ExecSession session()
+    {
+        ExecSession session = Exec.session.get();
+        if (session == null) {
+            new NullPointerException().printStackTrace();
+            throw new NullPointerException("Exec is used outside of Exec.doWith");
+        }
+        return session;
+    }
+
+    public static Logger getLogger(String name)
+    {
+        return session().getLogger(name);
+    }
+
+    public static Logger getLogger(Class<?> name)
+    {
+        return session().getLogger(name);
+    }
+
+    public static BufferAllocator getBufferAllocator()
+    {
+        return session().getBufferAllocator();
+    }
+
+    public static <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        return session().newPlugin(iface, type);
+    }
+
+    public static CommitReport newCommitReport()
+    {
+        return session().newCommitReport();
+    }
+
+    public static NextConfig newNextConfig()
+    {
+        return session().newNextConfig();
+    }
+
+    public static ConfigSource newConfigSource()
+    {
+        return session().newConfigSource();
+    }
+
+    public static TaskSource newTaskSource()
+    {
+        return session().newTaskSource();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java

@@ -0,0 +1,6 @@
+package org.embulk.spi;
+
+public interface ExecAction <T>
+{
+    public T run();
+}

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -0,0 +1,105 @@
+package org.embulk.spi;
+
+import org.joda.time.DateTimeZone;
+import org.slf4j.Logger;
+import org.slf4j.ILoggerFactory;
+import com.google.inject.Injector;
+import org.embulk.config.ModelManager;
+import org.embulk.config.CommitReport;
+import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.PluginManager;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampFormatter.FormatterTask;
+
+public class ExecSession
+{
+    private final Injector injector;
+    private final ILoggerFactory loggerFactory;
+    private final ModelManager modelManager;
+    private final PluginManager pluginManager;
+    private final BufferAllocator bufferAllocator;
+    private final Timestamp transactionTime;
+    private final DateTimeZone transactionTimeZone;
+
+    public ExecSession(Injector injector, ConfigSource execConfig)
+    {
+        super();
+        this.injector = injector;
+        this.loggerFactory = injector.getInstance(ILoggerFactory.class);
+        this.modelManager = injector.getInstance(ModelManager.class);
+        this.pluginManager = injector.getInstance(PluginManager.class);
+        this.bufferAllocator = injector.getInstance(BufferAllocator.class);
+
+        this.transactionTime = execConfig.get(Timestamp.class, "transaction_time",
+                Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
+        this.transactionTimeZone = execConfig.get(DateTimeZone.class, "transaction_time_zone", DateTimeZone.UTC);
+    }
+
+    public Injector getInjector()
+    {
+        return injector;
+    }
+
+    public Timestamp getTransactionTime()
+    {
+        return transactionTime;
+    }
+
+    public DateTimeZone getTransactionTimeZone()
+    {
+        return transactionTimeZone;
+    }
+
+    public Logger getLogger(String name)
+    {
+        return loggerFactory.getLogger(name);
+    }
+
+    public Logger getLogger(Class<?> name)
+    {
+        return loggerFactory.getLogger(name.getName());
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return bufferAllocator;
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        return pluginManager.newPlugin(iface, type);
+    }
+
+    public CommitReport newCommitReport()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public NextConfig newNextConfig()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public ConfigSource newConfigSource()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public TaskSource newTaskSource()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public TimestampFormatter newTimestampFormatter(String format, DateTimeZone timezone)
+    {
+        ConfigSource config = Exec.newConfigSource();
+        config.set("timezone", timezone.getID());
+        FormatterTask formatterTask = config.loadConfig(FormatterTask.class);
+        return new TimestampFormatter(format, formatterTask);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/Extension.java

@@ -0,0 +1,42 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+
+/**
+ * Extension is a module to extend the execution framework using Guice.
+ * Unlike plugins, extensions can overwrite or add core components such as
+ * BufferManager, PluginSource, etc.
+ * Extension is not designed for users but for framework developpers to make
+ * core components loosely coupled.
+ *
+ * An example extention to add a custom PluginSource will be as following:
+ *
+ * class MyPluginSourceExtension
+ *         implements Extension, Module
+ * {
+ *     public static class MyPluginSource
+ *             implements PluginSource
+ *     {
+ *         // ...
+ *     }
+ *
+ *     @Override
+ *     public void configure(Binder binder)
+ *     {
+ *         Multibinder<PluginSource> multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+ *         multibinder.addBinding().to(MyPluginSource.class);
+ *     }
+ *
+ *     @Override
+ *     public List<Module> getModules()
+ *     {
+ *         return ImmutableList.<Module>of(this);
+ *     }
+ * }
+ */
+public interface Extension
+{
+    public List<Module> getModules(ConfigSource systemConfig);
+}