embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -0,0 +1,112 @@
+package org.embulk.spi;
+
+import java.util.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.TimestampType;
+
+public class ColumnConfig
+{
+    private final String name;
+    private final Type type;
+    private final ConfigSource option;
+
+    @Deprecated
+    public ColumnConfig(String name, Type type, String format)
+    {
+        this.name = name;
+        this.type = type;
+        this.option = Exec.newConfigSource();  // only for backward compatibility
+        if (format != null) {
+            option.set("format", format);
+        }
+    }
+
+    public ColumnConfig(String name, Type type, ConfigSource option)
+    {
+        this.name = name;
+        this.type = type;
+        this.option = option;
+    }
+
+    @JsonCreator
+    public ColumnConfig(ConfigSource conf)
+    {
+        this.name = conf.get(String.class, "name");
+        this.type = conf.get(Type.class, "type");
+        this.option = conf.deepCopy();
+        option.remove("name");
+        option.remove("type");
+    }
+
+    public String getName()
+    {
+        return name;
+    }
+
+    public Type getType()
+    {
+        return type;
+    }
+
+    public ConfigSource getOption()
+    {
+        return option;
+    }
+
+    @JsonValue
+    public ConfigSource getConfigSource()
+    {
+        ConfigSource conf = option.deepCopy();
+        conf.set("name", name);
+        conf.set("type", type);
+        return conf;
+    }
+
+    @Deprecated
+    public String getFormat()
+    {
+        return option.get(String.class, "format", null);
+    }
+
+    public Column toColumn(int index)
+    {
+        String format = option.get(String.class, "format", null);
+        if (type instanceof TimestampType && format != null) {
+            // this behavior is only for backward compatibility. TimestampType#getFormat is @Deprecated
+            return new Column(index, name, ((TimestampType) type).withFormat(format));
+        } else {
+            return new Column(index, name, type);
+        }
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof ColumnConfig)) {
+            return false;
+        }
+        ColumnConfig other = (ColumnConfig) obj;
+        return Objects.equals(this.name, other.name) &&
+            Objects.equals(type, other.type) &&
+            Objects.equals(option, other.option);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hash(name, type);
+    }
+
+    @Override
+    public String toString()
+    {
+        return String.format("ColumnConfig[%s, %s]",
+                getName(), getType());
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/ColumnVisitor.java

@@ -0,0 +1,14 @@
+package org.embulk.spi;
+
+public interface ColumnVisitor
+{
+    void booleanColumn(Column column);
+
+    void longColumn(Column column);
+
+    void doubleColumn(Column column);
+
+    void stringColumn(Column column);
+
+    void timestampColumn(Column column);
+}

data/embulk-core/src/main/java/org/embulk/spi/DecoderPlugin.java

@@ -0,0 +1,16 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface DecoderPlugin
+{
+    interface Control
+    {
+        void run(TaskSource taskSource);
+    }
+
+    void transaction(ConfigSource config, DecoderPlugin.Control control);
+
+    FileInput open(TaskSource taskSource, FileInput input);
+}

data/embulk-core/src/main/java/org/embulk/spi/EncoderPlugin.java

@@ -0,0 +1,16 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface EncoderPlugin
+{
+    interface Control
+    {
+        void run(TaskSource taskSource);
+    }
+
+    void transaction(ConfigSource config, EncoderPlugin.Control control);
+
+    FileOutput open(TaskSource taskSource, FileOutput fileOutput);
+}

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -0,0 +1,113 @@
+package org.embulk.spi;
+
+import java.util.concurrent.ExecutionException;
+import org.slf4j.Logger;
+import com.google.inject.Injector;
+import org.embulk.config.ModelManager;
+import org.embulk.config.CommitReport;  // Deprecated
+import org.embulk.config.TaskReport;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.time.Timestamp;
+
+public class Exec
+{
+    private static final InheritableThreadLocal<ExecSession> session = new InheritableThreadLocal<ExecSession>();
+
+    private Exec() { }
+
+    public static <T> T doWith(ExecSession session, ExecAction<T> action) throws ExecutionException
+    {
+        Exec.session.set(session);
+        try {
+            return action.run();
+        } catch (Exception ex) {
+            throw new ExecutionException(ex);
+        } finally {
+            Exec.session.set(null);
+        }
+    }
+
+    public static ExecSession session()
+    {
+        ExecSession session = Exec.session.get();
+        if (session == null) {
+            throw new NullPointerException("Exec is used outside of Exec.doWith");
+        }
+        return session;
+    }
+
+    public static Injector getInjector()
+    {
+        return session().getInjector();
+    }
+
+    public static Timestamp getTransactionTime()
+    {
+        return session().getTransactionTime();
+    }
+
+    public static Logger getLogger(String name)
+    {
+        return session().getLogger(name);
+    }
+
+    public static Logger getLogger(Class<?> name)
+    {
+        return session().getLogger(name);
+    }
+
+    public static BufferAllocator getBufferAllocator()
+    {
+        return session().getBufferAllocator();
+    }
+
+    public static ModelManager getModelManager()
+    {
+        return session().getModelManager();
+    }
+
+    public static <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        return session().newPlugin(iface, type);
+    }
+
+    public static TaskReport newTaskReport()
+    {
+        return session().newTaskReport();
+    }
+
+    @Deprecated
+    public static CommitReport newCommitReport()
+    {
+        return session().newCommitReport();
+    }
+
+    public static ConfigDiff newConfigDiff()
+    {
+        return session().newConfigDiff();
+    }
+
+    public static ConfigSource newConfigSource()
+    {
+        return session().newConfigSource();
+    }
+
+    public static TaskSource newTaskSource()
+    {
+        return session().newTaskSource();
+    }
+
+    // TODO this method is still beta
+    public static TempFileSpace getTempFileSpace()
+    {
+        return session().getTempFileSpace();
+    }
+
+    public static boolean isPreview()
+    {
+        return session().isPreview();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java

@@ -0,0 +1,6 @@
+package org.embulk.spi;
+
+public interface ExecAction <T>
+{
+    T run() throws Exception;
+}

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -0,0 +1,217 @@
+package org.embulk.spi;
+
+import org.joda.time.DateTimeZone;
+import org.slf4j.Logger;
+import org.slf4j.ILoggerFactory;
+import com.google.common.base.Optional;
+import com.google.inject.Injector;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ModelManager;
+import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.exec.TempFileAllocator;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.PluginManager;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampFormatter.FormatterTask;
+
+public class ExecSession
+{
+    private final Injector injector;
+    private final ILoggerFactory loggerFactory;
+    private final ModelManager modelManager;
+    private final PluginManager pluginManager;
+    private final BufferAllocator bufferAllocator;
+
+    private final Timestamp transactionTime;
+    private final TempFileSpace tempFileSpace;
+
+    private final boolean preview;
+
+    @Deprecated
+    public interface SessionTask
+            extends Task
+    {
+        @Config("transaction_time")
+        @ConfigDefault("null")
+        Optional<Timestamp> getTransactionTime();
+    }
+
+    public static class Builder
+    {
+        private final Injector injector;
+        private Timestamp transactionTime;
+
+        public Builder(Injector injector)
+        {
+            this.injector = injector;
+        }
+
+        public Builder fromExecConfig(ConfigSource configSource)
+        {
+            this.transactionTime = configSource.get(Timestamp.class, "transaction_time", null);
+            return this;
+        }
+
+        public Builder setTransactionTime(Timestamp timestamp)
+        {
+            this.transactionTime = timestamp;
+            return this;
+        }
+
+        public ExecSession build()
+        {
+            if (transactionTime == null) {
+                transactionTime = Timestamp.ofEpochMilli(System.currentTimeMillis());  // TODO get nanoseconds for default
+            }
+            return new ExecSession(injector, transactionTime);
+        }
+    }
+
+    public static Builder builder(Injector injector)
+    {
+        return new Builder(injector);
+    }
+
+    @Deprecated
+    public ExecSession(Injector injector, ConfigSource configSource)
+    {
+        this(injector,
+                configSource.loadConfig(SessionTask.class).getTransactionTime().or(
+                    Timestamp.ofEpochMilli(System.currentTimeMillis())
+                )
+            );  // TODO get nanoseconds for default
+    }
+
+    private ExecSession(Injector injector, Timestamp transactionTime)
+    {
+        this.injector = injector;
+        this.loggerFactory = injector.getInstance(ILoggerFactory.class);
+        this.modelManager = injector.getInstance(ModelManager.class);
+        this.pluginManager = injector.getInstance(PluginManager.class);
+        this.bufferAllocator = injector.getInstance(BufferAllocator.class);
+
+        this.transactionTime = transactionTime;
+
+        TempFileAllocator tempFileAllocator = injector.getInstance(TempFileAllocator.class);
+        this.tempFileSpace = tempFileAllocator.newSpace(transactionTime.toString());
+
+        this.preview = false;
+    }
+
+    private ExecSession(ExecSession copy, boolean preview)
+    {
+        this.injector = copy.injector;
+        this.loggerFactory = copy.loggerFactory;
+        this.modelManager = copy.modelManager;
+        this.pluginManager = copy.pluginManager;
+        this.bufferAllocator = copy.bufferAllocator;
+
+        this.transactionTime = copy.transactionTime;
+        this.tempFileSpace = copy.tempFileSpace;
+
+        this.preview = preview;
+    }
+
+    public ExecSession forPreview()
+    {
+        return new ExecSession(this, true);
+    }
+
+    public ConfigSource getSessionExecConfig()
+    {
+        return newConfigSource()
+            .set("transaction_time", transactionTime);
+    }
+
+    public Injector getInjector()
+    {
+        return injector;
+    }
+
+    public Timestamp getTransactionTime()
+    {
+        return transactionTime;
+    }
+
+    public Logger getLogger(String name)
+    {
+        return loggerFactory.getLogger(name);
+    }
+
+    public Logger getLogger(Class<?> name)
+    {
+        return loggerFactory.getLogger(name.getName());
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return bufferAllocator;
+    }
+
+    public ModelManager getModelManager()
+    {
+        return modelManager;
+    }
+
+    public <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        return pluginManager.newPlugin(iface, type);
+    }
+
+    public TaskReport newTaskReport()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    @Deprecated
+    public CommitReport newCommitReport()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public ConfigDiff newConfigDiff()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public ConfigSource newConfigSource()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public TaskSource newTaskSource()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    public TimestampFormatter newTimestampFormatter(String format, DateTimeZone timezone)
+    {
+        ConfigSource config = Exec.newConfigSource();
+        config.set("timezone", timezone.getID());
+        FormatterTask formatterTask = config.loadConfig(FormatterTask.class);
+        return new TimestampFormatter(format, formatterTask);
+    }
+
+    public TempFileSpace getTempFileSpace()
+    {
+        return tempFileSpace;
+    }
+
+    public boolean isPreview()
+    {
+        return preview;
+    }
+
+    public void cleanup()
+    {
+        tempFileSpace.cleanup();
+    }
+}