embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java

@@ -0,0 +1,38 @@
+package org.embulk.spi.util;
+
+public enum Newline
+{
+    CRLF("\r\n"),
+    LF("\n"),
+    CR("\r");
+
+    private final String string;
+    private final char firstCharCode;
+    private final char secondCharCode;
+
+    private Newline(String string)
+    {
+        this.string = string;
+        this.firstCharCode = string.charAt(0);
+        if (string.length() > 1) {
+            this.secondCharCode = string.charAt(1);
+        } else {
+            this.secondCharCode = 0;
+        }
+    }
+
+    public String getString()
+    {
+        return string;
+    }
+
+    public char getFirstCharCode()
+    {
+        return firstCharCode;
+    }
+
+    public char getSecondCharCode()
+    {
+        return secondCharCode;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java

@@ -0,0 +1,102 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.type.TimestampType;
+
+public class PagePrinter
+{
+    private final Schema schema;
+    private final TimestampFormatter[] timestampFormatters;
+    private final ArrayList<String> record;
+
+    public PagePrinter(Schema schema, TimestampFormatter.FormatterTask task)
+    {
+        this.schema = schema;
+        this.timestampFormatters = new TimestampFormatter[schema.getColumnCount()];
+        for (int i=0; i < timestampFormatters.length; i++) {
+            if (schema.getColumnType(i) instanceof TimestampType) {
+                TimestampType type = (TimestampType) schema.getColumnType(i);
+                timestampFormatters[i] = new TimestampFormatter(type.getFormat(), task);
+            }
+        }
+
+        this.record = new ArrayList(schema.getColumnCount());
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            record.add("");
+        }
+    }
+
+    public String printRecord(PageReader reader, String delimiter)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (Column c : schema.getColumns()) {
+            if (c.getIndex() != 0) {
+                sb.append(delimiter);
+            }
+            sb.append(printColumn(reader, c));
+        }
+        return sb.toString();
+    }
+
+    public List<String> printRecord(PageReader reader)
+    {
+        for (Column c : schema.getColumns()) {
+            record.set(c.getIndex(), printColumn(reader, c));
+        }
+        return record;
+    }
+
+    public String printColumn(PageReader reader, Column column)
+    {
+        if (reader.isNull(column)) {
+            return "";
+        }
+        ToStringSchemaVisitor visitor = new ToStringSchemaVisitor(reader);
+        column.visit(visitor);
+        return visitor.string;
+    }
+
+    private class ToStringSchemaVisitor
+            implements SchemaVisitor
+    {
+        private final PageReader reader;
+        String string = "";
+
+        public ToStringSchemaVisitor(PageReader reader)
+        {
+            this.reader = reader;
+        }
+
+        public void booleanColumn(Column column)
+        {
+            string = Boolean.toString(reader.getBoolean(column));
+        }
+
+        public void longColumn(Column column)
+        {
+            string = Long.toString(reader.getLong(column));
+        }
+
+        public void doubleColumn(Column column)
+        {
+            string = Double.toString(reader.getDouble(column));
+        }
+
+        public void stringColumn(Column column)
+        {
+            string = reader.getString(column);
+        }
+
+        public void timestampColumn(Column column)
+        {
+            string = timestampFormatters[column.getIndex()].format(reader.getTimestamp(column));
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java

@@ -0,0 +1,139 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.Iterator;
+import com.google.common.collect.ImmutableList;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+
+public class Pages
+{
+    public static List<Object[]> toObjects(Schema schema, Page page)
+    {
+        return toObjects(schema, ImmutableList.of(page));
+    }
+
+    // TODO use streaming and return Iterable
+    public static List<Object[]> toObjects(Schema schema, Iterable<Page> pages)
+    {
+        ImmutableList.Builder<Object[]> builder = ImmutableList.builder();
+        Iterator<Page> ite = pages.iterator();
+        try (PageReader reader = new PageReader(schema)) {
+            while (ite.hasNext()) {
+                reader.setPage(ite.next());
+                while (reader.nextRecord()) {
+                    builder.add(toObjects(reader));
+                }
+            }
+        }
+        return builder.build();
+    }
+
+    public static Object[] toObjects(final PageReader record)
+    {
+        final Object[] values = new Object[record.getSchema().getColumns().size()];
+        record.getSchema().visitColumns(new ObjectSchemaVisitor(record) {
+            @Override
+            public void visit(Column column, Object object)
+            {
+                values[column.getIndex()] = object;
+            }
+        });
+        return values;
+    }
+
+    public static abstract class ObjectSchemaVisitor
+            implements SchemaVisitor
+    {
+        private final PageReader record;
+
+        public ObjectSchemaVisitor(PageReader record)
+        {
+            this.record = record;
+        }
+
+        public abstract void visit(Column column, Object obj);
+
+        @Override
+        public void booleanColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getBoolean(column));
+            }
+        }
+
+        @Override
+        public void longColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getLong(column));
+            }
+        }
+
+        @Override
+        public void doubleColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getDouble(column));
+            }
+        }
+
+        @Override
+        public void stringColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getString(column));
+            }
+        }
+
+        @Override
+        public void timestampColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getTimestamp(column));
+            }
+        }
+    }
+
+    public static Object getObject(PageReader record, Column column)
+    {
+        GetObjectColumnVisitor visitor = new GetObjectColumnVisitor(record);
+        column.visit(visitor);
+        return visitor.get();
+    }
+
+    private static class GetObjectColumnVisitor
+            extends ObjectSchemaVisitor
+    {
+        private Object object;
+
+        public GetObjectColumnVisitor(PageReader record)
+        {
+            super(record);
+        }
+
+        public Object get()
+        {
+            return object;
+        }
+
+        public void visit(Column column, Object object)
+        {
+            this.object = object;
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java

@@ -0,0 +1,110 @@
+package org.embulk;
+
+import java.util.Random;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.config.ModelManager;
+import org.embulk.exec.SystemConfigModule;
+import org.embulk.exec.ExecModule;
+import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.jruby.JRubyScriptingModule;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+
+public class EmbulkTestRuntime
+        extends GuiceBinder
+{
+    private static ConfigSource getSystemConfig()
+    {
+        // TODO set some default values
+        return new DataSourceImpl(null);
+    }
+
+    public static class TestRuntimeModule
+            implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            ConfigSource systemConfig = getSystemConfig();
+            new SystemConfigModule(systemConfig).configure(binder);
+            new ExecModule().configure(binder);
+            new ExtensionServiceLoaderModule(systemConfig).configure(binder);
+            new BuiltinPluginSourceModule().configure(binder);
+            new JRubyScriptingModule(systemConfig).configure(binder);
+            new TestUtilityModule().configure(binder);
+            new TestPluginSourceModule().configure(binder);
+        }
+    }
+
+    private ExecSession exec;
+
+    public EmbulkTestRuntime()
+    {
+        super(new TestRuntimeModule());
+        ConfigSource execConfig = new DataSourceImpl(null);
+        this.exec = new ExecSession(getInjector(), execConfig);
+    }
+
+    public ExecSession getExec()
+    {
+        return exec;
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return getInstance(BufferAllocator.class);
+    }
+
+    public ModelManager getModelManager()
+    {
+        return getInstance(ModelManager.class);
+    }
+
+    public Random getRandom()
+    {
+        return getInstance(RandomManager.class).getRandom();
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        final Statement superStatement = EmbulkTestRuntime.super.apply(base, description);
+        return new Statement() {
+            public void evaluate() throws Throwable
+            {
+                try {
+                    Exec.doWith(exec, new ExecAction<Void>() {
+                        public Void run()
+                        {
+                            try {
+                                superStatement.evaluate();
+                            } catch (Throwable ex) {
+                                throw new RuntimeExecutionException(ex);
+                            }
+                            return null;
+                        }
+                    });
+                } catch (RuntimeException ex) {
+                    throw ex.getCause();
+                }
+            }
+        };
+    }
+
+    private static class RuntimeExecutionException
+            extends RuntimeException
+    {
+        public RuntimeExecutionException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/GuiceBinder.java

@@ -0,0 +1,72 @@
+package org.embulk;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.rules.TestRule;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+
+public class GuiceBinder
+        implements TestRule
+{
+    private final List<Module> baseModules;
+    private List<Module> extraModules;
+    private Injector injector;
+
+    public GuiceBinder(Module... baseModules)
+    {
+        this.baseModules = ImmutableList.copyOf(baseModules);
+        reset();
+    }
+
+    private void reset()
+    {
+        extraModules = new ArrayList<Module>();
+        injector = null;
+    }
+
+    public synchronized void addModule(Module module)
+    {
+        if (injector != null) {
+            throw new IllegalStateException("Injector is already initialized. Call addModule before getInjector or getInstance");
+        }
+        extraModules.add(module);
+    }
+
+    public synchronized Injector getInjector()
+    {
+        if (injector == null) {
+            ImmutableList.Builder modules = ImmutableList.builder();
+            modules.addAll(baseModules);
+            modules.addAll(extraModules);
+            injector = Guice.createInjector(modules.build());
+        }
+        return injector;
+    }
+
+    public <T> T getInstance(Class<T> klass)
+    {
+        return getInjector().getInstance(klass);
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        return new GuceBinderWatcher().apply(base, description);
+    }
+
+    private class GuceBinderWatcher
+            extends TestWatcher
+    {
+        @Override
+        protected void starting(Description description)
+        {
+            reset();
+        }
+    }
+}