embulk 0.8.38-java → 0.10.27-java

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -1,199 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.TaskReport;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.util.Pages;
-
-public class TestFileInputRunner
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    @Before
-    public void tearDown()
-    {
-        MockParserPlugin.raiseException = false;
-    }
-
-    private static class MockFileInputPlugin implements FileInputPlugin
-    {
-        Boolean transactionCompleted = null;
-        Queue<Buffer> buffers;
-
-        public MockFileInputPlugin(Queue<Buffer> buffers)
-        {
-            this.buffers = buffers;
-        }
-
-        @Override
-        public ConfigDiff transaction(ConfigSource config,
-                FileInputPlugin.Control control)
-        {
-            control.run(Exec.newTaskSource(), 1);
-            return null;
-        }
-
-        @Override
-        public ConfigDiff resume(TaskSource taskSource,
-                int taskCount,
-                FileInputPlugin.Control control)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void cleanup(TaskSource taskSource,
-                int taskCount,
-                List<TaskReport> successTaskReports)
-        {
-        }
-
-        public TransactionalFileInput open(TaskSource taskSource,
-                int taskIndex)
-        {
-            return new TransactionalFileInput()
-            {
-                @Override
-                public Buffer poll()
-                {
-                    return buffers.poll();
-                }
-
-                @Override
-                public boolean nextFile()
-                {
-                    return !buffers.isEmpty();
-                }
-
-                @Override
-                public void close()
-                {
-                }
-
-                @Override
-                public void abort()
-                {
-                    transactionCompleted = false;
-                }
-
-                @Override
-                public TaskReport commit()
-                {
-                    transactionCompleted = true;
-                    return null;
-                }
-            };
-        }
-    }
-
-    @Test
-    public void testMockParserIteration()
-    {
-        Buffer[] buffers = new Buffer[] {
-                runtime.getBufferAllocator().allocate(),
-                runtime.getBufferAllocator().allocate() };
-        MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(
-                new LinkedList<Buffer>(Arrays.asList(buffers)));
-        final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
-
-        ConfigSource config = Exec.newConfigSource().set(
-                "parser",
-                ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
-
-        final MockPageOutput output = new MockPageOutput();
-        runner.transaction(config, new InputPlugin.Control()
-        {
-            public List<TaskReport> run(TaskSource inputTaskSource,
-                    Schema schema, int taskCount)
-            {
-                List<TaskReport> reports = new ArrayList<>();
-                reports.add(runner.run(inputTaskSource, schema, 0, output));
-                return reports;
-            }
-        });
-
-        assertEquals(true, fileInputPlugin.transactionCompleted);
-        assertEquals(1, output.pages.size());
-
-        Schema schema = config.getNested("parser")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        List<Object[]> records = Pages.toObjects(schema, output.pages);
-        assertEquals(2, records.size());
-        for (Object[] record : records) {
-            assertEquals(6, record.length);
-            assertEquals(true, record[0]);
-            assertEquals(2L, record[1]);
-            assertEquals(3.0D, (Double) record[2], 0.01D);
-            assertEquals("45", record[3]);
-            assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
-            assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
-        }
-    }
-
-    @Test
-    public void testTransactionAborted()
-    {
-        Buffer[] buffers = new Buffer[] {
-                runtime.getBufferAllocator().allocate(),
-                runtime.getBufferAllocator().allocate() };
-        MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(
-                new LinkedList<Buffer>(Arrays.asList(buffers)));
-        final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
-
-        ConfigSource config = Exec.newConfigSource().set(
-                "parser",
-                ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
-
-        final MockPageOutput output = new MockPageOutput();
-
-        MockParserPlugin.raiseException = true;
-
-        try {
-            runner.transaction(config, new InputPlugin.Control()
-            {
-                public List<TaskReport> run(TaskSource inputTaskSource,
-                        Schema schema, int taskCount)
-                {
-                    List<TaskReport> reports = new ArrayList<>();
-                    reports.add(runner.run(inputTaskSource, schema, 0, output));
-                    return reports;
-                }
-            });
-        } catch (RuntimeException re) {
-        }
-        assertEquals(false, fileInputPlugin.transactionCompleted);
-        assertEquals(0, output.pages.size());
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -1,221 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.TaskReport;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.Schema;
-import org.junit.Rule;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.msgpack.value.ImmutableMapValue;
-import static org.msgpack.value.ValueFactory.newBoolean;
-import static org.msgpack.value.ValueFactory.newInteger;
-import static org.msgpack.value.ValueFactory.newMap;
-import static org.msgpack.value.ValueFactory.newString;
-
-public class TestFileOutputRunner
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    public interface PluginTask extends Task
-    {
-    }
-
-    private static class MockFileOutputPlugin implements FileOutputPlugin
-    {
-        Boolean transactionCompleted = null;
-
-        @Override
-        public ConfigDiff transaction(ConfigSource config, int taskCount,
-                FileOutputPlugin.Control control)
-        {
-            PluginTask task = config.loadConfig(PluginTask.class);
-            control.run(task.dump());
-            return Exec.newConfigDiff();
-        }
-
-        @Override
-        public ConfigDiff resume(TaskSource taskSource,
-                int taskCount,
-                FileOutputPlugin.Control control)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void cleanup(TaskSource taskSource,
-                int taskCount,
-                List<TaskReport> successTaskReports)
-        {
-        }
-
-        @Override
-        public TransactionalFileOutput open(TaskSource taskSource,
-                final int taskIndex)
-        {
-            return new TransactionalFileOutput()
-            {
-
-                @Override
-                public void nextFile()
-                {
-                }
-
-                @Override
-                public void add(Buffer buffer)
-                {
-                }
-
-                @Override
-                public void finish()
-                {
-                }
-
-                @Override
-                public void close()
-                {
-                }
-
-                @Override
-                public void abort()
-                {
-                    transactionCompleted = false;
-                }
-
-                @Override
-                public TaskReport commit()
-                {
-                    transactionCompleted = true;
-                    return Exec.newTaskReport();
-                }
-            };
-        }
-    }
-
-    @Test
-    public void testMockFormatterIteration()
-    {
-        MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
-        final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
-
-        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
-                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
-        ConfigSource config = Exec
-                .newConfigSource()
-                .set("type", "unused?")
-                .set("formatter",
-                        ImmutableMap.of("type", "mock", "columns", columns));
-        final Schema schema = config.getNested("formatter")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        runner.transaction(config, schema, 1, new OutputPlugin.Control()
-        {
-            public List<TaskReport> run(final TaskSource outputTask)
-            {
-                TransactionalPageOutput tran = runner.open(outputTask, schema,
-                        1);
-                boolean committed = false;
-                try {
-                    ImmutableMapValue jsonValue = newMap(
-                            newString("_c1"), newBoolean(true),
-                            newString("_c2"), newInteger(10),
-                            newString("_c3"), newString("embulk"),
-                            newString("_c4"), newMap(newString("k"), newString("v"))
-                    );
-                    for (Page page : PageTestUtils.buildPage(
-                            runtime.getBufferAllocator(), schema, true, 2L,
-                            3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L,
-                            3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) {
-                        tran.add(page);
-                    }
-                    tran.commit();
-                    committed = true;
-                } finally {
-                    if (!committed) {
-                        tran.abort();
-                    }
-                    tran.close();
-                }
-                return new ArrayList<TaskReport>();
-            }
-        });
-
-        assertEquals(true, fileOutputPlugin.transactionCompleted);
-        assertEquals(2, MockFormatterPlugin.records.size());
-        for (List<Object> record : MockFormatterPlugin.records) {
-            assertEquals(Boolean.TRUE, record.get(0));
-            assertEquals(2L, record.get(1));
-            assertEquals(3.0D, (Double) record.get(2), 0.1D);
-            assertEquals("45", record.get(3));
-            assertEquals(678L, ((Timestamp) record.get(4)).toEpochMilli());
-            assertEquals("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}", record.get(5).toString());
-        }
-    }
-
-    @Test
-    public void testTransactionAborted()
-    {
-        MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
-        final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
-
-        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
-                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
-        ConfigSource config = Exec
-                .newConfigSource()
-                .set("type", "unused?")
-                .set("formatter",
-                        ImmutableMap.of("type", "mock", "columns", columns));
-        final Schema schema = config.getNested("formatter")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        try {
-            runner.transaction(config, schema, 1, new OutputPlugin.Control()
-            {
-                public List<TaskReport> run(final TaskSource outputTask)
-                {
-                    TransactionalPageOutput tran = runner.open(outputTask,
-                            schema, 1);
-                    boolean committed = false;
-                    try {
-                        tran.add(null);
-                        tran.commit();
-                        committed = true;
-                    } finally {
-                        if (!committed) {
-                            tran.abort();
-                        }
-                        tran.close();
-                    }
-                    return new ArrayList<TaskReport>();
-                }
-            });
-        } catch (NullPointerException npe) {
-        }
-
-        assertEquals(false, fileOutputPlugin.transactionCompleted);
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestInputStreamFileInput.java

@@ -1,188 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import org.junit.Rule;
-import org.junit.Test;
-import com.google.common.collect.ImmutableList;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.spi.util.InputStreamFileInput;
-
-public class TestInputStreamFileInput
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    @Test
-    public void testSingleProvider() throws IOException
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                provider(new ByteArrayInputStream("abcdef".getBytes("UTF-8"))));
-        assertEquals(true, subject.nextFile());
-        assertEquals("abcdef", bufferToString(subject.poll()));
-        assertEquals(null, subject.poll());
-        subject.close();
-    }
-
-    @Test
-    public void testMultipleProvider() throws IOException
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(), provider(
-                        new ByteArrayInputStream("abcdef".getBytes("UTF-8")),
-                        new ByteArrayInputStream("ghijkl".getBytes("UTF-8")),
-                        new ByteArrayInputStream("mnopqr".getBytes("UTF-8"))));
-        assertEquals(true, subject.nextFile());
-        assertEquals("abcdef", bufferToString(subject.poll()));
-        assertEquals(true, subject.nextFile());
-        assertEquals("ghijkl", bufferToString(subject.poll()));
-        assertEquals(true, subject.nextFile());
-        assertEquals("mnopqr", bufferToString(subject.poll()));
-        subject.close();
-    }
-
-    @Test
-    public void testEmptyStream() throws IOException
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                provider(new ByteArrayInputStream(new byte[0])));
-        assertEquals(true, subject.nextFile());
-        assertEquals(null, subject.poll());
-        subject.close();
-    }
-
-    @Test
-    public void testPollFirstException() throws IOException
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                provider(new ByteArrayInputStream("abcdef".getBytes("UTF-8"))));
-        try {
-            subject.poll();
-            fail();
-        } catch (IllegalStateException ile) {
-            // OK
-        }
-        subject.close();
-    }
-
-    @Test
-    public void testEmptyProvider() throws IOException
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(), provider(new InputStream[0]));
-        assertEquals(false, subject.nextFile());
-        subject.close();
-    }
-
-    @Test
-    public void testProviderOpenNextException()
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                new InputStreamFileInput.Provider()
-                {
-                    @Override
-                    public InputStream openNext() throws IOException
-                    {
-                        throw new IOException("emulated exception");
-                    }
-
-                    @Override
-                    public void close() throws IOException
-                    {
-                    }
-                });
-
-        try {
-            subject.nextFile();
-            fail();
-        } catch (RuntimeException re) {
-            // OK
-        }
-        subject.close();
-    }
-
-    @Test
-    public void testProviderCloseException()
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                new InputStreamFileInput.Provider()
-                {
-                    @Override
-                    public InputStream openNext() throws IOException
-                    {
-                        return new ByteArrayInputStream(new byte[0]);
-                    }
-
-                    @Override
-                    public void close() throws IOException
-                    {
-                        throw new IOException("emulated exception");
-                    }
-                });
-
-        try {
-            subject.close();
-            fail();
-        } catch (RuntimeException re) {
-            // OK
-        }
-    }
-
-    @Test
-    public void testInputStreamReadException()
-    {
-        InputStreamFileInput subject = new InputStreamFileInput(
-                runtime.getBufferAllocator(),
-                new InputStreamFileInput.Provider()
-                {
-                    @Override
-                    public InputStream openNext() throws IOException
-                    {
-                        return new InputStream()
-                        {
-                            @Override
-                            public int read() throws IOException
-                            {
-                                throw new IOException("emulated exception");
-                            }
-                        };
-                    }
-
-                    @Override
-                    public void close() throws IOException
-                    {
-                    }
-                });
-
-        assertEquals(true, subject.nextFile());
-        try {
-            subject.poll();
-            fail();
-        } catch (RuntimeException re) {
-            // OK
-        }
-        subject.close();
-    }
-
-    private InputStreamFileInput.IteratorProvider provider(
-            InputStream... inputStreams) throws IOException
-    {
-        return new InputStreamFileInput.IteratorProvider(
-                ImmutableList.copyOf(inputStreams));
-    }
-
-    private String bufferToString(Buffer buffer) throws IOException
-    {
-        byte[] buf = new byte[buffer.limit()];
-        buffer.getBytes(0, buf, 0, buffer.limit());
-        return new String(buf, "UTF-8");
-    }
-}