embulk 0.8.39-java → 0.10.24-java

data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java

@@ -1,59 +0,0 @@
-package org.embulk.spi;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnConfig;
-import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.type.Type;
-import org.msgpack.value.Value;
-import com.google.common.collect.ImmutableList;
-
-public class PageTestUtils
-{
-    private PageTestUtils()
-    { }
-
-    public static List<Page> buildPage(BufferAllocator bufferAllocator,
-            Schema schema, Object... values)
-    {
-        MockPageOutput output = new MockPageOutput();
-        try (PageBuilder builder = new PageBuilder(bufferAllocator, schema,
-                output)) {
-            int idx = 0;
-            while (idx < values.length) {
-                for (int column = 0; column < builder.getSchema()
-                        .getColumnCount(); ++column) {
-                    Object value = values[idx++];
-                    if (value == null) {
-                        builder.setNull(column);
-                    } else if (value instanceof Boolean) {
-                        builder.setBoolean(column, (Boolean) value);
-                    } else if (value instanceof Double) {
-                        builder.setDouble(column, (Double) value);
-                    } else if (value instanceof Long) {
-                        builder.setLong(column, (Long) value);
-                    } else if (value instanceof String) {
-                        builder.setString(column, (String) value);
-                    } else if (value instanceof Timestamp) {
-                        builder.setTimestamp(column, (Timestamp) value);
-                    } else if (value instanceof Value) {
-                        builder.setJson(column, (Value) value);
-                    } else {
-                        throw new IllegalStateException(
-                                "Unsupported type in test utils: "
-                                        + value.toString());
-                    }
-                }
-                builder.addRecord();
-            }
-            builder.finish();
-        }
-        return output.pages;
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java

@@ -1,24 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertFalse;
-import org.junit.Test;
-
-public class TestBuffer
-{
-    @Test
-    public void testEquals() throws Exception
-    {
-        byte[] bytes = new byte[] { 1, 2, 3, 2, 3 };
-        Buffer b1 = Buffer.wrap(bytes, 0, 2);  // [1, 2]
-        Buffer b2 = Buffer.wrap(bytes, 1, 2);  // [2, 3]
-        Buffer b3 = Buffer.wrap(bytes, 3, 2);  // [2, 3]
-
-        assertFalse(b1.equals(b2));
-        assertTrue(b2.equals(b3));
-
-        assertFalse(b1.hashCode() == b2.hashCode());
-        assertTrue(b2.hashCode() == b3.hashCode());
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputInputStream.java

@@ -1,89 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import java.io.ByteArrayOutputStream;
-import java.util.Random;
-import org.junit.Rule;
-import org.junit.Test;
-import org.embulk.spi.util.ListFileInput;
-import org.embulk.spi.util.FileInputInputStream;
-import org.embulk.spi.util.FileOutputOutputStream;
-import org.embulk.EmbulkTestRuntime;
-
-public class TestFileInputInputStream
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private ListFileInput fileInput;
-    private MockFileOutput fileOutput;
-
-    private FileInputInputStream in;
-    private FileOutputOutputStream out;
-
-    private void newOutputStream()
-    {
-        fileOutput = new MockFileOutput();
-        out = new FileOutputOutputStream(fileOutput, runtime.getBufferAllocator(), FileOutputOutputStream.CloseMode.CLOSE);
-    }
-
-    private void newInputStream()
-    {
-        fileInput = new ListFileInput(fileOutput.getFiles());
-        in = new FileInputInputStream(fileInput);
-    }
-
-    @Test
-    public void testRandomReadWrite() throws Exception
-    {
-        newOutputStream();
-        out.nextFile();
-        ByteArrayOutputStream expectedOut = new ByteArrayOutputStream();
-        Random rand = runtime.getRandom();
-        byte[] buffer = new byte[rand.nextInt() % 1024 + 1024];
-        for (int i = 0; i < 256; i++) {
-            rand.nextBytes(buffer);
-            expectedOut.write(buffer);
-            out.write(buffer);
-        }
-        out.finish();
-        byte[] expected = expectedOut.toByteArray();
-        byte[] actual = new byte[expected.length];
-
-        newInputStream();
-        in.nextFile();
-        int pos = 0;
-        while (pos < actual.length) {
-            int n = in.read(actual, pos, actual.length - pos);
-            if (n < 0) {
-                break;
-            }
-            pos += n;
-        }
-        assertEquals(expected.length, pos);
-        assertArrayEquals(expected, actual);
-    }
-
-    @Test
-    public void testSkipReturnsZeroForNoData() {
-        FileInputInputStream in = new FileInputInputStream(new MockFileInput());
-        assertEquals("Verify skip() returns 0 when there is no data.", 0L, in.skip(1));
-    }
-
-    private static class MockFileInput implements FileInput {
-        @Override
-        public boolean nextFile() {
-            return false;
-        }
-
-        @Override
-        public Buffer poll() {
-            return null;
-        }
-
-        @Override
-        public void close() {
-        }
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -1,199 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Queue;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.TaskReport;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.util.Pages;
-
-public class TestFileInputRunner
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    @Before
-    public void tearDown()
-    {
-        MockParserPlugin.raiseException = false;
-    }
-
-    private static class MockFileInputPlugin implements FileInputPlugin
-    {
-        Boolean transactionCompleted = null;
-        Queue<Buffer> buffers;
-
-        public MockFileInputPlugin(Queue<Buffer> buffers)
-        {
-            this.buffers = buffers;
-        }
-
-        @Override
-        public ConfigDiff transaction(ConfigSource config,
-                FileInputPlugin.Control control)
-        {
-            control.run(Exec.newTaskSource(), 1);
-            return null;
-        }
-
-        @Override
-        public ConfigDiff resume(TaskSource taskSource,
-                int taskCount,
-                FileInputPlugin.Control control)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void cleanup(TaskSource taskSource,
-                int taskCount,
-                List<TaskReport> successTaskReports)
-        {
-        }
-
-        public TransactionalFileInput open(TaskSource taskSource,
-                int taskIndex)
-        {
-            return new TransactionalFileInput()
-            {
-                @Override
-                public Buffer poll()
-                {
-                    return buffers.poll();
-                }
-
-                @Override
-                public boolean nextFile()
-                {
-                    return !buffers.isEmpty();
-                }
-
-                @Override
-                public void close()
-                {
-                }
-
-                @Override
-                public void abort()
-                {
-                    transactionCompleted = false;
-                }
-
-                @Override
-                public TaskReport commit()
-                {
-                    transactionCompleted = true;
-                    return null;
-                }
-            };
-        }
-    }
-
-    @Test
-    public void testMockParserIteration()
-    {
-        Buffer[] buffers = new Buffer[] {
-                runtime.getBufferAllocator().allocate(),
-                runtime.getBufferAllocator().allocate() };
-        MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(
-                new LinkedList<Buffer>(Arrays.asList(buffers)));
-        final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
-
-        ConfigSource config = Exec.newConfigSource().set(
-                "parser",
-                ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
-
-        final MockPageOutput output = new MockPageOutput();
-        runner.transaction(config, new InputPlugin.Control()
-        {
-            public List<TaskReport> run(TaskSource inputTaskSource,
-                    Schema schema, int taskCount)
-            {
-                List<TaskReport> reports = new ArrayList<>();
-                reports.add(runner.run(inputTaskSource, schema, 0, output));
-                return reports;
-            }
-        });
-
-        assertEquals(true, fileInputPlugin.transactionCompleted);
-        assertEquals(1, output.pages.size());
-
-        Schema schema = config.getNested("parser")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        List<Object[]> records = Pages.toObjects(schema, output.pages);
-        assertEquals(2, records.size());
-        for (Object[] record : records) {
-            assertEquals(6, record.length);
-            assertEquals(true, record[0]);
-            assertEquals(2L, record[1]);
-            assertEquals(3.0D, (Double) record[2], 0.01D);
-            assertEquals("45", record[3]);
-            assertEquals(678L, ((Timestamp) record[4]).toEpochMilli());
-            assertEquals("{\"_c2\":10,\"_c1\":true,\"_c4\":{\"k\":\"v\"},\"_c3\":\"embulk\"}", record[5].toString());
-        }
-    }
-
-    @Test
-    public void testTransactionAborted()
-    {
-        Buffer[] buffers = new Buffer[] {
-                runtime.getBufferAllocator().allocate(),
-                runtime.getBufferAllocator().allocate() };
-        MockFileInputPlugin fileInputPlugin = new MockFileInputPlugin(
-                new LinkedList<Buffer>(Arrays.asList(buffers)));
-        final FileInputRunner runner = new FileInputRunner(fileInputPlugin);
-
-        ConfigSource config = Exec.newConfigSource().set(
-                "parser",
-                ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                        ImmutableMap.of("name", "col6", "type", "json", "option", ImmutableMap.of()))));
-
-        final MockPageOutput output = new MockPageOutput();
-
-        MockParserPlugin.raiseException = true;
-
-        try {
-            runner.transaction(config, new InputPlugin.Control()
-            {
-                public List<TaskReport> run(TaskSource inputTaskSource,
-                        Schema schema, int taskCount)
-                {
-                    List<TaskReport> reports = new ArrayList<>();
-                    reports.add(runner.run(inputTaskSource, schema, 0, output));
-                    return reports;
-                }
-            });
-        } catch (RuntimeException re) {
-        }
-        assertEquals(false, fileInputPlugin.transactionCompleted);
-        assertEquals(0, output.pages.size());
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -1,221 +0,0 @@
-package org.embulk.spi;
-
-import static org.junit.Assert.assertEquals;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.TaskReport;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.Schema;
-import org.junit.Rule;
-import org.junit.Test;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.msgpack.value.ImmutableMapValue;
-import static org.msgpack.value.ValueFactory.newBoolean;
-import static org.msgpack.value.ValueFactory.newInteger;
-import static org.msgpack.value.ValueFactory.newMap;
-import static org.msgpack.value.ValueFactory.newString;
-
-public class TestFileOutputRunner
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    public interface PluginTask extends Task
-    {
-    }
-
-    private static class MockFileOutputPlugin implements FileOutputPlugin
-    {
-        Boolean transactionCompleted = null;
-
-        @Override
-        public ConfigDiff transaction(ConfigSource config, int taskCount,
-                FileOutputPlugin.Control control)
-        {
-            PluginTask task = config.loadConfig(PluginTask.class);
-            control.run(task.dump());
-            return Exec.newConfigDiff();
-        }
-
-        @Override
-        public ConfigDiff resume(TaskSource taskSource,
-                int taskCount,
-                FileOutputPlugin.Control control)
-        {
-            throw new UnsupportedOperationException();
-        }
-
-        @Override
-        public void cleanup(TaskSource taskSource,
-                int taskCount,
-                List<TaskReport> successTaskReports)
-        {
-        }
-
-        @Override
-        public TransactionalFileOutput open(TaskSource taskSource,
-                final int taskIndex)
-        {
-            return new TransactionalFileOutput()
-            {
-
-                @Override
-                public void nextFile()
-                {
-                }
-
-                @Override
-                public void add(Buffer buffer)
-                {
-                }
-
-                @Override
-                public void finish()
-                {
-                }
-
-                @Override
-                public void close()
-                {
-                }
-
-                @Override
-                public void abort()
-                {
-                    transactionCompleted = false;
-                }
-
-                @Override
-                public TaskReport commit()
-                {
-                    transactionCompleted = true;
-                    return Exec.newTaskReport();
-                }
-            };
-        }
-    }
-
-    @Test
-    public void testMockFormatterIteration()
-    {
-        MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
-        final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
-
-        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
-                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
-        ConfigSource config = Exec
-                .newConfigSource()
-                .set("type", "unused?")
-                .set("formatter",
-                        ImmutableMap.of("type", "mock", "columns", columns));
-        final Schema schema = config.getNested("formatter")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        runner.transaction(config, schema, 1, new OutputPlugin.Control()
-        {
-            public List<TaskReport> run(final TaskSource outputTask)
-            {
-                TransactionalPageOutput tran = runner.open(outputTask, schema,
-                        1);
-                boolean committed = false;
-                try {
-                    ImmutableMapValue jsonValue = newMap(
-                            newString("_c1"), newBoolean(true),
-                            newString("_c2"), newInteger(10),
-                            newString("_c3"), newString("embulk"),
-                            newString("_c4"), newMap(newString("k"), newString("v"))
-                    );
-                    for (Page page : PageTestUtils.buildPage(
-                            runtime.getBufferAllocator(), schema, true, 2L,
-                            3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue, true, 2L,
-                            3.0D, "45", Timestamp.ofEpochMilli(678L), jsonValue)) {
-                        tran.add(page);
-                    }
-                    tran.commit();
-                    committed = true;
-                } finally {
-                    if (!committed) {
-                        tran.abort();
-                    }
-                    tran.close();
-                }
-                return new ArrayList<TaskReport>();
-            }
-        });
-
-        assertEquals(true, fileOutputPlugin.transactionCompleted);
-        assertEquals(2, MockFormatterPlugin.records.size());
-        for (List<Object> record : MockFormatterPlugin.records) {
-            assertEquals(Boolean.TRUE, record.get(0));
-            assertEquals(2L, record.get(1));
-            assertEquals(3.0D, (Double) record.get(2), 0.1D);
-            assertEquals("45", record.get(3));
-            assertEquals(678L, ((Timestamp) record.get(4)).toEpochMilli());
-            assertEquals("{\"_c1\":true,\"_c2\":10,\"_c3\":\"embulk\",\"_c4\":{\"k\":\"v\"}}", record.get(5).toString());
-        }
-    }
-
-    @Test
-    public void testTransactionAborted()
-    {
-        MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
-        final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
-
-        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
-                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()),
-                ImmutableMap.<String,Object>of("name", "col6", "type", "json", "option", ImmutableMap.of()));
-        ConfigSource config = Exec
-                .newConfigSource()
-                .set("type", "unused?")
-                .set("formatter",
-                        ImmutableMap.of("type", "mock", "columns", columns));
-        final Schema schema = config.getNested("formatter")
-                .loadConfig(MockParserPlugin.PluginTask.class)
-                .getSchemaConfig().toSchema();
-
-        try {
-            runner.transaction(config, schema, 1, new OutputPlugin.Control()
-            {
-                public List<TaskReport> run(final TaskSource outputTask)
-                {
-                    TransactionalPageOutput tran = runner.open(outputTask,
-                            schema, 1);
-                    boolean committed = false;
-                    try {
-                        tran.add(null);
-                        tran.commit();
-                        committed = true;
-                    } finally {
-                        if (!committed) {
-                            tran.abort();
-                        }
-                        tran.close();
-                    }
-                    return new ArrayList<TaskReport>();
-                }
-            });
-        } catch (NullPointerException npe) {
-        }
-
-        assertEquals(false, fileOutputPlugin.transactionCompleted);
-    }
-}