embulk 0.8.38-java → 0.10.27-java

data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java

@@ -1,45 +0,0 @@
-package org.embulk.spi.type;
-
-import org.junit.Rule;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import com.google.inject.Inject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.embulk.EmbulkTestRuntime;
-
-public class TestTypeSerDe
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private static class HasType
-    {
-        private Type type;
-        // TODO test TimestampType
-
-        @JsonCreator
-        public HasType(
-                @JsonProperty("type") Type type)
-        {
-            this.type = type;
-        }
-
-        @JsonProperty("type")
-        public Type getType()
-        {
-            return type;
-        }
-    }
-
-    @Test
-    public void testGetType()
-    {
-        HasType type = new HasType(StringType.STRING);
-        String json = runtime.getModelManager().writeObject(type);
-        HasType decoded = runtime.getModelManager().readObject(HasType.class, json);
-        assertTrue(StringType.STRING == decoded.getType());
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java

@@ -1,79 +0,0 @@
-package org.embulk.spi.unit;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import org.junit.Test;
-
-public class TestByteSize
-{
-    @Test
-    public void testUnitPatterns()
-    {
-        assertByteSize(42L, "42");
-        assertByteSize(42L, "42B");
-        assertByteSize(42L*(1L << 10), "42KB");
-        assertByteSize(42L*(1L << 20), "42MB");
-        assertByteSize(42L*(1L << 30), "42GB");
-        assertByteSize(42L*(1L << 40), "42TB");
-        assertByteSize(42L*(1L << 50), "42PB");
-        assertByteSize(42L, "42 B");
-        assertByteSize(42L*(1L << 10), "42 KB");
-    }
-
-    @Test
-    public void testUnknownUnits()
-    {
-        assertInvalidByteSize("42XB");
-        assertInvalidByteSize("42 XB");
-    }
-
-    @Test
-    public void testInvalidPatterns()
-    {
-        assertInvalidByteSize(" 42");
-        assertInvalidByteSize("42  B");
-        assertInvalidByteSize("42 B ");
-        assertInvalidByteSize("42B ");
-        assertInvalidByteSize("42  KB");
-        assertInvalidByteSize("42 KB ");
-        assertInvalidByteSize("42KB ");
-    }
-
-    @Test
-    public void testInvalidValues()
-    {
-        assertInvalidByteSize("9223372036854775KB");
-    }
-
-    @Test
-    public void testToString()
-    {
-        assertByteSizeString("42B", "42 B");
-        assertByteSizeString("42KB", "42 KB");
-        assertByteSizeString("42MB", "42 MB");
-        assertByteSizeString("42GB", "42 GB");
-        assertByteSizeString("42TB", "42 TB");
-        assertByteSizeString("42PB", "42 PB");
-        assertByteSizeString("42.20KB", "42.2 KB");
-        assertByteSizeString("42.33KB", "42.33KB");
-    }
-
-    private static void assertByteSize(long bytes, String string)
-    {
-        assertEquals(bytes, ByteSize.parseByteSize(string).getBytes());
-    }
-
-    private static void assertByteSizeString(String expected, String string)
-    {
-        assertEquals(expected, ByteSize.parseByteSize(string).toString());
-    }
-
-    private static void assertInvalidByteSize(String string)
-    {
-        try {
-            ByteSize.parseByteSize(string);
-            fail();
-        } catch (IllegalArgumentException ex) {
-        }
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java

@@ -1,237 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.StandardCharsets;
-import java.nio.charset.UnsupportedCharsetException;
-import com.google.common.collect.ImmutableList;
-import org.junit.Rule;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.Exec;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.util.ListFileInput;
-import org.embulk.EmbulkTestRuntime;
-
-public class TestLineDecoder
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    @Test
-    public void testDefaultValues()
-    {
-        ConfigSource config = Exec.newConfigSource();
-        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
-        assertEquals(StandardCharsets.UTF_8, task.getCharset());
-        assertEquals(Newline.CRLF, task.getNewline());
-    }
-
-    @Test
-    public void testLoadConfig()
-    {
-        ConfigSource config = Exec.newConfigSource()
-            .set("charset", "utf-16")
-            .set("newline", "CRLF");
-        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
-        assertEquals(StandardCharsets.UTF_16, task.getCharset());
-        assertEquals(Newline.CRLF, task.getNewline());
-    }
-
-    private static LineDecoder.DecoderTask getExampleConfig(Charset charset, Newline newline)
-    {
-        ConfigSource config = Exec.newConfigSource()
-            .set("charset", charset)
-            .set("newline", newline);
-        return config.loadConfig(LineDecoder.DecoderTask.class);
-    }
-
-    private static LineDecoder newDecoder(Charset charset, Newline newline, List<Buffer> buffers)
-    {
-        ListFileInput input = new ListFileInput(ImmutableList.of(buffers));
-        return new LineDecoder(input, getExampleConfig(charset, newline));
-    }
-
-    private static List<String> doDecode(Charset charset, Newline newline, List<Buffer> buffers)
-    {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-        LineDecoder decoder = newDecoder(charset, newline, buffers);
-        decoder.nextFile();
-        while (true) {
-            String line = decoder.poll();
-            if (line == null) {
-                break;
-            }
-            builder.add(line);
-        }
-        return builder.build();
-    }
-
-    private static List<Buffer> bufferList(Charset charset, String... sources) throws UnsupportedCharsetException
-    {
-        List<Buffer> buffers = new ArrayList<Buffer>();
-        for (String source : sources) {
-            ByteBuffer buffer = charset.encode(source);
-            buffers.add(Buffer.wrap(buffer.array(), 0, buffer.limit()));
-        }
-
-        return buffers;
-    }
-
-    @Test
-    public void testDecodeBasicAscii() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "test1\ntest2\ntest3\n"));
-        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicAsciiCRLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.CRLF,
-                bufferList(StandardCharsets.UTF_8, "test1\r\ntest2\r\ntest3\r\n"));
-        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicAsciiTail() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "test1"));
-        assertEquals(ImmutableList.of("test1"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "t", "1", "\n", "t", "2"));
-        assertEquals(ImmutableList.of("t1", "t2"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksCRLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.CRLF,
-                bufferList(StandardCharsets.UTF_8, "t", "1", "\r\n", "t", "2", "\r", "\n", "t3"));
-        assertEquals(ImmutableList.of("t1", "t2", "t3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicUTF8() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "てすと1\nテスト2\nてすと3\n"));
-        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicUTF8Tail() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "てすと1"));
-        assertEquals(ImmutableList.of("てすと1"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksUTF8LF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.LF,
-                bufferList(StandardCharsets.UTF_8, "て", "1", "\n", "す", "2"));
-        assertEquals(ImmutableList.of("て1", "す2"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksUTF8CRLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_8, Newline.CRLF,
-                bufferList(StandardCharsets.UTF_8, "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
-        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicUTF16LE() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_16LE, Newline.LF,
-                bufferList(StandardCharsets.UTF_16LE, "てすと1\nテスト2\nてすと3\n"));
-        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicUTF16LETail() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_16LE, Newline.LF,
-                bufferList(StandardCharsets.UTF_16LE, "てすと1"));
-        assertEquals(ImmutableList.of("てすと1"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksUTF16LELF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_16LE, Newline.LF,
-                bufferList(StandardCharsets.UTF_16LE, "て", "1", "\n", "す", "2"));
-        assertEquals(ImmutableList.of("て1", "す2"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksUTF16LECRLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                StandardCharsets.UTF_16LE, Newline.CRLF,
-                bufferList(StandardCharsets.UTF_16LE, "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
-        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicMS932() throws Exception
-    {
-        List<String> decoded = doDecode(
-                Charset.forName("ms932"), Newline.LF,
-                bufferList(Charset.forName("ms932"), "てすと1\nテスト2\nてすと3\n"));
-        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
-    }
-
-    @Test
-    public void testDecodeBasicMS932Tail() throws Exception
-    {
-        List<String> decoded = doDecode(
-                Charset.forName("ms932"), Newline.LF,
-                bufferList(Charset.forName("ms932"), "てすと1"));
-        assertEquals(ImmutableList.of("てすと1"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksMS932LF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                Charset.forName("ms932"), Newline.LF,
-                bufferList(Charset.forName("ms932"), "て", "1", "\n", "す", "2"));
-        assertEquals(ImmutableList.of("て1", "す2"), decoded);
-    }
-
-    @Test
-    public void testDecodeChunksMS932CRLF() throws Exception
-    {
-        List<String> decoded = doDecode(
-                Charset.forName("ms932"), Newline.CRLF,
-                bufferList(Charset.forName("ms932"), "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
-        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
-    }
-}

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java

@@ -1,123 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.io.UnsupportedEncodingException;
-import java.nio.ByteBuffer;
-import java.nio.charset.Charset;
-import java.nio.charset.UnsupportedCharsetException;
-import com.google.common.collect.ImmutableList;
-import org.junit.Rule;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.Exec;
-import org.embulk.spi.MockFileOutput;
-import org.embulk.spi.FileOutput;
-import org.embulk.EmbulkTestRuntime;
-
-public class TestLineEncoder
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private LineEncoder newEncoder(String charset, String newline,
-            FileOutput output) throws Exception
-    {
-        ConfigSource config = Exec.newConfigSource()
-            .set("charset", charset)
-            .set("newline", newline);
-        return new LineEncoder(output, config.loadConfig(LineEncoder.EncoderTask.class));
-    }
-
-    @Test
-    public void testAddLine() throws Exception
-    {
-        try (MockFileOutput output = new MockFileOutput()) {
-            LineEncoder encoder = newEncoder("utf-8", "LF", output);
-            encoder.nextFile();
-            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
-                encoder.addLine(line);
-            }
-            encoder.finish();
-            // TODO
-            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
-            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
-            //assertFalse(ite.hasNext());
-        }
-    }
-
-    @Test
-    public void testAddTextAddNewLine() throws Exception
-    {
-        try (MockFileOutput output = new MockFileOutput()) {
-            LineEncoder encoder = newEncoder("utf-8", "LF", output);
-            encoder.nextFile();
-            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
-                encoder.addText(line);
-                encoder.addNewLine();
-            }
-            encoder.finish();
-            // TODO
-            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
-            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
-            //assertFalse(ite.hasNext());
-        }
-    }
-
-    @Test
-    public void testNewLine() throws Exception
-    {
-        try (MockFileOutput output = new MockFileOutput()) {
-            LineEncoder encoder = newEncoder("utf-8", "CRLF", output);
-            encoder.nextFile();
-            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
-                encoder.addLine(line);
-            }
-            encoder.finish();
-            // TODO
-            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
-            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\r\n", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
-            //assertEquals("\r\n", bufferToString(ite.next(), "utf-8"));
-            //assertFalse(ite.hasNext());
-        }
-    }
-
-    @Test
-    public void testCharset() throws Exception
-    {
-        try (MockFileOutput output = new MockFileOutput()) {
-            LineEncoder encoder = newEncoder("MS932", "CR", output);
-            encoder.nextFile();
-            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
-                encoder.addLine(line);
-            }
-            encoder.finish();
-            // TODO
-            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
-            //assertEquals("abc", bufferToString(ite.next(), "MS932"));
-            //assertEquals("\r", bufferToString(ite.next(), "MS932"));
-            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "MS932"));
-            //assertEquals("\r", bufferToString(ite.next(), "MS932"));
-            //assertFalse(ite.hasNext());
-        }
-    }
-
-    private String bufferToString(Buffer buffer, String charset)
-            throws UnsupportedEncodingException
-    {
-        return new String(buffer.array(), buffer.offset(), buffer.limit(), charset);
-    }
-}