embulk 0.7.0-java

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java

@@ -0,0 +1,116 @@
+package org.embulk.spi.time;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.junit.Test;
+
+public class TestTimestamp
+{
+    @Test
+    public void testEqualsToTimestamp()
+    {
+        assertEqualsMethods(Timestamp.ofEpochSecond(0), Timestamp.ofEpochSecond(0));
+        assertEqualsMethods(Timestamp.ofEpochSecond(10), Timestamp.ofEpochSecond(10));
+        assertEqualsMethods(Timestamp.ofEpochSecond(10, 2), Timestamp.ofEpochSecond(10, 2));
+    }
+
+    private void assertEqualsMethods(Timestamp t1, Timestamp t2)
+    {
+        assertEquals(t1, t2);
+        assertEquals(t1.hashCode(), t2.hashCode());
+        assertEquals(0, t1.compareTo(t2));
+    }
+
+    @Test
+    public void testNotEqualsToTimestamp()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(Timestamp.ofEpochSecond(1)));
+        assertFalse(Timestamp.ofEpochSecond(10).equals(Timestamp.ofEpochSecond(10, 2)));
+        assertFalse(Timestamp.ofEpochSecond(10, 2).equals(Timestamp.ofEpochSecond(20, 2)));
+    }
+
+    @Test
+    public void testEqualsToNull()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(null));
+        assertFalse(Timestamp.ofEpochSecond(1, 2).equals(null));
+    }
+
+    @Test
+    public void testEqualsOtherClass()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(new Object()));
+        assertFalse(Timestamp.ofEpochSecond(1, 2).equals("other"));
+    }
+
+    @Test
+    public void testAdjustMillisToNanos()
+    {
+        Timestamp t = Timestamp.ofEpochMilli(3);  // 3 msec = 3_000 usec == 3_000_000 nsec
+        assertEquals(0L, t.getEpochSecond());
+        assertEquals(3_000_000, t.getNano());
+    }
+
+    @Test
+    public void testAdjustMillisToSeconds()
+    {
+        Timestamp t = Timestamp.ofEpochMilli(3_000);  // 3_000 msec = 3 sec
+        assertEquals(3L, t.getEpochSecond());
+        assertEquals(0, t.getNano());
+    }
+
+    @Test
+    public void testAdjustNano()
+    {
+        Timestamp t = Timestamp.ofEpochSecond(0, 1_000_000_000);  // 1_000_000_000 nsec = 1_000_000 usec = 1_000 msec = 1 sec
+        assertEquals(1L, t.getEpochSecond());
+        assertEquals(0, t.getNano());
+    }
+
+    @Test
+    public void testCompareTo()
+    {
+        assertEquals(-1, Timestamp.ofEpochSecond(3).compareTo(Timestamp.ofEpochSecond(4)));
+        assertEquals(-1, Timestamp.ofEpochSecond(3).compareTo(Timestamp.ofEpochSecond(3, 4)));
+        assertEquals( 1, Timestamp.ofEpochSecond(4).compareTo(Timestamp.ofEpochSecond(3)));
+        assertEquals( 1, Timestamp.ofEpochSecond(3, 4).compareTo(Timestamp.ofEpochSecond(3)));
+    }
+
+    @Test
+    public void testToString()
+    {
+        assertEquals("1970-01-01 00:00:00 UTC", Timestamp.ofEpochSecond(0).toString());
+        assertEquals("2015-01-19 07:36:10 UTC", Timestamp.ofEpochSecond(1421652970).toString());
+        assertEquals("2015-01-19 07:36:10.100 UTC",       Timestamp.ofEpochSecond(1421652970, 100*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.120 UTC",       Timestamp.ofEpochSecond(1421652970, 120*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123 UTC",       Timestamp.ofEpochSecond(1421652970, 123*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123400 UTC",    Timestamp.ofEpochSecond(1421652970, 123400*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123450 UTC",    Timestamp.ofEpochSecond(1421652970, 123450*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123456 UTC",    Timestamp.ofEpochSecond(1421652970, 123456*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123456700 UTC", Timestamp.ofEpochSecond(1421652970, 123456700).toString());
+        assertEquals("2015-01-19 07:36:10.123456780 UTC", Timestamp.ofEpochSecond(1421652970, 123456780).toString());
+        assertEquals("2015-01-19 07:36:10.123456789 UTC", Timestamp.ofEpochSecond(1421652970, 123456789).toString());
+    }
+
+    @Test
+    public void testFromString()
+    {
+        checkToStringFromString(Timestamp.ofEpochSecond(0));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 100*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 120*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123400*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123450*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456700));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456780));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456789));
+    }
+
+    private void checkToStringFromString(Timestamp timestamp)
+    {
+        assertEquals(timestamp, Timestamp.fromString(timestamp.toString()));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java

@@ -0,0 +1,63 @@
+package org.embulk.spi.time;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import com.google.common.base.Optional;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTimestampFormatterParser
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private interface FormatterTestTask
+            extends Task, TimestampFormatter.Task
+    { }
+
+    private interface ParserTestTask
+            extends Task, TimestampParser.Task
+    { }
+
+    @Test
+    public void testSimpleFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S.%9N %Z");
+        FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
+
+        TimestampFormatter formatter = new TimestampFormatter(task, Optional.<TimestampFormatter.TimestampColumnOption>absent());
+        assertEquals("2014-11-19 02:46:29.123456000 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
+    }
+
+    @Test
+    public void testSimpleParse() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S %Z");
+        ParserTestTask task = config.loadConfig(ParserTestTask.class);
+
+        TimestampParser parser = new TimestampParser(task);
+        assertEquals(Timestamp.ofEpochSecond(1416365189, 0), parser.parse("2014-11-19 02:46:29 UTC"));
+    }
+
+    @Test
+    public void testUnixtimeFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("default_timestamp_format", "%s");
+
+        FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class);
+        TimestampFormatter formatter = new TimestampFormatter(ftask, Optional.<TimestampFormatter.TimestampColumnOption>absent());
+        assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+
+        ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
+        TimestampParser parser = new TimestampParser(ptask);
+        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java

@@ -0,0 +1,67 @@
+package org.embulk.spi.time;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTimestampFormatterParserDeprecated
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private interface FormatterTestTask
+            extends TimestampFormatter.FormatterTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    private interface ParserTestTask
+            extends TimestampParser.ParserTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    @Test
+    public void testSimpleFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%6N %Z");
+        FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
+
+        TimestampFormatter formatter = task.getTimeFormat().newFormatter(task);
+        assertEquals("2014-11-19 02:46:29.123456 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
+    }
+
+    @Test
+    public void testSimpleParse() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%N %Z");
+        ParserTestTask task = config.loadConfig(ParserTestTask.class);
+
+        TimestampParser parser = task.getTimeFormat().newParser(task);
+        assertEquals(Timestamp.ofEpochSecond(1416365189, 123456*1000), parser.parse("2014-11-19 02:46:29.123456 UTC"));
+    }
+
+    @Test
+    public void testUnixtimeFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%s");
+
+        FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class);
+        TimestampFormatter formatter = ftask.getTimeFormat().newFormatter(ftask);
+        assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+
+        ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
+        TimestampParser parser = ptask.getTimeFormat().newParser(ptask);
+        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java

@@ -0,0 +1,45 @@
+package org.embulk.spi.type;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import com.google.inject.Inject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTypeSerDe
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private static class HasType
+    {
+        private Type type;
+        // TODO test TimestampType
+
+        @JsonCreator
+        public HasType(
+                @JsonProperty("type") Type type)
+        {
+            this.type = type;
+        }
+
+        @JsonProperty("type")
+        public Type getType()
+        {
+            return type;
+        }
+    }
+
+    @Test
+    public void testGetType()
+    {
+        HasType type = new HasType(StringType.STRING);
+        String json = runtime.getModelManager().writeObject(type);
+        HasType decoded = runtime.getModelManager().readObject(HasType.class, json);
+        assertTrue(StringType.STRING == decoded.getType());
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java

@@ -0,0 +1,79 @@
+package org.embulk.spi.unit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import org.junit.Test;
+
+public class TestByteSize
+{
+    @Test
+    public void testUnitPatterns()
+    {
+        assertByteSize(42L, "42");
+        assertByteSize(42L, "42B");
+        assertByteSize(42L*(1L << 10), "42KB");
+        assertByteSize(42L*(1L << 20), "42MB");
+        assertByteSize(42L*(1L << 30), "42GB");
+        assertByteSize(42L*(1L << 40), "42TB");
+        assertByteSize(42L*(1L << 50), "42PB");
+        assertByteSize(42L, "42 B");
+        assertByteSize(42L*(1L << 10), "42 KB");
+    }
+
+    @Test
+    public void testUnknownUnits()
+    {
+        assertInvalidByteSize("42XB");
+        assertInvalidByteSize("42 XB");
+    }
+
+    @Test
+    public void testInvalidPatterns()
+    {
+        assertInvalidByteSize(" 42");
+        assertInvalidByteSize("42  B");
+        assertInvalidByteSize("42 B ");
+        assertInvalidByteSize("42B ");
+        assertInvalidByteSize("42  KB");
+        assertInvalidByteSize("42 KB ");
+        assertInvalidByteSize("42KB ");
+    }
+
+    @Test
+    public void testInvalidValues()
+    {
+        assertInvalidByteSize("9223372036854775KB");
+    }
+
+    @Test
+    public void testToString()
+    {
+        assertByteSizeString("42B", "42 B");
+        assertByteSizeString("42KB", "42 KB");
+        assertByteSizeString("42MB", "42 MB");
+        assertByteSizeString("42GB", "42 GB");
+        assertByteSizeString("42TB", "42 TB");
+        assertByteSizeString("42PB", "42 PB");
+        assertByteSizeString("42.20KB", "42.2 KB");
+        assertByteSizeString("42.33KB", "42.33KB");
+    }
+
+    private static void assertByteSize(long bytes, String string)
+    {
+        assertEquals(bytes, ByteSize.parseByteSize(string).getBytes());
+    }
+
+    private static void assertByteSizeString(String expected, String string)
+    {
+        assertEquals(expected, ByteSize.parseByteSize(string).toString());
+    }
+
+    private static void assertInvalidByteSize(String string)
+    {
+        try {
+            ByteSize.parseByteSize(string);
+            fail();
+        } catch (IllegalArgumentException ex) {
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java

@@ -0,0 +1,132 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import com.google.common.collect.ImmutableList;
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.util.ListFileInput;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestLineDecoder
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource();
+        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
+        assertEquals(Charset.forName("utf-8"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+    }
+
+    @Test
+    public void testLoadConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("charset", "utf-16")
+            .set("newline", "CRLF");
+        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
+        assertEquals(Charset.forName("utf-16"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+    }
+
+    private static LineDecoder.DecoderTask getExampleConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("charset", "utf-8")
+            .set("newline", "LF");
+        return config.loadConfig(LineDecoder.DecoderTask.class);
+    }
+
+    private static LineDecoder newDecoder(Charset charset, Newline newline, List<Buffer> buffers)
+    {
+        ListFileInput input = new ListFileInput(ImmutableList.of(buffers));
+        return new LineDecoder(input, getExampleConfig());
+    }
+
+    private static List<String> doDecode(Charset charset, Newline newline, List<Buffer> buffers)
+    {
+        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        LineDecoder decoder = newDecoder(charset, newline, buffers);
+        decoder.nextFile();
+        while (true) {
+            String line = decoder.poll();
+            if (line == null) {
+                break;
+            }
+            builder.add(line);
+        }
+        return builder.build();
+    }
+
+    private static List<Buffer> bufferList(String charsetName, String... sources) throws UnsupportedCharsetException
+    {
+        Charset charset = Charset.forName(charsetName);
+
+        List<Buffer> buffers = new ArrayList<Buffer>();
+        for (String source : sources) {
+            ByteBuffer buffer = charset.encode(source);
+            buffers.add(Buffer.wrap(buffer.array(), 0, buffer.limit()));
+        }
+
+        return buffers;
+    }
+
+    @Test
+    public void testDecodeBasicAscii() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "test1\ntest2\ntest3\n"));
+        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicAsciiCRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.CRLF,
+                bufferList("utf-8", "test1\r\ntest2\r\ntest3\r\n"));
+        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicAsciiTail() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "test1"));
+        assertEquals(ImmutableList.of("test1"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "t", "1", "\n", "t", "2"));
+        assertEquals(ImmutableList.of("t1", "t2"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksCRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.CRLF,
+                bufferList("utf-8", "t", "1", "\r\n", "t", "2", "\r", "\n", "t3"));
+        assertEquals(ImmutableList.of("t1", "t2", "t3"), decoded);
+    }
+
+    // TODO test multibytes
+}