embulk 0.1.0

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java

@@ -0,0 +1,116 @@
+package org.embulk.spi.time;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.junit.Test;
+
+public class TestTimestamp
+{
+    @Test
+    public void testEqualsToTimestamp()
+    {
+        assertEqualsMethods(Timestamp.ofEpochSecond(0), Timestamp.ofEpochSecond(0));
+        assertEqualsMethods(Timestamp.ofEpochSecond(10), Timestamp.ofEpochSecond(10));
+        assertEqualsMethods(Timestamp.ofEpochSecond(10, 2), Timestamp.ofEpochSecond(10, 2));
+    }
+
+    private void assertEqualsMethods(Timestamp t1, Timestamp t2)
+    {
+        assertEquals(t1, t2);
+        assertEquals(t1.hashCode(), t2.hashCode());
+        assertEquals(0, t1.compareTo(t2));
+    }
+
+    @Test
+    public void testNotEqualsToTimestamp()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(Timestamp.ofEpochSecond(1)));
+        assertFalse(Timestamp.ofEpochSecond(10).equals(Timestamp.ofEpochSecond(10, 2)));
+        assertFalse(Timestamp.ofEpochSecond(10, 2).equals(Timestamp.ofEpochSecond(20, 2)));
+    }
+
+    @Test
+    public void testEqualsToNull()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(null));
+        assertFalse(Timestamp.ofEpochSecond(1, 2).equals(null));
+    }
+
+    @Test
+    public void testEqualsOtherClass()
+    {
+        assertFalse(Timestamp.ofEpochSecond(0).equals(new Object()));
+        assertFalse(Timestamp.ofEpochSecond(1, 2).equals("other"));
+    }
+
+    @Test
+    public void testAdjustMillisToNanos()
+    {
+        Timestamp t = Timestamp.ofEpochMilli(3);  // 3 msec = 3_000 usec == 3_000_000 nsec
+        assertEquals(0L, t.getEpochSecond());
+        assertEquals(3_000_000, t.getNano());
+    }
+
+    @Test
+    public void testAdjustMillisToSeconds()
+    {
+        Timestamp t = Timestamp.ofEpochMilli(3_000);  // 3_000 msec = 3 sec
+        assertEquals(3L, t.getEpochSecond());
+        assertEquals(0, t.getNano());
+    }
+
+    @Test
+    public void testAdjustNano()
+    {
+        Timestamp t = Timestamp.ofEpochSecond(0, 1_000_000_000);  // 1_000_000_000 nsec = 1_000_000 usec = 1_000 msec = 1 sec
+        assertEquals(1L, t.getEpochSecond());
+        assertEquals(0, t.getNano());
+    }
+
+    @Test
+    public void testCompareTo()
+    {
+        assertEquals(-1, Timestamp.ofEpochSecond(3).compareTo(Timestamp.ofEpochSecond(4)));
+        assertEquals(-1, Timestamp.ofEpochSecond(3).compareTo(Timestamp.ofEpochSecond(3, 4)));
+        assertEquals( 1, Timestamp.ofEpochSecond(4).compareTo(Timestamp.ofEpochSecond(3)));
+        assertEquals( 1, Timestamp.ofEpochSecond(3, 4).compareTo(Timestamp.ofEpochSecond(3)));
+    }
+
+    @Test
+    public void testToString()
+    {
+        assertEquals("1970-01-01 00:00:00 UTC", Timestamp.ofEpochSecond(0).toString());
+        assertEquals("2015-01-19 07:36:10 UTC", Timestamp.ofEpochSecond(1421652970).toString());
+        assertEquals("2015-01-19 07:36:10.100 UTC",       Timestamp.ofEpochSecond(1421652970, 100*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.120 UTC",       Timestamp.ofEpochSecond(1421652970, 120*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123 UTC",       Timestamp.ofEpochSecond(1421652970, 123*1000*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123400 UTC",    Timestamp.ofEpochSecond(1421652970, 123400*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123450 UTC",    Timestamp.ofEpochSecond(1421652970, 123450*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123456 UTC",    Timestamp.ofEpochSecond(1421652970, 123456*1000).toString());
+        assertEquals("2015-01-19 07:36:10.123456700 UTC", Timestamp.ofEpochSecond(1421652970, 123456700).toString());
+        assertEquals("2015-01-19 07:36:10.123456780 UTC", Timestamp.ofEpochSecond(1421652970, 123456780).toString());
+        assertEquals("2015-01-19 07:36:10.123456789 UTC", Timestamp.ofEpochSecond(1421652970, 123456789).toString());
+    }
+
+    @Test
+    public void testFromString()
+    {
+        checkToStringFromString(Timestamp.ofEpochSecond(0));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 100*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 120*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123*1000*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123400*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123450*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456*1000));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456700));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456780));
+        checkToStringFromString(Timestamp.ofEpochSecond(1421652970, 123456789));
+    }
+
+    private void checkToStringFromString(Timestamp timestamp)
+    {
+        assertEquals(timestamp, Timestamp.fromString(timestamp.toString()));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java

@@ -0,0 +1,52 @@
+package org.embulk.spi.time;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTimestampFormatterParser
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private interface FormatterTestTask
+            extends TimestampFormatter.FormatterTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    private interface ParserTestTask
+            extends TimestampParser.ParserTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    @Test
+    public void testSimpleFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S %Z");
+        FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
+
+        TimestampFormatter formatter = task.getTimeFormat().newFormatter(task);
+        assertEquals("2014-11-19 02:46:29 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+    }
+
+    @Test
+    public void testSimpleParse() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S %Z");
+        ParserTestTask task = config.loadConfig(ParserTestTask.class);
+
+        TimestampParser parser = task.getTimeFormat().newParser(task);
+        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("2014-11-19 02:46:29 UTC"));
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java

@@ -0,0 +1,45 @@
+package org.embulk.spi.type;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import com.google.inject.Inject;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTypeSerDe
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private static class HasType
+    {
+        private Type type;
+        // TODO test TimestampType
+
+        @JsonCreator
+        public HasType(
+                @JsonProperty("type") Type type)
+        {
+            this.type = type;
+        }
+
+        @JsonProperty("type")
+        public Type getType()
+        {
+            return type;
+        }
+    }
+
+    @Test
+    public void testGetType()
+    {
+        HasType type = new HasType(StringType.STRING);
+        String json = runtime.getModelManager().writeObject(type);
+        HasType decoded = runtime.getModelManager().readObject(HasType.class, json);
+        assertTrue(StringType.STRING == decoded.getType());
+    }
+}

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java

@@ -0,0 +1,132 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import com.google.common.collect.ImmutableList;
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.util.ListFileInput;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestLineDecoder
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource();
+        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
+        assertEquals(Charset.forName("utf-8"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+    }
+
+    @Test
+    public void testLoadConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("charset", "utf-16")
+            .set("newline", "CRLF");
+        LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
+        assertEquals(Charset.forName("utf-16"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+    }
+
+    private static LineDecoder.DecoderTask getExampleConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("charset", "utf-8")
+            .set("newline", "LF");
+        return config.loadConfig(LineDecoder.DecoderTask.class);
+    }
+
+    private static LineDecoder newDecoder(Charset charset, Newline newline, List<Buffer> buffers)
+    {
+        ListFileInput input = new ListFileInput(ImmutableList.of(buffers));
+        return new LineDecoder(input, getExampleConfig());
+    }
+
+    private static List<String> doDecode(Charset charset, Newline newline, List<Buffer> buffers)
+    {
+        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        LineDecoder decoder = newDecoder(charset, newline, buffers);
+        decoder.nextFile();
+        while (true) {
+            String line = decoder.poll();
+            if (line == null) {
+                break;
+            }
+            builder.add(line);
+        }
+        return builder.build();
+    }
+
+    private static List<Buffer> bufferList(String charsetName, String... sources) throws UnsupportedCharsetException
+    {
+        Charset charset = Charset.forName(charsetName);
+
+        List<Buffer> buffers = new ArrayList<Buffer>();
+        for (String source : sources) {
+            ByteBuffer buffer = charset.encode(source);
+            buffers.add(Buffer.wrap(buffer.array(), 0, buffer.limit()));
+        }
+
+        return buffers;
+    }
+
+    @Test
+    public void testDecodeBasicAscii() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "test1\ntest2\ntest3\n"));
+        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicAsciiCRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.CRLF,
+                bufferList("utf-8", "test1\r\ntest2\r\ntest3\r\n"));
+        assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicAsciiTail() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "test1"));
+        assertEquals(ImmutableList.of("test1"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.LF,
+                bufferList("utf-8", "t", "1", "\n", "t", "2"));
+        assertEquals(ImmutableList.of("t1", "t2"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksCRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("utf-8"), Newline.CRLF,
+                bufferList("utf-8", "t", "1", "\r\n", "t", "2", "\r", "\n", "t3"));
+        assertEquals(ImmutableList.of("t1", "t2", "t3"), decoded);
+    }
+
+    // TODO test multibytes
+}

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineEncoder.java

@@ -0,0 +1,123 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+import com.google.common.collect.ImmutableList;
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.MockFileOutput;
+import org.embulk.spi.FileOutput;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestLineEncoder
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private LineEncoder newEncoder(String charset, String newline,
+            FileOutput output) throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("charset", charset)
+            .set("newline", newline);
+        return new LineEncoder(output, config.loadConfig(LineEncoder.EncoderTask.class));
+    }
+
+    @Test
+    public void testAddLine() throws Exception
+    {
+        try (MockFileOutput output = new MockFileOutput()) {
+            LineEncoder encoder = newEncoder("utf-8", "LF", output);
+            encoder.nextFile();
+            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
+                encoder.addLine(line);
+            }
+            encoder.finish();
+            // TODO
+            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
+            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
+            //assertFalse(ite.hasNext());
+        }
+    }
+
+    @Test
+    public void testAddTextAddNewLine() throws Exception
+    {
+        try (MockFileOutput output = new MockFileOutput()) {
+            LineEncoder encoder = newEncoder("utf-8", "LF", output);
+            encoder.nextFile();
+            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
+                encoder.addText(line);
+                encoder.addNewLine();
+            }
+            encoder.finish();
+            // TODO
+            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
+            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\n", bufferToString(ite.next(), "utf-8"));
+            //assertFalse(ite.hasNext());
+        }
+    }
+
+    @Test
+    public void testNewLine() throws Exception
+    {
+        try (MockFileOutput output = new MockFileOutput()) {
+            LineEncoder encoder = newEncoder("utf-8", "CRLF", output);
+            encoder.nextFile();
+            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
+                encoder.addLine(line);
+            }
+            encoder.finish();
+            // TODO
+            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
+            //assertEquals("abc", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\r\n", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "utf-8"));
+            //assertEquals("\r\n", bufferToString(ite.next(), "utf-8"));
+            //assertFalse(ite.hasNext());
+        }
+    }
+
+    @Test
+    public void testCharset() throws Exception
+    {
+        try (MockFileOutput output = new MockFileOutput()) {
+            LineEncoder encoder = newEncoder("MS932", "CR", output);
+            encoder.nextFile();
+            for (String line : new String[] { "abc", "日本語(Japanese)" }) {
+                encoder.addLine(line);
+            }
+            encoder.finish();
+            // TODO
+            //Iterator<Buffer> ite = output.getLastBuffers().iterator();
+            //assertEquals("abc", bufferToString(ite.next(), "MS932"));
+            //assertEquals("\r", bufferToString(ite.next(), "MS932"));
+            //assertEquals("日本語(Japanese)", bufferToString(ite.next(), "MS932"));
+            //assertEquals("\r", bufferToString(ite.next(), "MS932"));
+            //assertFalse(ite.hasNext());
+        }
+    }
+
+    private String bufferToString(Buffer buffer, String charset)
+            throws UnsupportedEncodingException
+    {
+        return new String(buffer.array(), buffer.offset(), buffer.limit(), charset);
+    }
+}