embulk 0.6.13 → 0.6.14

data/embulk-core/src/main/java/org/embulk/spi/SchemaConfigException.java

@@ -0,0 +1,22 @@
+package org.embulk.spi;
+
+import org.embulk.config.ConfigException;
+
+public class SchemaConfigException
+        extends ConfigException
+{
+    public SchemaConfigException(String message)
+    {
+        super(message);
+    }
+
+    public SchemaConfigException(Throwable cause)
+    {
+        super(cause);
+    }
+
+    public SchemaConfigException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -26,11 +26,13 @@ public class TimestampFormat
         return format;
     }
 
+    @Deprecated
     public TimestampFormatter newFormatter(TimestampFormatter.FormatterTask task)
     {
         return new TimestampFormatter(format, task);
     }
 
+    @Deprecated
     public TimestampParser newParser(TimestampParser.ParserTask task)
     {
         return new TimestampParser(format, task);

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -3,9 +3,9 @@ package org.embulk.spi.time;
 import java.util.Locale;
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import com.google.common.base.Optional;
 import org.jruby.embed.ScriptingContainer;
 import org.jruby.util.RubyDateFormat;
-import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
@@ -14,8 +14,9 @@ import org.embulk.spi.util.LineEncoder;
 
 public class TimestampFormatter
 {
+    @Deprecated
     public interface FormatterTask
-            extends Task
+            extends org.embulk.config.Task
     {
         @Config("timezone")
         @ConfigDefault("\"UTC\"")
@@ -25,14 +26,51 @@ public class TimestampFormatter
         public ScriptingContainer getJRuby();
     }
 
+    public interface Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N %z\"")
+        public String getDefaultTimestampFormat();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface TimestampColumnOption
+    {
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+
+        @Config("format")
+        @ConfigDefault("null")
+        public Optional<String> getFormat();
+    }
+
     private final RubyDateFormat dateFormat;
     private final DateTimeZone timeZone;
 
+    @Deprecated
     public TimestampFormatter(String format, FormatterTask task)
     {
         this(task.getJRuby(), format, task.getTimeZone());
     }
 
+    public TimestampFormatter(Task task, Optional<? extends TimestampColumnOption> columnOption)
+    {
+        this(task.getJRuby(),
+                columnOption.isPresent() ?
+                    columnOption.get().getFormat().or(task.getDefaultTimestampFormat())
+                    : task.getDefaultTimestampFormat(),
+                columnOption.isPresent() ?
+                    columnOption.get().getTimeZone().or(task.getDefaultTimeZone())
+                    : task.getDefaultTimeZone());
+    }
+
     public TimestampFormatter(ScriptingContainer jruby, String format, DateTimeZone timeZone)
     {
         this.timeZone = timeZone;

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -1,8 +1,8 @@
 package org.embulk.spi.time;
 
 import org.joda.time.DateTimeZone;
+import com.google.common.base.Optional;
 import org.jruby.embed.ScriptingContainer;
-import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
@@ -11,8 +11,9 @@ import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
 
 public class TimestampParser
 {
+    @Deprecated
     public interface ParserTask
-            extends Task
+            extends org.embulk.config.Task
     {
         @Config("default_timezone")
         @ConfigDefault("\"UTC\"")
@@ -22,17 +23,50 @@ public class TimestampParser
         public ScriptingContainer getJRuby();
     }
 
+    public interface Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("default_timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%N %z\"")
+        public String getDefaultTimestampFormat();
+
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public interface TimestampColumnOption
+    {
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+
+        @Config("format")
+        @ConfigDefault("null")
+        public Optional<String> getFormat();
+    }
+
     private final JRubyTimeParserHelper helper;
     private final DateTimeZone defaultTimeZone;
 
+    @Deprecated
     public TimestampParser(String format, ParserTask task)
     {
         this(task.getJRuby(), format, task.getDefaultTimeZone());
     }
 
-    public DateTimeZone getDefaultTimeZone()
+    TimestampParser(Task task)
     {
-        return defaultTimeZone;
+        this(task.getJRuby(), task.getDefaultTimestampFormat(), task.getDefaultTimeZone());
+    }
+
+    public TimestampParser(Task task, TimestampColumnOption columnOption)
+    {
+        this(task.getJRuby(),
+                columnOption.getFormat().or(task.getDefaultTimestampFormat()),
+                columnOption.getTimeZone().or(task.getDefaultTimeZone()));
     }
 
     public TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone)
@@ -43,6 +77,11 @@ public class TimestampParser
         this.defaultTimeZone = defaultTimeZone;
     }
 
+    public DateTimeZone getDefaultTimeZone()
+    {
+        return defaultTimeZone;
+    }
+
     public Timestamp parse(String text) throws TimestampParseException
     {
         long localUsec = helper.strptimeUsec(text);

data/embulk-core/src/main/java/org/embulk/spi/type/TimestampType.java

@@ -22,12 +22,14 @@ public class TimestampType
         this.format = format;
     }
 
+    @Deprecated
     public TimestampType withFormat(String format)
     {
         // TODO is this correct design...?
         return new TimestampType(format);
     }
 
+    @Deprecated
     public String getFormat()
     {
         if (format == null) {

data/embulk-core/src/test/java/org/embulk/spi/PageTestUtils.java

@@ -17,28 +17,7 @@ import com.google.common.collect.ImmutableList;
 public class PageTestUtils
 {
     private PageTestUtils()
-    {
-    }
-
-    public static Schema newSchema()
-    {
-        return new Schema(ImmutableList.<Column> of());
-    }
-
-    public static Schema newSchema(Column... columns)
-    {
-        return new Schema(ImmutableList.copyOf(columns));
-    }
-
-    public static Schema newSchema(ColumnConfig... columns)
-    {
-        return new SchemaConfig(ImmutableList.copyOf(columns)).toSchema();
-    }
-
-    public static ColumnConfig newColumn(String name, Type type)
-    {
-        return new ColumnConfig(name, type, null);
-    }
+    { }
 
     public static List<Page> buildPage(BufferAllocator bufferAllocator,
             Schema schema, Object... values)

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -117,11 +117,11 @@ public class TestFileInputRunner
         ConfigSource config = Exec.newConfigSource().set(
                 "parser",
                 ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean"),
-                        ImmutableMap.of("name", "col2", "type", "long"),
-                        ImmutableMap.of("name", "col3", "type", "double"),
-                        ImmutableMap.of("name", "col4", "type", "string"),
-                        ImmutableMap.of("name", "col5", "type", "timestamp"))));
+                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()))));
 
         final MockPageOutput output = new MockPageOutput();
         runner.transaction(config, new InputPlugin.Control()
@@ -167,11 +167,11 @@ public class TestFileInputRunner
         ConfigSource config = Exec.newConfigSource().set(
                 "parser",
                 ImmutableMap.of("type", "mock", "columns", ImmutableList.of(
-                        ImmutableMap.of("name", "col1", "type", "boolean"),
-                        ImmutableMap.of("name", "col2", "type", "long"),
-                        ImmutableMap.of("name", "col3", "type", "double"),
-                        ImmutableMap.of("name", "col4", "type", "string"),
-                        ImmutableMap.of("name", "col5", "type", "timestamp"))));
+                        ImmutableMap.of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col2", "type", "long", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col3", "type", "double", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col4", "type", "string", "option", ImmutableMap.of()),
+                        ImmutableMap.of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()))));
 
         final MockPageOutput output = new MockPageOutput();
 

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -105,12 +105,12 @@ public class TestFileOutputRunner
         MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
         final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
 
-        ImmutableList<ImmutableMap<String, String>> columns = ImmutableList.of(
-                ImmutableMap.of("name", "col1", "type", "boolean"),
-                ImmutableMap.of("name", "col2", "type", "long"),
-                ImmutableMap.of("name", "col3", "type", "double"),
-                ImmutableMap.of("name", "col4", "type", "string"),
-                ImmutableMap.of("name", "col5", "type", "timestamp"));
+        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
+                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()));
         ConfigSource config = Exec
                 .newConfigSource()
                 .set("type", "unused?")
@@ -163,12 +163,12 @@ public class TestFileOutputRunner
         MockFileOutputPlugin fileOutputPlugin = new MockFileOutputPlugin();
         final FileOutputRunner runner = new FileOutputRunner(fileOutputPlugin);
 
-        ImmutableList<ImmutableMap<String, String>> columns = ImmutableList.of(
-                ImmutableMap.of("name", "col1", "type", "boolean"),
-                ImmutableMap.of("name", "col2", "type", "long"),
-                ImmutableMap.of("name", "col3", "type", "double"),
-                ImmutableMap.of("name", "col4", "type", "string"),
-                ImmutableMap.of("name", "col5", "type", "timestamp"));
+        ImmutableList<ImmutableMap<String, Object>> columns = ImmutableList.of(
+                ImmutableMap.<String,Object>of("name", "col1", "type", "boolean", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col2", "type", "long", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col3", "type", "double", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col4", "type", "string", "option", ImmutableMap.of()),
+                ImmutableMap.<String,Object>of("name", "col5", "type", "timestamp", "option", ImmutableMap.of()));
         ConfigSource config = Exec
                 .newConfigSource()
                 .set("type", "unused?")

data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java

@@ -1,7 +1,5 @@
 package org.embulk.spi;
 
-import static org.embulk.spi.PageTestUtils.newColumn;
-import static org.embulk.spi.PageTestUtils.newSchema;
 import static org.embulk.spi.type.Types.BOOLEAN;
 import static org.embulk.spi.type.Types.DOUBLE;
 import static org.embulk.spi.type.Types.LONG;
@@ -77,66 +75,81 @@ public class TestPageBuilderReader
     @Test
     public void testBoolean()
     {
-        check(newSchema(newColumn("col1", BOOLEAN)), false, true, true);
+        check(Schema.builder().add("col1", BOOLEAN).build(),
+                false, true, true);
     }
 
     @Test
     public void testLong()
     {
-        check(newSchema(newColumn("col1", LONG)), 1L, Long.MIN_VALUE,
-                Long.MAX_VALUE);
+        check(Schema.builder().add("col1", LONG).build(),
+                1L, Long.MIN_VALUE, Long.MAX_VALUE);
     }
 
     @Test
     public void testDouble()
     {
-        check(newSchema(newColumn("col1", DOUBLE)), 8.1, 3.141592, 4.3);
+        check(Schema.builder().add("col1", DOUBLE).build(),
+                8.1, 3.141592, 4.3);
     }
 
     @Test
     public void testUniqueStrings()
     {
-        check(newSchema(newColumn("col1", STRING)), "test1", "test2", "test0");
+        check(Schema.builder().add("col1", STRING).build(),
+                "test1", "test2", "test0");
     }
 
     @Test
     public void testDuplicateStrings()
     {
-        check(newSchema(newColumn("col1", STRING)), "test1", "test1", "test1");
+        check(Schema.builder().add("col1", STRING).build(),
+            "test1", "test1", "test1");
     }
 
     @Test
     public void testDuplicateStringsMultiColumns()
     {
-        check(newSchema(newColumn("col1", STRING), newColumn("col1", STRING)),
-                "test2", "test1", "test1", "test2", "test2", "test0", "test1",
-                "test1");
+        check(Schema.builder().add("col1", STRING).add("col1", STRING).build(),
+                "test2", "test1",
+                "test1", "test2",
+                "test2", "test0",
+                "test1", "test1");
     }
 
     @Test
     public void testTimestamp()
     {
-        check(newSchema(newColumn("col1", TIMESTAMP)),
+        check(Schema.builder().add("col1", TIMESTAMP).build(),
                 Timestamp.ofEpochMilli(0), Timestamp.ofEpochMilli(10));
     }
 
     @Test
     public void testNull()
     {
-        check(newSchema(newColumn("col3", DOUBLE), newColumn("col1", STRING),
-                newColumn("col3", LONG), newColumn("col3", BOOLEAN),
-                newColumn("col2", TIMESTAMP)), null, null, null, null, null,
+        check(Schema.builder()
+                    .add("col3", DOUBLE)
+                    .add("col1", STRING)
+                    .add("col3", LONG)
+                    .add("col3", BOOLEAN)
+                    .add("col2", TIMESTAMP)
+                    .build(),
+                null, null, null, null, null,
                 null, null, null, null, null);
     }
 
     @Test
     public void testMixedTypes()
     {
-        check(newSchema(newColumn("col3", DOUBLE), newColumn("col1", STRING),
-                newColumn("col3", LONG), newColumn("col3", BOOLEAN),
-                newColumn("col2", TIMESTAMP)), 8122.0, "val1", 3L, false,
-                Timestamp.ofEpochMilli(0), 140.15, "val2", Long.MAX_VALUE,
-                true, Timestamp.ofEpochMilli(10));
+        check(Schema.builder()
+                    .add("col3", DOUBLE)
+                    .add("col1", STRING)
+                    .add("col3", LONG)
+                    .add("col3", BOOLEAN)
+                    .add("col2", TIMESTAMP)
+                    .build(),
+                8122.0, "val1", 3L, false, Timestamp.ofEpochMilli(0),
+                140.15, "val2", Long.MAX_VALUE, true, Timestamp.ofEpochMilli(10));
     }
 
     private void check(Schema schema, Object... objects)
@@ -218,12 +231,12 @@ public class TestPageBuilderReader
     public void testEmptySchema()
     {
         MockPageOutput output = new MockPageOutput();
-        this.builder = new PageBuilder(bufferAllocator, newSchema(), output);
+        this.builder = new PageBuilder(bufferAllocator, Schema.builder().build(), output);
         builder.addRecord();
         builder.addRecord();
         builder.flush();
         builder.close();
-        this.reader = new PageReader(newSchema());
+        this.reader = new PageReader(Schema.builder().build());
         assertEquals(1, output.pages.size());
         reader.setPage(output.pages.get(0));
         assertTrue(reader.nextRecord());
@@ -250,8 +263,8 @@ public class TestPageBuilderReader
         };
         assertEquals(
                 9,
-                buildPages(newSchema(newColumn("col1", LONG)), 0L, 1L, 2L, 3L,
-                        4L, 5L, 6L, 7L, 8L).size());
+                buildPages(Schema.builder().add("col1", LONG).build(),
+                    0L, 1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L).size());
     }
 
     @Test
@@ -274,17 +287,22 @@ public class TestPageBuilderReader
         assertEquals(
                 3,
                 buildPages(
-                        newSchema(newColumn("col1", LONG),
-                                newColumn("col1", STRING)), 0L, "record0", 1L,
-                        "record1", 3L, "record3").size());
+                        Schema.builder()
+                            .add("col1", LONG)
+                            .add("col1", STRING)
+                            .build(),
+                        0L, "record0",
+                        1L, "record1",
+                        3L, "record3"
+                ).size());
     }
 
     @Test
     public void testRepeatableClose()
     {
         MockPageOutput output = new MockPageOutput();
-        this.builder = new PageBuilder(bufferAllocator, newSchema(newColumn(
-                "col1", STRING)), output);
+        this.builder = new PageBuilder(bufferAllocator,
+                Schema.builder().add("col1", STRING).build(), output);
         builder.close();
         builder.close();
     }
@@ -293,8 +311,8 @@ public class TestPageBuilderReader
     public void testRepeatableFlush()
     {
         MockPageOutput output = new MockPageOutput();
-        this.builder = new PageBuilder(bufferAllocator, newSchema(newColumn(
-                "col1", STRING)), output);
+        this.builder = new PageBuilder(bufferAllocator,
+                Schema.builder().add("col1", STRING).build(), output);
         builder.flush();
         builder.flush();
     }