embulk 0.6.13 → 0.6.14

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java

@@ -3,7 +3,9 @@ package org.embulk.spi.time;
 import org.junit.Rule;
 import org.junit.Before;
 import org.junit.Test;
+import com.google.common.base.Optional;
 import static org.junit.Assert.assertEquals;
+import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.Exec;
@@ -15,53 +17,47 @@ public class TestTimestampFormatterParser
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
     private interface FormatterTestTask
-            extends TimestampFormatter.FormatterTask
-    {
-        @Config("time_format")
-        public TimestampFormat getTimeFormat();
-    }
+            extends Task, TimestampFormatter.Task
+    { }
 
     private interface ParserTestTask
-            extends TimestampParser.ParserTask
-    {
-        @Config("time_format")
-        public TimestampFormat getTimeFormat();
-    }
+            extends Task, TimestampParser.Task
+    { }
 
     @Test
     public void testSimpleFormat() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("time_format", "%Y-%m-%d %H:%M:%S.%6N %Z");
+            .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S.%9N %Z");
         FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
 
-        TimestampFormatter formatter = task.getTimeFormat().newFormatter(task);
-        assertEquals("2014-11-19 02:46:29.123456 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
+        TimestampFormatter formatter = new TimestampFormatter(task, Optional.<TimestampFormatter.TimestampColumnOption>absent());
+        assertEquals("2014-11-19 02:46:29.123456000 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
     }
 
     @Test
     public void testSimpleParse() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("time_format", "%Y-%m-%d %H:%M:%S.%N %Z");
+            .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S %Z");
         ParserTestTask task = config.loadConfig(ParserTestTask.class);
 
-        TimestampParser parser = task.getTimeFormat().newParser(task);
-        assertEquals(Timestamp.ofEpochSecond(1416365189, 123456*1000), parser.parse("2014-11-19 02:46:29.123456 UTC"));
+        TimestampParser parser = new TimestampParser(task);
+        assertEquals(Timestamp.ofEpochSecond(1416365189, 0), parser.parse("2014-11-19 02:46:29 UTC"));
     }
 
     @Test
     public void testUnixtimeFormat() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("time_format", "%s");
+            .set("default_timestamp_format", "%s");
 
         FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class);
-        TimestampFormatter formatter = ftask.getTimeFormat().newFormatter(ftask);
+        TimestampFormatter formatter = new TimestampFormatter(ftask, Optional.<TimestampFormatter.TimestampColumnOption>absent());
         assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
 
         ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
-        TimestampParser parser = ptask.getTimeFormat().newParser(ptask);
+        TimestampParser parser = new TimestampParser(ptask);
         assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
     }
 }

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java

@@ -0,0 +1,67 @@
+package org.embulk.spi.time;
+
+import org.junit.Rule;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestTimestampFormatterParserDeprecated
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private interface FormatterTestTask
+            extends TimestampFormatter.FormatterTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    private interface ParserTestTask
+            extends TimestampParser.ParserTask
+    {
+        @Config("time_format")
+        public TimestampFormat getTimeFormat();
+    }
+
+    @Test
+    public void testSimpleFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%6N %Z");
+        FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
+
+        TimestampFormatter formatter = task.getTimeFormat().newFormatter(task);
+        assertEquals("2014-11-19 02:46:29.123456 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
+    }
+
+    @Test
+    public void testSimpleParse() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%N %Z");
+        ParserTestTask task = config.loadConfig(ParserTestTask.class);
+
+        TimestampParser parser = task.getTimeFormat().newParser(task);
+        assertEquals(Timestamp.ofEpochSecond(1416365189, 123456*1000), parser.parse("2014-11-19 02:46:29.123456 UTC"));
+    }
+
+    @Test
+    public void testUnixtimeFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%s");
+
+        FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class);
+        TimestampFormatter formatter = ftask.getTimeFormat().newFormatter(ftask);
+        assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+
+        ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
+        TimestampParser parser = ptask.getTimeFormat().newParser(ptask);
+        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
+    }
+}

data/embulk-docs/src/built-in.rst

@@ -309,6 +309,10 @@ Options
 +----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
 | charset              | enum    | Character encoding (eg. ISO-8859-1, UTF-8)                                                            | ``UTF-8`` by default   |
 +----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| default\_timezone    | string  | Time zone of timestamp columns. This can be overwritten for each column using ``column_options``      | ``UTC`` by default     |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| column\_options      | hash    | See bellow                                                                                            | optional               |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
 
 The ``quote_policy`` option is used to determine field type to quote.
 
@@ -322,6 +326,16 @@ The ``quote_policy`` option is used to determine field type to quote.
 | NONE       | Never quote fields. When the delimiter occurs in field, escape with escape char                        |
 +------------+--------------------------------------------------------------------------------------------------------+
 
+The ``column_options`` option is a map whose keys are name of columns, and values are configuration with following parameters:
+
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+| name                 | type    | description                                                                                           | required?                               |
++======================+=========+=======================================================================================================+=========================================+
+| timezone             | string  | Time zone if type of this column is timestamp. If not set, ``default\_timezone`` is used.             | optional                                |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+| format               | string  | Timestamp format if type of this column is timestamp.                                                 | ``%Y-%m-%d %H:%M:%S.%6N %z`` by default |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
+
 Example
 ~~~~~~~~~~~~~~~~~~
 
@@ -339,6 +353,10 @@ Example
         quote: '"'
         escape: '\\'
         null_string: '\\N'
+        default_timezone: 'UTC'
+        column_options:
+          mycol1: {format: '%Y-%m-%d %H:%M:%S'}
+          mycol2: {format: '%Y-%m-%d %H:%M:%S', timezone: 'America/Los_Angeles'}
 
 Gzip encoder plugin
 ------------------

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.14
     release/release-0.6.13
     release/release-0.6.12
     release/release-0.6.11

data/embulk-docs/src/release/release-0.6.14.rst

@@ -0,0 +1,47 @@
+Release 0.6.14
+==================================
+
+Built-in plugins
+------------------
+
+* ``formatter-csv`` plugin supports ``default_timezone`` and ``column_options`` parameters so that we can set timestamp format for each columns individually.
+* **IMPORTANT**: ``formatter-csv`` does not use the same timestamp format with input timestamp columns. If you're using ``formatter: type: csv`` with ``parser: type: csv-``, you need to set ``format`` option for each columns to keep using current behavior. See ``column_options`` option of CSV formatter plugin described at :doc:`../built-in`.
+* ``guess-csv`` plugin keeps using ``delimiter`` option if already set rather than to overwrite it everytime.
+
+
+Plugin API
+------------------
+
+* Added ``config.DataSource#remove(String)`` method.
+* Added ``spi.ColumnConfig.getOption().``
+
+  * ``spi.type.TimestampType.getFormat()`` is deprecated.
+  * ``spi.ColumnConfig.getFormat()`` is deprecated.
+  * ``spi.ProcessTask`` does not serialize ``TimestampType.format`` any more.
+
+* Added utility methods for ``spi.Schema`` and ``spi.SchemaConfig``.
+
+  * Added ``Schema.Builder`` class and ``Schema.builder()`` method.
+  * Added ``Schema#lookupColumn(String)``.
+  * Added utility methods to ``SchemaConfig``.
+
+* Added ``spi.time.TimestampFormatter.Task`` and TimestampColumnOption
+
+  * ``TimestampFormatter.FormatterTask`` is deprecated
+
+* Added ``spi.time.TimestampParser.Task`` and TimestampColumnOption
+
+  * ``TimestampParser.ParserTask`` is deprecated
+
+* ``spi.time.TimestampFormat`` is deprecated
+
+
+General Changes
+------------------
+
+* Fixed a problem where embulk shows input plugin name and version twice to log.
+
+
+Release Date
+------------------
+2015-06-29

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -8,6 +8,7 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.Column;
@@ -47,7 +48,7 @@ public class CsvFormatterPlugin
     }
 
     public interface PluginTask
-            extends LineEncoder.EncoderTask, TimestampFormatter.FormatterTask
+            extends Task, LineEncoder.EncoderTask, TimestampFormatter.Task
     {
         @Config("header_line")
         @ConfigDefault("true")
@@ -76,27 +77,43 @@ public class CsvFormatterPlugin
         @Config("newline_in_field")
         @ConfigDefault("\"LF\"")
         public Newline getNewlineInField();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        public Map<String, TimestampColumnOption> getColumnOptions();
     }
 
+    public interface TimestampColumnOption
+            extends Task, TimestampFormatter.TimestampColumnOption
+    { }
+
     @Override
     public void transaction(ConfigSource config, Schema schema,
             FormatterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
+
+        // validate column_options
+        for (String columnName : task.getColumnOptions().keySet()) {
+            schema.lookupColumn(columnName);  // throws SchemaConfigException
+        }
+
         control.run(task.dump());
     }
 
-    private Map<Integer, TimestampFormatter> newTimestampFormatters(
-            TimestampFormatter.FormatterTask task, Schema schema)
+    private TimestampFormatter[] newTimestampFormatters(
+            TimestampFormatter.Task formatterTask, Schema schema,
+            Map<String, TimestampColumnOption> columnOptions)
     {
-        ImmutableMap.Builder<Integer, TimestampFormatter> builder = new ImmutableBiMap.Builder<>();
+        TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()];
+        int i = 0;
         for (Column column : schema.getColumns()) {
             if (column.getType() instanceof TimestampType) {
-                TimestampType tt = (TimestampType) column.getType();
-                builder.put(column.getIndex(), new TimestampFormatter(tt.getFormat(), task));
+                Optional<TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName()));
+                formatters[i] = new TimestampFormatter(formatterTask, option);
             }
         }
-        return builder.build();
+        return formatters;
     }
 
     @Override
@@ -105,8 +122,7 @@ public class CsvFormatterPlugin
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final LineEncoder encoder = new LineEncoder(output, task);
-        final Map<Integer, TimestampFormatter> timestampFormatters =
-                newTimestampFormatters(task, schema);
+        final TimestampFormatter[] timestampFormatters = newTimestampFormatters(task, schema, task.getColumnOptions());
         final char delimiter = task.getDelimiterChar();
         final QuotePolicy quotePolicy = task.getQuotePolicy();
         final char quote = task.getQuoteChar() != '\0' ? task.getQuoteChar() : '"';
@@ -176,7 +192,7 @@ public class CsvFormatterPlugin
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
                                 Timestamp value = pageReader.getTimestamp(column);
-                                addValue(timestampFormatters.get(column.getIndex()).format(value));
+                                addValue(timestampFormatters[column.getIndex()].format(value));
                             } else {
                                 addNullString();
                             }

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -9,9 +9,11 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.time.TimestampFormat;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.ColumnVisitor;
@@ -35,7 +37,7 @@ public class CsvParserPlugin
                 "1");
 
     public interface PluginTask
-            extends Task, LineDecoder.DecoderTask, TimestampParser.ParserTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
     {
         @Config("columns")
         public SchemaConfig getSchemaConfig();
@@ -86,8 +88,19 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         public boolean getAllowExtraColumns();
+
+        @Config("error")
+        @ConfigDefault("{\"type\": \"warning\"}")
+        public ConfigSource getErrorConfig();
+
+        public TaskSource getErorrTaskSource();
+        public void setErrorTaskSource(TaskSource errorTask);
     }
 
+    public interface TimestampColumnOption
+            extends Task, TimestampParser.TimestampColumnOption
+    { }
+
     private final Logger log;
 
     public CsvParserPlugin()
@@ -116,14 +129,16 @@ public class CsvParserPlugin
     }
 
     private TimestampParser[] newTimestampParsers(
-            TimestampParser.ParserTask task, Schema schema)
+            TimestampParser.Task parserTask, SchemaConfig schema)
     {
         TimestampParser[] parsers = new TimestampParser[schema.getColumnCount()];
-        for (Column column : schema.getColumns()) {
+        int i = 0;
+        for (ColumnConfig column : schema.getColumns()) {
             if (column.getType() instanceof TimestampType) {
-                TimestampType tt = (TimestampType) column.getType();
-                parsers[column.getIndex()] = new TimestampParser(tt.getFormat(), task);
+                TimestampColumnOption option = column.getOption().loadConfig(TimestampColumnOption.class);
+                parsers[i] = new TimestampParser(parserTask, option);
             }
+            i++;
         }
         return parsers;
     }
@@ -133,7 +148,7 @@ public class CsvParserPlugin
             FileInput input, PageOutput output)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        final TimestampParser[] timestampFormatters = newTimestampParsers(task, schema);
+        final TimestampParser[] timestampFormatters = newTimestampParsers(task, task.getSchemaConfig());
         LineDecoder lineDecoder = new LineDecoder(input, task);
         final CsvTokenizer tokenizer = new CsvTokenizer(lineDecoder, task);
         final String nullStringOrNull = task.getNullString().orNull();

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.standards;
 
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.joda.time.DateTimeZone;
 import org.junit.Rule;
 import org.junit.Test;
 import java.lang.reflect.InvocationTargetException;
@@ -32,6 +33,8 @@ public class TestCsvFormatterPlugin
         assertEquals(CsvFormatterPlugin.QuotePolicy.MINIMAL, task.getQuotePolicy());
         assertEquals(false, task.getEscapeChar().isPresent());
         assertEquals("", task.getNullString());
+        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
+        assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
         assertEquals(Newline.LF, task.getNewlineInField());
     }
 

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java

@@ -6,6 +6,7 @@ import static org.junit.Assert.assertEquals;
 import java.nio.charset.Charset;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
+import org.joda.time.DateTimeZone;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
@@ -34,6 +35,8 @@ public class TestCsvParserPlugin
         assertEquals(',', task.getDelimiterChar());
         assertEquals('\"', task.getQuoteChar());
         assertEquals(false, task.getAllowOptionalColumns());
+        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
+        assertEquals("%Y-%m-%d %H:%M:%S.%N %z", task.getDefaultTimestampFormat());
     }
 
     @Test(expected = ConfigException.class)

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java

@@ -41,12 +41,10 @@ public class TestCsvTokenizer
         config = Exec.newConfigSource()
             .set("newline", "LF")
             .set("columns", ImmutableList.of(
-                        ImmutableMap.of(
-                            "name", "date_code",
-                            "type", "string"),
-                        ImmutableMap.of(
-                            "name", "foo",
-                            "type", "string"))
+                        ImmutableMap.<String,Object>of(
+                            "name", "date_code", "type", "string", "option", ImmutableMap.of()),
+                        ImmutableMap.<String,Object>of(
+                            "name", "foo", "type", "string", "option", ImmutableMap.of()))
                 );
         reloadPluginTask();
     }

data/lib/embulk/guess/csv.rb

@@ -35,13 +35,17 @@ module Embulk
       def guess_lines(config, sample_lines)
         return {} unless config.fetch("parser", {}).fetch("type", "csv") == "csv"
 
-        delim = guess_delimiter(sample_lines)
-        unless delim
-          # not CSV file
-          return {}
+        parser_config = config["parser"] || {}
+        if parser_config["type"] == "csv" && parser_config["delimiter"]
+          delim = parser_config["delimiter"]
+        else
+          delim = guess_delimiter(sample_lines)
+          unless delim
+            # not CSV file
+            return {}
+          end
         end
 
-        parser_config = config["parser"] || {}
         parser_guessed = DataSource.new.merge(parser_config).merge({"type" => "csv", "delimiter" => delim})
 
         unless parser_guessed.has_key?("quote")