embulk-parser-avro 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 62671c1f3feefa2bb7feecccf4362b04f0ad7d3c
-  data.tar.gz: 68e76d9d06bcc2f30c3d6e376e1fe867027b6e09
+  metadata.gz: 2ee00ffbaf0cf78dfc2c7bd0861e68603b528565
+  data.tar.gz: 08786ae000735a7f8d2aad1514a02de9c5095ec4
 SHA512:
-  metadata.gz: 79e628c578df06d5aa54e17a0038eeb65fa667318be469cdceab9161a4b40716634d1c9cbdd88d17ef47a8d85cf9d4d63939c6340b3efde74995070e3ad37758
-  data.tar.gz: d1120457b865feed3eab7560f3dff20ab0b767fe17653d6970626836460426135334f0f681f901405ce8107d4a0f3c3fb5aa14e302023132fbad48a22c538168
+  metadata.gz: 2697fb6b9cd4fb2cf6a72194450031f11ade7780d2afb050d40d65de39a10a5b56da507125b5580de820ba78e399ff39348726bb0eb82edb0c7aae13de27a555
+  data.tar.gz: 839d69c7a19dfc6eeddff3b4fb9f2bee3172b1d4ab05793753bae1fb466ef1bfa91c5e86e099f3fe44bc5c0e0047ed3a34eb789ae26c7e0507b29c25a58ce4ca

data/.gitignore

@@ -5,6 +5,7 @@
 .gradle/
 /classpath/
 build/
+out/
 .idea
 /.settings/
 /.metadata/

data/README.md

@@ -12,11 +12,37 @@
 - **type**: Specify this parser as avro
 - **avsc**: Specify avro schema file.
 - **columns**: Specify column name and type. See below (array, optional)
+  - timestamp_unit: Specify unit of time. (This config is effective only if avro value is `long`, `int`, `float`, `double`)
 * **default_timezone**: Default timezone of the timestamp (string, default: UTC)
 * **default_timestamp_format**: Default timestamp format of the timestamp (string, default: `%Y-%m-%d %H:%M:%S.%N %z`)
 
 If columns is not set, this plugin detect schema automatically by using avsc schema.
 
+support `timestamp_unit` type is below.
+
+- "Second"
+- "second"
+- "sec"
+- "s"
+- "MilliSecond"
+- "millisecond"
+- "milli_second"
+- "milli"
+- "msec"
+- "ms"
+- "MicroSecond"
+- "microsecond"
+- "micro_second"
+- "micro"
+- "usec"
+- "us"
+- "NanoSecond"
+- "nanosecond"
+- "nano_second"
+- "nano"
+- "nsec"
+- "ns"
+
 ## Example
 
 ```yaml
@@ -38,7 +64,7 @@ in:
       - {name: "options", type: "json"}
       - {name: "spec", type: "json"}
       - {name: "created_at", type: "timestamp", format: "%Y-%m-%dT%H:%M:%S%:z"}
-      - {name: "created_at_utc", type: "timestamp"}
+      - {name: "created_at_utc", type: "timestamp", timestamp_unit: "second"}
 
 out:
   type: stdout

data/build.gradle

@@ -13,19 +13,19 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
 
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.14"
-    provided "org.embulk:embulk-core:0.8.14"
-    compile "org.apache.avro:avro:1.8.0"
+    compile  "org.embulk:embulk-core:0.9.4"
+    provided "org.embulk:embulk-core:0.9.4"
+    compile "org.apache.avro:avro:1.8.2"
     testCompile "junit:junit:4.+"
 
-    testCompile  "org.embulk:embulk-core:0.8.14:tests"
-    testCompile  "org.embulk:embulk-standards:0.8.14"
+    testCompile  "org.embulk:embulk-core:0.9.4:tests"
+    testCompile  "org.embulk:embulk-standards:0.9.4"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/parser/avro/AvroParserPlugin.java

@@ -15,6 +15,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.parser.avro.getter.BaseColumnGetter;
 import org.embulk.parser.avro.getter.ColumnGetterFactory;
 import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.PageBuilder;
@@ -23,6 +24,7 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Types;
 import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.util.FileInputInputStream;
@@ -46,6 +48,13 @@ public class AvroParserPlugin
         LocalFile getAvsc();
     }
 
+    public interface TimestampUnitConfig extends Task
+    {
+        @Config("timestamp_unit")
+        @ConfigDefault("\"second\"")
+        public TimestampUnit getTimestampUnit();
+    }
+
     @Override
     public void transaction(ConfigSource config, ParserPlugin.Control control)
     {
@@ -128,6 +137,7 @@ public class AvroParserPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
         List<Column> columns = schema.getColumns();
         final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getColumns());
+        final TimestampUnit[] timestampUnits = newTimestampUnits(task.getColumns());
         File avsc = task.getAvsc().getFile();
         final org.apache.avro.Schema avroSchema;
         try {
@@ -137,7 +147,7 @@ public class AvroParserPlugin
         }
 
         try (FileInputInputStream is = new FileInputInputStream(input); final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-            ColumnGetterFactory factory = new ColumnGetterFactory(avroSchema, pageBuilder, timestampParsers);
+            ColumnGetterFactory factory = new ColumnGetterFactory(avroSchema, pageBuilder, timestampParsers, timestampUnits);
             ImmutableMap.Builder<String, BaseColumnGetter> columnGettersBuilder = ImmutableMap.builder();
             for (Column column : columns) {
                 BaseColumnGetter columnGetter = factory.newColumnGetter(column);
@@ -165,4 +175,17 @@ public class AvroParserPlugin
             throw new RuntimeException(e);
         }
     }
+
+    private TimestampUnit[] newTimestampUnits(SchemaConfig columns) {
+        TimestampUnit[] units = new TimestampUnit[columns.getColumnCount()];
+        int i = 0;
+        for (ColumnConfig column : columns.getColumns()) {
+            if (column.getType() instanceof TimestampType) {
+                TimestampUnitConfig option = column.getOption().loadConfig(TimestampUnitConfig.class);
+                units[i] = option.getTimestampUnit();
+            }
+            i++;
+        }
+        return units;
+    }
 }

data/src/main/java/org/embulk/parser/avro/TimestampUnit.java

@@ -0,0 +1,70 @@
+package org.embulk.parser.avro;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import org.embulk.spi.time.Timestamp;
+
+
+@JsonDeserialize(using=TimestampUnitDeserializer.class)
+public enum TimestampUnit {
+    Second {
+        @Override
+        public Timestamp toTimestamp(Long value)
+        {
+            return Timestamp.ofEpochSecond(value);
+        }
+
+        @Override
+        public Timestamp toTimestamp(Double value)
+        {
+            long sec = value.longValue();
+            double rest = value - sec;
+            return Timestamp.ofEpochSecond(0, sec * 1000000000L + (long) (rest * 1000000000L));
+        }
+    },
+    MilliSecond {
+        @Override
+        public Timestamp toTimestamp(Long value)
+        {
+            return Timestamp.ofEpochSecond(0, value * 1000000L);
+        }
+
+        @Override
+        public Timestamp toTimestamp(Double value)
+        {
+            long sec = value.longValue();
+            double rest = value - sec;
+            return Timestamp.ofEpochSecond(0, sec * 1000000L + (long) (rest * 1000000L));
+        }
+    },
+    MicroSecond {
+        @Override
+        public Timestamp toTimestamp(Long value)
+        {
+            return Timestamp.ofEpochSecond(0, value * 1000L);
+        }
+
+        @Override
+        public Timestamp toTimestamp(Double value)
+        {
+            long sec = value.longValue();
+            double rest = value - sec;
+            return Timestamp.ofEpochSecond(0, sec * 1000L + (long) (rest * 1000L));
+        }
+    },
+    NanoSecond {
+        @Override
+        public Timestamp toTimestamp(Long value)
+        {
+            return Timestamp.ofEpochSecond(0, value);
+        }
+
+        @Override
+        public Timestamp toTimestamp(Double value)
+        {
+            return Timestamp.ofEpochSecond(0, value.longValue());
+        }
+    };
+
+    abstract public Timestamp toTimestamp(Long value);
+    abstract public Timestamp toTimestamp(Double value);
+}

data/src/main/java/org/embulk/parser/avro/TimestampUnitDeserializer.java

@@ -0,0 +1,55 @@
+package org.embulk.parser.avro;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableMap;
+
+import java.io.IOException;
+
+public class TimestampUnitDeserializer extends FromStringDeserializer<TimestampUnit>
+{
+
+    public static ImmutableMap<String, TimestampUnit> mapping = ImmutableMap.<String, TimestampUnit>builder()
+            .put("Second", TimestampUnit.Second)
+            .put("second", TimestampUnit.Second)
+            .put("sec", TimestampUnit.Second)
+            .put("s", TimestampUnit.Second)
+            .put("MilliSecond", TimestampUnit.MilliSecond)
+            .put("millisecond", TimestampUnit.MilliSecond)
+            .put("milli_second", TimestampUnit.MilliSecond)
+            .put("milli", TimestampUnit.MilliSecond)
+            .put("msec", TimestampUnit.MilliSecond)
+            .put("ms", TimestampUnit.MilliSecond)
+            .put("MicroSecond", TimestampUnit.MicroSecond)
+            .put("microsecond", TimestampUnit.MicroSecond)
+            .put("micro_second", TimestampUnit.MicroSecond)
+            .put("micro", TimestampUnit.MicroSecond)
+            .put("usec", TimestampUnit.MicroSecond)
+            .put("us", TimestampUnit.MicroSecond)
+            .put("NanoSecond", TimestampUnit.NanoSecond)
+            .put("nanosecond", TimestampUnit.NanoSecond)
+            .put("nano_second", TimestampUnit.NanoSecond)
+            .put("nano", TimestampUnit.NanoSecond)
+            .put("nsec", TimestampUnit.NanoSecond)
+            .put("ns", TimestampUnit.NanoSecond)
+            .build();
+
+    public TimestampUnitDeserializer() {
+        super(TimestampUnit.class);
+    }
+
+    @Override
+    protected TimestampUnit _deserialize(String value, DeserializationContext ctxt) throws IOException, JsonProcessingException {
+        TimestampUnit unit = mapping.get(value);
+        if (unit == null) {
+            throw new JsonMappingException(
+                    String.format("Unknown type name '%s'. Supported types are: %s",
+                            value,
+                            Joiner.on(", ").join(mapping.keySet())));
+        }
+        return unit;
+    }
+}

data/src/main/java/org/embulk/parser/avro/getter/ColumnGetterFactory.java

@@ -1,6 +1,7 @@
 package org.embulk.parser.avro.getter;
 
 import org.apache.avro.Schema;
+import org.embulk.parser.avro.TimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.DataException;
 import org.embulk.spi.PageBuilder;
@@ -10,12 +11,14 @@ public class ColumnGetterFactory {
     private org.apache.avro.Schema avroSchema;
     private PageBuilder pageBuilder;
     private TimestampParser[] timestampParsers;
+    private TimestampUnit[] timestampUnits;
 
-    public ColumnGetterFactory(org.apache.avro.Schema avroSchema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
+    public ColumnGetterFactory(org.apache.avro.Schema avroSchema, PageBuilder pageBuilder, TimestampParser[] timestampParsers, TimestampUnit[] timestampUnits)
     {
         this.avroSchema = avroSchema;
         this.pageBuilder = pageBuilder;
         this.timestampParsers = timestampParsers;
+        this.timestampUnits = timestampUnits;
     }
 
     public BaseColumnGetter newColumnGetter(Column column)
@@ -43,13 +46,13 @@ public class ColumnGetterFactory {
             case ENUM:
                 return new StringColumnGetter(pageBuilder, timestampParsers);
             case INT:
-                return new IntegerColumnGetter(pageBuilder, timestampParsers);
+                return new IntegerColumnGetter(pageBuilder, timestampParsers, timestampUnits);
             case LONG:
-                return new LongColumnGetter(pageBuilder, timestampParsers);
+                return new LongColumnGetter(pageBuilder, timestampParsers, timestampUnits);
             case FLOAT:
-                return new FloatColumnGetter(pageBuilder, timestampParsers);
+                return new FloatColumnGetter(pageBuilder, timestampParsers, timestampUnits);
             case DOUBLE:
-                return new DoubleColumnGetter(pageBuilder, timestampParsers);
+                return new DoubleColumnGetter(pageBuilder, timestampParsers, timestampUnits);
             case BOOLEAN:
                 return new BooleanColumnGetter(pageBuilder, timestampParsers);
             case ARRAY:

data/src/main/java/org/embulk/parser/avro/getter/DoubleColumnGetter.java

@@ -1,15 +1,18 @@
 package org.embulk.parser.avro.getter;
 
+import org.embulk.parser.avro.TimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 
 public class DoubleColumnGetter extends BaseColumnGetter {
     protected Double value;
 
-    public DoubleColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+    private final TimestampUnit[] timestampUnits;
+
+    public DoubleColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers, TimestampUnit[] timestampUnits) {
         super(pageBuilder, timestampParsers);
+        this.timestampUnits = timestampUnits;
     }
 
     @Override
@@ -51,8 +54,8 @@ public class DoubleColumnGetter extends BaseColumnGetter {
             pageBuilder.setNull(column);
         }
         else {
-            long milliSec = (long) (value * 1000);
-            pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(milliSec));
+            TimestampUnit unit = timestampUnits[column.getIndex()];
+            pageBuilder.setTimestamp(column, unit.toTimestamp(value));
         }
     }
 }

data/src/main/java/org/embulk/parser/avro/getter/FloatColumnGetter.java

@@ -1,15 +1,18 @@
 package org.embulk.parser.avro.getter;
 
+import org.embulk.parser.avro.TimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 
 public class FloatColumnGetter extends BaseColumnGetter {
     protected Float value;
 
-    public FloatColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+    private final TimestampUnit[] timestampUnits;
+
+    public FloatColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers, TimestampUnit[] timestampUnits) {
         super(pageBuilder, timestampParsers);
+        this.timestampUnits = timestampUnits;
     }
 
     @Override
@@ -51,8 +54,8 @@ public class FloatColumnGetter extends BaseColumnGetter {
             pageBuilder.setNull(column);
         }
         else {
-            long milliSec = (long) (value * 1000);
-            pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(milliSec));
+            TimestampUnit unit = timestampUnits[column.getIndex()];
+            pageBuilder.setTimestamp(column, unit.toTimestamp(value.doubleValue()));
         }
     }
 }

data/src/main/java/org/embulk/parser/avro/getter/IntegerColumnGetter.java

@@ -1,15 +1,18 @@
 package org.embulk.parser.avro.getter;
 
+import org.embulk.parser.avro.TimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 
 public class IntegerColumnGetter extends BaseColumnGetter {
     protected Integer value;
 
-    public IntegerColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+    private final TimestampUnit[] timestampUnits;
+
+    public IntegerColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers, TimestampUnit[] timestampUnits) {
         super(pageBuilder, timestampParsers);
+        this.timestampUnits = timestampUnits;
     }
 
     @Override
@@ -51,7 +54,8 @@ public class IntegerColumnGetter extends BaseColumnGetter {
             pageBuilder.setNull(column);
         }
         else {
-            pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(value.longValue()));
+            TimestampUnit unit = timestampUnits[column.getIndex()];
+            pageBuilder.setTimestamp(column, unit.toTimestamp(value.longValue()));
         }
     }
 }

data/src/main/java/org/embulk/parser/avro/getter/LongColumnGetter.java

@@ -1,15 +1,18 @@
 package org.embulk.parser.avro.getter;
 
+import org.embulk.parser.avro.TimestampUnit;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 
 public class LongColumnGetter extends BaseColumnGetter {
     protected Long value;
 
-    public LongColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+    private final TimestampUnit[] timestampUnits;
+
+    public LongColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers, TimestampUnit[] timestampUnits) {
         super(pageBuilder, timestampParsers);
+        this.timestampUnits = timestampUnits;
     }
 
     @Override
@@ -52,7 +55,8 @@ public class LongColumnGetter extends BaseColumnGetter {
             pageBuilder.setNull(column);
         }
         else {
-            pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(value));
+            TimestampUnit unit = timestampUnits[column.getIndex()];
+            pageBuilder.setTimestamp(column, unit.toTimestamp(value));
         }
     }
 }

data/src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java

@@ -34,6 +34,7 @@ import static org.embulk.spi.type.Types.JSON;
 import static org.embulk.spi.type.Types.TIMESTAMP;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 public class TestAvroParserPlugin
 {
@@ -50,6 +51,7 @@ public class TestAvroParserPlugin
     {
         config = config().set("type", "avro");
         plugin = new AvroParserPlugin();
+
         recreatePageOutput();
     }
 
@@ -69,7 +71,7 @@ public class TestAvroParserPlugin
                 column("options", JSON),
                 column("spec", JSON),
                 column("created_at", TIMESTAMP, config().set("format", "%Y-%m-%dT%H:%M:%S%:z")),
-                column("created_at_utc", TIMESTAMP)
+                column("created_at_utc", TIMESTAMP, config().set("timestamp_unit", "second"))
         );
 
         ConfigSource config = this.config.deepCopy().set("columns", schema).set("avsc", this.getClass().getResource("item.avsc").getPath());
@@ -89,7 +91,44 @@ public class TestAvroParserPlugin
         assertEquals("bar", ((MapValue)record[8]).map().get(ValueFactory.newString("foo")).toString());
         assertEquals("opt1", ((MapValue)record[9]).map().get(ValueFactory.newString("key")).toString());
         assertEquals("2016-05-08 19:35:43 UTC", record[10].toString());
-        assertEquals("2016-05-08 19:35:25.952 UTC", record[11].toString());
+        assertEquals("2016-05-08 19:35:28 UTC", record[11].toString());
+    }
+
+    @Test
+    public void useTimestampUnit()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("timestamp", TIMESTAMP, config().set("timestamp_unit", "second")),
+                column("timestamp_long", TIMESTAMP, config().set("timestamp_unit", "second")),
+                column("timestamp_milli", TIMESTAMP, config().set("timestamp_unit", "milli")),
+                column("timestamp_micro", TIMESTAMP, config().set("timestamp_unit", "micro")),
+                column("timestamp_nano", TIMESTAMP, config().set("timestamp_unit", "nano")),
+                column("timestamp_float", TIMESTAMP, config().set("timestamp_unit", "second")),
+                column("timestamp_double", TIMESTAMP, config().set("timestamp_unit", "second")),
+                column("timestamp_double_milli", TIMESTAMP, config().set("timestamp_unit", "milli")),
+                column("timestamp_double_micro", TIMESTAMP, config().set("timestamp_unit", "micro")),
+                column("timestamp_double_nano", TIMESTAMP, config().set("timestamp_unit", "nano"))
+        );
+
+        ConfigSource config = this.config.deepCopy().set("columns", schema).set("avsc", this.getClass().getResource("item2.avsc").getPath());
+
+        transaction(config, fileInput(new File(this.getClass().getResource("items2.avro").getPath())));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(1, records.size());
+
+        Object[] record = records.get(0);
+        assertEquals("2018-02-23 12:13:52 UTC", record[0].toString());
+        assertEquals("2018-02-23 12:13:52 UTC", record[1].toString());
+        assertEquals("2018-02-23 12:13:52.717 UTC", record[2].toString());
+        assertEquals("2018-02-23 12:13:52.717249 UTC", record[3].toString());
+        assertEquals("2018-02-23 12:13:52.717249634 UTC", record[4].toString());
+        assertEquals("2018-02-23 12:13:52 UTC", record[5].toString());
+        assertTrue(record[6].toString().matches("2018-02-23 12:13:52.717249.* UTC"));
+        assertTrue(record[7].toString().matches("2018-02-23 12:13:52.717249.* UTC"));
+        assertTrue(record[8].toString().matches("2018-02-23 12:13:52.717249.* UTC"));
+        assertTrue(record[9].toString().matches("2018-02-23 12:13:52.717249.* UTC"));
     }
 
     @Test

data/src/test/resources/org/embulk/parser/avro/.gitignore

@@ -0,0 +1 @@
+.bundle

data/src/test/resources/org/embulk/parser/avro/Gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+gem "avro"

data/src/test/resources/org/embulk/parser/avro/Gemfile.lock

@@ -0,0 +1,15 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    avro (1.8.2)
+      multi_json
+    multi_json (1.13.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  avro
+
+BUNDLED WITH
+   1.16.1

data/src/test/resources/org/embulk/parser/avro/data.json

@@ -0,0 +1,6 @@
+{"id":1,"code":123456789012345678,"name":"Desktop","description":"Office and Personal Usage","flag":true,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":30000.0,"spec":{"key":"opt1","value":"optvalue1"},"tags":["tag1","tag2"],"options":{"hoge":null,"foo":"bar"},"item_type":"D","dummy":null}
+{"id":2,"code":123456789012345679,"name":"Laptop","description":null,"flag":false,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":50000.0,"spec":{"key":"opt1","value":null},"tags":null,"options":{},"item_type":"M","dummy":null}
+{"id":3,"code":123456789012345680,"name":"Tablet","description":"Personal Usage","flag":true,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":null,"spec":{"key":"opt1","value":"optvalue1"},"tags":["tag3"],"options":{},"item_type":"M","dummy":null}
+{"id":4,"code":123456789012345681,"name":"Mobile","description":"Personal Usage","flag":true,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":10000.0,"spec":{"key":"opt1","value":"optvalue1"},"tags":[],"options":{},"item_type":"M","dummy":null}
+{"id":5,"code":123456789012345682,"name":"Notepad","description":null,"flag":true,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":20000.0,"spec":{"key":"opt1","value":"optvalue1"},"tags":["tag1","tag2"],"options":{},"item_type":"M","dummy":null}
+{"id":6,"code":123456789012345683,"name":"SmartPhone","description":"Multipurpose","flag":true,"created_at":"2016-05-09T04:35:43+09:00","created_at_utc":1.46273613E9,"price":40000.0,"spec":{"key":"opt1","value":"optvalue1"},"tags":["tag1","tag2"],"options":{},"item_type":"M","dummy":null}

data/src/test/resources/org/embulk/parser/avro/data2.json

@@ -0,0 +1 @@
+{"timestamp": 1519388032, "timestamp_long": 1519388032, "timestamp_milli": 1519388032717, "timestamp_micro": 1519388032717249, "timestamp_nano": 1519388032717249634, "timestamp_float": 1519388032.7172496, "timestamp_float_milli": 1519388032717.2496, "timestamp_float_micro": 1519388032717249.6, "timestamp_float_nano": 1519388032717249634.0,"timestamp_double": 1519388032.717249634, "timestamp_double_milli": 1519388032717.249634, "timestamp_double_micro": 1519388032717249.634, "timestamp_double_nano": 1519388032717249634.0}

data/src/test/resources/org/embulk/parser/avro/data_creator.rb

@@ -0,0 +1,17 @@
+require 'avro'
+require 'json'
+
+schema = Avro::Schema.parse(File.read(ARGV[0]))
+file = File.open(ARGV[1], 'wb')
+writer = Avro::IO::DatumWriter.new(schema)
+dw = Avro::DataFile::Writer.new(file, writer, schema)
+
+data = File.read(ARGV[2]).each_line.map do |l|
+  JSON.load(l)
+end
+
+data.each do |d|
+  dw << d
+end
+
+dw.close

data/src/test/resources/org/embulk/parser/avro/item2.avsc

@@ -0,0 +1,20 @@
+{
+  "type" : "record",
+  "name" : "Item",
+  "namespace" : "example.avro",
+  "fields" : [
+    {"name": "timestamp", "type": "int"},
+    {"name": "timestamp_long", "type": "long"},
+    {"name": "timestamp_milli", "type": "long"},
+    {"name": "timestamp_micro", "type": "long"},
+    {"name": "timestamp_nano", "type": "long"},
+    {"name": "timestamp_float", "type": "float"},
+    {"name": "timestamp_float_milli", "type": "float"},
+    {"name": "timestamp_float_micro", "type": "float"},
+    {"name": "timestamp_float_nano", "type": "float"},
+    {"name": "timestamp_double", "type": "double"},
+    {"name": "timestamp_double_milli", "type": "double"},
+    {"name": "timestamp_double_micro", "type": "double"},
+    {"name": "timestamp_double_nano", "type": "double"}
+  ]
+}

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-avro
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-04 00:00:00.000000000 Z
+date: 2018-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Parses Avro files read by other file input plugins.
 email:
 - kakyoin.hierophant@gmail.com
@@ -64,6 +64,8 @@ files:
 - lib/embulk/guess/avro.rb
 - lib/embulk/parser/avro.rb
 - src/main/java/org/embulk/parser/avro/AvroParserPlugin.java
+- src/main/java/org/embulk/parser/avro/TimestampUnit.java
+- src/main/java/org/embulk/parser/avro/TimestampUnitDeserializer.java
 - src/main/java/org/embulk/parser/avro/getter/AvroGenericDataConverter.java
 - src/main/java/org/embulk/parser/avro/getter/BaseColumnGetter.java
 - src/main/java/org/embulk/parser/avro/getter/BooleanColumnGetter.java
@@ -75,16 +77,24 @@ files:
 - src/main/java/org/embulk/parser/avro/getter/LongColumnGetter.java
 - src/main/java/org/embulk/parser/avro/getter/StringColumnGetter.java
 - src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java
+- src/test/resources/org/embulk/parser/avro/.gitignore
+- src/test/resources/org/embulk/parser/avro/Gemfile
+- src/test/resources/org/embulk/parser/avro/Gemfile.lock
+- src/test/resources/org/embulk/parser/avro/data.json
+- src/test/resources/org/embulk/parser/avro/data2.json
+- src/test/resources/org/embulk/parser/avro/data_creator.rb
 - src/test/resources/org/embulk/parser/avro/item.avsc
+- src/test/resources/org/embulk/parser/avro/item2.avsc
 - src/test/resources/org/embulk/parser/avro/items.avro
-- classpath/avro-1.8.0.jar
+- src/test/resources/org/embulk/parser/avro/items2.avro
+- classpath/avro-1.8.2.jar
+- classpath/paranamer-2.7.jar
+- classpath/xz-1.5.jar
 - classpath/commons-compress-1.8.1.jar
-- classpath/embulk-parser-avro-0.2.0.jar
+- classpath/embulk-parser-avro-0.3.0.jar
 - classpath/jackson-core-asl-1.9.13.jar
-- classpath/jackson-mapper-asl-1.9.13.jar
-- classpath/paranamer-2.7.jar
 - classpath/snappy-java-1.1.1.3.jar
-- classpath/xz-1.5.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
 homepage: https://github.com/joker1007/embulk-parser-avro
 licenses:
 - MIT