embulk-parser-avro 0.1.0

data/src/main/java/org/embulk/parser/avro/getter/GenericDataColumnGetter.java

@@ -0,0 +1,34 @@
+package org.embulk.parser.avro.getter;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.value.Value;
+
+public class GenericDataColumnGetter extends BaseColumnGetter {
+    public GenericDataColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+        super(pageBuilder, timestampParsers);
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            Value converted = AvroGenericDataConverter.convert(value);
+            pageBuilder.setString(column, converted.toString());
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            Value converted = AvroGenericDataConverter.convert(value);
+            pageBuilder.setJson(column, converted);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/avro/getter/IntegerColumnGetter.java

@@ -0,0 +1,57 @@
+package org.embulk.parser.avro.getter;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+
+public class IntegerColumnGetter extends BaseColumnGetter {
+    protected Integer value;
+
+    public IntegerColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+        super(pageBuilder, timestampParsers);
+    }
+
+    @Override
+    public void setValue(Object value)
+    {
+        this.value = (Integer) value;
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setLong(column, value.longValue());
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setDouble(column, value.doubleValue());
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setString(column, value.toString());
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(value.longValue()));
+        }
+    }
+}

data/src/main/java/org/embulk/parser/avro/getter/LongColumnGetter.java

@@ -0,0 +1,58 @@
+package org.embulk.parser.avro.getter;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+
+public class LongColumnGetter extends BaseColumnGetter {
+    protected Long value;
+
+    public LongColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+        super(pageBuilder, timestampParsers);
+    }
+
+    @Override
+    public void setValue(Object value)
+    {
+        this.value = (Long) value;
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setLong(column, value);
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            pageBuilder.setDouble(column, value.doubleValue());
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        if (value == null) {
+            pageBuilder.setNull(column);
+        } else {
+            Long casted = (Long) value;
+            pageBuilder.setString(column, value.toString());
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setTimestamp(column, Timestamp.ofEpochSecond(value));
+        }
+    }
+}

data/src/main/java/org/embulk/parser/avro/getter/StringColumnGetter.java

@@ -0,0 +1,85 @@
+package org.embulk.parser.avro.getter;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.TimestampParser;
+
+public class StringColumnGetter extends BaseColumnGetter {
+    protected String value;
+    private final JsonParser jsonParser = new JsonParser();
+
+    public StringColumnGetter(PageBuilder pageBuilder, TimestampParser[] timestampParsers) {
+        super(pageBuilder, timestampParsers);
+    }
+
+    @Override
+    public void setValue(Object value)
+    {
+        if (value == null)
+            this.value = null;
+        else
+            this.value = value.toString();
+    }
+
+    @Override
+    public void booleanColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setBoolean(column, Boolean.parseBoolean(value));
+        }
+    }
+
+    @Override
+    public void longColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setLong(column, Long.parseLong(value));
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setDouble(column, Double.parseDouble(value));
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setString(column, value);
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            TimestampParser parser = timestampParsers[column.getIndex()];
+            pageBuilder.setTimestamp(column, parser.parse(value));
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column) {
+        if (this.value == null) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            pageBuilder.setJson(column, jsonParser.parse(value));
+        }
+    }
+}

data/src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java

@@ -0,0 +1,145 @@
+package org.embulk.parser.avro;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.Pages;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.ValueFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import static org.embulk.spi.type.Types.BOOLEAN;
+import static org.embulk.spi.type.Types.DOUBLE;
+import static org.embulk.spi.type.Types.LONG;
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.spi.type.Types.JSON;
+import static org.embulk.spi.type.Types.TIMESTAMP;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class TestAvroParserPlugin
+{
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private AvroParserPlugin plugin;
+    private MockPageOutput output;
+
+    @Before
+    public void createResource()
+    {
+        config = config().set("type", "avro");
+        plugin = new AvroParserPlugin();
+        recreatePageOutput();
+    }
+
+    @Test
+    public void useNormal()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("id", LONG),
+                column("code", STRING),
+                column("name", STRING),
+                column("description", STRING),
+                column("flag", BOOLEAN),
+                column("price", DOUBLE),
+                column("item_type", STRING),
+                column("tags", JSON),
+                column("options", JSON),
+                column("spec", JSON),
+                column("created_at", TIMESTAMP, config().set("format", "%Y-%m-%dT%H:%M:%S%:z")),
+                column("created_at_utc", TIMESTAMP)
+        );
+
+        ConfigSource config = this.config.deepCopy().set("columns", schema).set("avsc", this.getClass().getResource("item.avsc").getPath());
+
+        transaction(config, fileInput(new File(this.getClass().getResource("items.avro").getPath())));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(6, records.size());
+
+        Object[] record = records.get(0);
+        assertEquals(1L, record[0]);
+        assertEquals("123456789012345678", record[1]);
+        assertEquals("Desktop", record[2]);
+        assertEquals(true, record[4]);
+        assertEquals("D", record[6]);
+        assertEquals("[\"tag1\",\"tag2\"]", record[7].toString());
+        assertEquals("bar", ((MapValue)record[8]).map().get(ValueFactory.newString("foo")).toString());
+        assertEquals("opt1", ((MapValue)record[9]).map().get(ValueFactory.newString("key")).toString());
+        assertEquals("2016-05-08 19:35:43 UTC", record[10].toString());
+        assertEquals("2016-05-08 19:35:25.952 UTC", record[11].toString());
+    }
+
+    private void recreatePageOutput()
+    {
+        output = new MockPageOutput();
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private void transaction(ConfigSource config, final FileInput input)
+    {
+        plugin.transaction(config, new ParserPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput fileInput(File file)
+            throws Exception
+    {
+        FileInputStream in = new FileInputStream(file);
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+            throws IOException
+    {
+        return new InputStreamFileInput.IteratorProvider(
+                ImmutableList.copyOf(inputStreams));
+    }
+
+    private SchemaConfig schema(ColumnConfig... columns)
+    {
+        return new SchemaConfig(Lists.newArrayList(columns));
+    }
+
+    private ColumnConfig column(String name, Type type)
+    {
+        return column(name, type, config());
+    }
+
+    private ColumnConfig column(String name, Type type, ConfigSource option)
+    {
+        return new ColumnConfig(name, type, option);
+    }
+}

data/src/test/resources/org/embulk/parser/avro/item.avsc

@@ -0,0 +1,27 @@
+{
+  "type" : "record",
+  "name" : "Item",
+  "namespace" : "example.avro",
+  "fields" : [
+    {"name": "id", "type": "int"},
+    {"name": "code", "type": "long"},
+    {"name": "name", "type": "string"},
+    {"name": "description", "type": ["string", "null"]},
+    {"name": "flag", "type": "boolean"},
+    {"name": "created_at", "type": "string"},
+    {"name": "created_at_utc", "type": "float"},
+    {"name": "price", "type": ["double", "null"]},
+    {"name": "spec", "type": {
+      "type": "record",
+      "name": "item_spec",
+      "fields" : [
+        {"name" : "key", "type" : "string"},
+        {"name" : "value", "type" : ["string", "null"]}
+      ]}
+    },
+    {"name": "tags", "type": [{"type": "array", "items": "string"}, "null"]},
+    {"name": "options", "type": {"type": "map", "values": ["string", "null"]}},
+    {"name": "item_type", "type": {"name": "item_type_enum", "type": "enum", "symbols": ["D", "M"]}},
+    {"name": "dummy", "type": "null"}
+  ]
+}

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-avro
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- joker1007
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  prerelease: false
+  type: :development
+description: Parses Avro files read by other file input plugins.
+email:
+- kakyoin.hierophant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- example/.gitignore
+- example/example.yml
+- example/generate.rb
+- example/item.avsc
+- example/items.avro
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/guess/avro.rb
+- lib/embulk/parser/avro.rb
+- src/main/java/org/embulk/parser/avro/AvroColumnOption.java
+- src/main/java/org/embulk/parser/avro/AvroParserPlugin.java
+- src/main/java/org/embulk/parser/avro/getter/AvroGenericDataConverter.java
+- src/main/java/org/embulk/parser/avro/getter/BaseColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/BooleanColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/ColumnGetterFactory.java
+- src/main/java/org/embulk/parser/avro/getter/DoubleColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/FloatColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/GenericDataColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/IntegerColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/LongColumnGetter.java
+- src/main/java/org/embulk/parser/avro/getter/StringColumnGetter.java
+- src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java
+- src/test/resources/org/embulk/parser/avro/item.avsc
+- src/test/resources/org/embulk/parser/avro/items.avro
+- classpath/avro-1.8.0.jar
+- classpath/commons-compress-1.8.1.jar
+- classpath/embulk-parser-avro-0.1.0.jar
+- classpath/jackson-core-asl-1.9.13.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/paranamer-2.7.jar
+- classpath/snappy-java-1.1.1.3.jar
+- classpath/xz-1.5.jar
+homepage: https://github.com/joker1007/embulk-parser-avro
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Avro parser plugin for Embulk
+test_files: []