embulk-parser-avro 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 42ab545f7eca444d73944d0e1b86076582e109a3
-  data.tar.gz: 297e047f1320b1e2d8c1e22b6ab24a2feed51a08
+  metadata.gz: 62671c1f3feefa2bb7feecccf4362b04f0ad7d3c
+  data.tar.gz: 68e76d9d06bcc2f30c3d6e376e1fe867027b6e09
 SHA512:
-  metadata.gz: 73e60189bdb763f31f16330cd6c1cee0e3de1a27df254206d16b2276489263442ea87349a2e240c489a47d6d0d505c18b721e28d305064439ade530dc82e8b48
-  data.tar.gz: 45b959df1da857a91ed391a8cdfbcc7200d1e55c8f6d3a821ed80a642c153a171babd4adc63cc345fbf64b5a172868b9ee92ff5e5f3d5856ad086c8c1b30c220
+  metadata.gz: 79e628c578df06d5aa54e17a0038eeb65fa667318be469cdceab9161a4b40716634d1c9cbdd88d17ef47a8d85cf9d4d63939c6340b3efde74995070e3ad37758
+  data.tar.gz: d1120457b865feed3eab7560f3dff20ab0b767fe17653d6970626836460426135334f0f681f901405ce8107d4a0f3c3fb5aa14e302023132fbad48a22c538168

data/README.md

@@ -11,10 +11,12 @@
 
 - **type**: Specify this parser as avro
 - **avsc**: Specify avro schema file.
-- **columns**: Specify column name and type. See below (array, required)
+- **columns**: Specify column name and type. See below (array, optional)
 * **default_timezone**: Default timezone of the timestamp (string, default: UTC)
 * **default_timestamp_format**: Default timestamp format of the timestamp (string, default: `%Y-%m-%d %H:%M:%S.%N %z`)
 
+If columns is not set, this plugin detect schema automatically by using avsc schema.
+
 ## Example
 
 ```yaml

data/build.gradle

@@ -13,19 +13,19 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.8"
-    provided "org.embulk:embulk-core:0.8.8"
+    compile  "org.embulk:embulk-core:0.8.14"
+    provided "org.embulk:embulk-core:0.8.14"
     compile "org.apache.avro:avro:1.8.0"
     testCompile "junit:junit:4.+"
 
-    testCompile  "org.embulk:embulk-core:0.8.8:tests"
-    testCompile  "org.embulk:embulk-standards:0.8.8"
+    testCompile  "org.embulk:embulk-core:0.8.14:tests"
+    testCompile  "org.embulk:embulk-standards:0.8.14"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/example/example_nocolumns.yml

@@ -0,0 +1,9 @@
+in:
+  type: file
+  path_prefix: "items"
+  parser:
+    type: avro
+    avsc : "./item.avsc"
+
+out:
+  type: stdout

data/src/main/java/org/embulk/parser/avro/AvroParserPlugin.java

@@ -1,11 +1,14 @@
 package org.embulk.parser.avro;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.apache.avro.file.DataFileStream;
 import org.apache.avro.generic.GenericDatumReader;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.io.DatumReader;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
@@ -20,6 +23,7 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Types;
 import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.util.FileInputInputStream;
 import org.embulk.spi.util.Timestamps;
@@ -35,6 +39,7 @@ public class AvroParserPlugin
             extends Task, TimestampParser.Task
     {
         @Config("columns")
+        @ConfigDefault("[]")
         public SchemaConfig getColumns();
 
         @Config("avsc")
@@ -46,22 +51,92 @@ public class AvroParserPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        Schema schema = task.getColumns().toSchema();
+        File avsc = task.getAvsc().getFile();
+        org.apache.avro.Schema avroSchema;
+        try {
+            avroSchema = new org.apache.avro.Schema.Parser().parse(avsc);
+        } catch (IOException e) {
+            throw new ConfigException("avsc file is not found");
+        }
+
+        Schema schema = buildSchema(task.getColumns(), avroSchema);
 
         control.run(task.dump(), schema);
     }
 
+    Schema buildSchema(SchemaConfig columns, org.apache.avro.Schema avroSchema) {
+        if (columns.size() > 0) {
+            return columns.toSchema();
+        } else {
+            int index = 0;
+            ImmutableList.Builder<Column> builder = ImmutableList.builder();
+            for (org.apache.avro.Schema.Field field : avroSchema.getFields()) {
+                String name = field.name();
+
+                org.apache.avro.Schema.Type avroType = null;
+                if (field.schema().getType() == org.apache.avro.Schema.Type.UNION) {
+                    for (org.apache.avro.Schema sc : field.schema().getTypes()) {
+                        if (sc.getType() != org.apache.avro.Schema.Type.NULL) {
+                            avroType = sc.getType();
+                            break;
+                        }
+                    }
+                } else {
+                    avroType = field.schema().getType();
+                }
+                switch (avroType) {
+                    case STRING:
+                    case BYTES:
+                    case FIXED:
+                    case ENUM:
+                    case NULL:
+                        builder.add(new Column(index, name, Types.STRING));
+                        index++;
+                        break;
+                    case INT:
+                    case LONG:
+                        builder.add(new Column(index, name, Types.LONG));
+                        index++;
+                        break;
+                    case FLOAT:
+                    case DOUBLE:
+                        builder.add(new Column(index, name, Types.DOUBLE));
+                        index++;
+                        break;
+                    case BOOLEAN:
+                        builder.add(new Column(index, name, Types.BOOLEAN));
+                        index++;
+                        break;
+                    case MAP:
+                    case ARRAY:
+                    case RECORD:
+                        builder.add(new Column(index, name, Types.JSON));
+                        index++;
+                        break;
+                    default:
+                        throw new RuntimeException("Unsupported type");
+                }
+            }
+            return new Schema(builder.build());
+        }
+    }
+
     @Override
     public void run(TaskSource taskSource, Schema schema,
             FileInput input, PageOutput output)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        File avsc = task.getAvsc().getFile();
         List<Column> columns = schema.getColumns();
         final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getColumns());
+        File avsc = task.getAvsc().getFile();
+        final org.apache.avro.Schema avroSchema;
+        try {
+            avroSchema = new org.apache.avro.Schema.Parser().parse(avsc);
+        } catch (IOException e) {
+            throw new ConfigException("avsc file is not found");
+        }
 
         try (FileInputInputStream is = new FileInputInputStream(input); final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-            org.apache.avro.Schema avroSchema = new org.apache.avro.Schema.Parser().parse(avsc);
             ColumnGetterFactory factory = new ColumnGetterFactory(avroSchema, pageBuilder, timestampParsers);
             ImmutableMap.Builder<String, BaseColumnGetter> columnGettersBuilder = ImmutableMap.builder();
             for (Column column : columns) {

data/src/main/java/org/embulk/parser/avro/getter/ColumnGetterFactory.java

@@ -1,5 +1,6 @@
 package org.embulk.parser.avro.getter;
 
+import org.apache.avro.Schema;
 import org.embulk.spi.Column;
 import org.embulk.spi.DataException;
 import org.embulk.spi.PageBuilder;
@@ -20,44 +21,46 @@ public class ColumnGetterFactory {
     public BaseColumnGetter newColumnGetter(Column column)
     {
         org.apache.avro.Schema fieldSchema = avroSchema.getField(column.getName()).schema();
-        switch (fieldSchema.getType().getName()) {
-            case "union" :
-                String typeName = "";
-                for (org.apache.avro.Schema type : fieldSchema.getTypes()) {
-                    if (!type.getName().equals("null")) {
-                        typeName = type.getName();
+        switch (fieldSchema.getType()) {
+            case UNION:
+                Schema.Type type = null;
+                for (org.apache.avro.Schema sc : fieldSchema.getTypes()) {
+                    if (sc.getType() != Schema.Type.NULL) {
+                        type = sc.getType();
                         break;
                     }
                 }
-                return getColumnGetterFromTypeName(typeName);
+                return getColumnGetterFromTypeName(type);
             default :
-                return getColumnGetterFromTypeName(fieldSchema.getType().getName());
+                return getColumnGetterFromTypeName(fieldSchema.getType());
         }
     }
 
-    private BaseColumnGetter getColumnGetterFromTypeName(String typeName)
+    private BaseColumnGetter getColumnGetterFromTypeName(Schema.Type type)
     {
-        switch (typeName) {
-            case "string":
-            case "enum":
+        switch (type) {
+            case STRING:
+            case ENUM:
                 return new StringColumnGetter(pageBuilder, timestampParsers);
-            case "int":
+            case INT:
                 return new IntegerColumnGetter(pageBuilder, timestampParsers);
-            case "long":
+            case LONG:
                 return new LongColumnGetter(pageBuilder, timestampParsers);
-            case "float":
+            case FLOAT:
                 return new FloatColumnGetter(pageBuilder, timestampParsers);
-            case "double":
+            case DOUBLE:
                 return new DoubleColumnGetter(pageBuilder, timestampParsers);
-            case "boolean":
+            case BOOLEAN:
                 return new BooleanColumnGetter(pageBuilder, timestampParsers);
-            case "array":
-            case "map":
-            case "record":
+            case ARRAY:
+            case MAP:
+            case RECORD:
                 return new GenericDataColumnGetter(pageBuilder, timestampParsers);
-            case "byte":
+            case NULL:
+                return new StringColumnGetter(pageBuilder, timestampParsers);
+            case BYTES:
             default:
-                throw new DataException(String.format("%s is not supported", typeName));
+                throw new DataException(String.format("%s is not supported", type.getName()));
         }
     }
 }

data/src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java

@@ -92,6 +92,46 @@ public class TestAvroParserPlugin
         assertEquals("2016-05-08 19:35:25.952 UTC", record[11].toString());
     }
 
+    @Test
+    public void useNoColumnsOption()
+            throws Exception
+    {
+        SchemaConfig schema = schema(
+                column("id", LONG),
+                column("code", LONG),
+                column("name", STRING),
+                column("description", STRING),
+                column("flag", BOOLEAN),
+                column("created_at", STRING),
+                column("created_at_utc", DOUBLE),
+                column("price", DOUBLE),
+                column("spec", JSON),
+                column("tags", JSON),
+                column("options", JSON),
+                column("item_type", STRING),
+                column("dummy", STRING)
+        );
+
+        ConfigSource config = this.config.deepCopy().set("avsc", this.getClass().getResource("item.avsc").getPath());
+
+        transaction(config, fileInput(new File(this.getClass().getResource("items.avro").getPath())));
+
+        List<Object[]> records = Pages.toObjects(schema.toSchema(), output.pages);
+        assertEquals(6, records.size());
+
+        Object[] record = records.get(0);
+        assertEquals(1L, record[0]);
+        assertEquals(123456789012345678L, record[1]);
+        assertEquals("Desktop", record[2]);
+        assertEquals(true, record[4]);
+        assertEquals("D", record[11]);
+        assertEquals("[\"tag1\",\"tag2\"]", record[9].toString());
+        assertEquals("bar", ((MapValue)record[10]).map().get(ValueFactory.newString("foo")).toString());
+        assertEquals("opt1", ((MapValue)record[8]).map().get(ValueFactory.newString("key")).toString());
+        assertEquals("2016-05-09T04:35:43+09:00", record[5].toString());
+        assertNull(record[12]);
+    }
+
     private void recreatePageOutput()
     {
         output = new MockPageOutput();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-avro
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-08 00:00:00.000000000 Z
+date: 2016-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,6 +53,7 @@ files:
 - config/checkstyle/default.xml
 - example/.gitignore
 - example/example.yml
+- example/example_nocolumns.yml
 - example/generate.rb
 - example/item.avsc
 - example/items.avro
@@ -62,7 +63,6 @@ files:
 - gradlew.bat
 - lib/embulk/guess/avro.rb
 - lib/embulk/parser/avro.rb
-- src/main/java/org/embulk/parser/avro/AvroColumnOption.java
 - src/main/java/org/embulk/parser/avro/AvroParserPlugin.java
 - src/main/java/org/embulk/parser/avro/getter/AvroGenericDataConverter.java
 - src/main/java/org/embulk/parser/avro/getter/BaseColumnGetter.java
@@ -79,7 +79,7 @@ files:
 - src/test/resources/org/embulk/parser/avro/items.avro
 - classpath/avro-1.8.0.jar
 - classpath/commons-compress-1.8.1.jar
-- classpath/embulk-parser-avro-0.1.0.jar
+- classpath/embulk-parser-avro-0.2.0.jar
 - classpath/jackson-core-asl-1.9.13.jar
 - classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/paranamer-2.7.jar

data/src/main/java/org/embulk/parser/avro/AvroColumnOption.java

@@ -1,16 +0,0 @@
-package org.embulk.parser.avro;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.spi.type.Type;
-
-import com.google.common.base.Optional;
-
-public interface AvroColumnOption
-        extends Task
-{
-    @Config("type")
-    @ConfigDefault("null")
-    Optional<Type> getType();
-}