embulk-formatter-avro 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fe7f1bec5a58a73ef465e0dbe1d905a11af5ff0
-  data.tar.gz: 40f003f4911ab7c635efb83b8a67820baad5ffa4
+  metadata.gz: 297627941b7e4e884febe21ab19701ee8d136d95
+  data.tar.gz: 9d6745e4e739064db4fe91e78c0cc39fe3948488
 SHA512:
-  metadata.gz: 96b5a25afb51f07f9e629e00fd7536140819eaeb2d70d7cb548decb9139de5919b33ca9ee9f72adfe9924a13e1c1822fa61dcabb286603ab596fd5ca40426b17
-  data.tar.gz: c105fe45e42fffa3a6c45a74b12b4f5a242018fb2d262b0b623ee8e4356aec5f7aaece17b5e0e785f5295d0e324698e24c7011376e718ec8b2b765eae5ff2757
+  metadata.gz: dd3fa9f5211bb629007fe942a68bfa99a66c31167864ef241258611c125bd77d232bbdcd97960369d3e04b56467c9dc9e11bf8cb8f955877fd9ea244f5242ffb
+  data.tar.gz: c8884bc84f00bf6720728db03033370ba5285d64431a12424f034a104488eb78aa4f9463e6c8e762d698d04992de40e13a1585c0da01479b10a6e3b4c85df1c2

data/README.md

@@ -28,6 +28,8 @@ See. [AvroValueConverters](https://github.com/joker1007/embulk-formatter-avro/tr
 ## Configuration
 
 - **avsc**: avro schema (avsc) filepath (string, required)
+- **codec**: avro codec type (enum: `deflate`, `bzip2`, `xz`, `snappy`, optional)
+- **compression\_level**: avro codec compression level (integer, optional, for only `deflate` and `xz` codec)
 - **skip\_error\_record**: If you want to skip error record, set true (boolean, default: `false`)
 
 ## Example

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/example/config.yml

@@ -23,6 +23,7 @@ in:
     - {name: data2, type: json}
     - {name: type, type: string}
     - {name: md5, type: string}
+    - {name: flag, type: boolean}
 
 out:
   type: file

data/example/sample_01.csv

@@ -1,7 +1,7 @@
-id,account,time,purchase,comment,data,data2,type,md5
-1,32864,2015-01-27 19:23:49,20150127,embulk,"{\"foo\": \"bar\", \"events\": [{\"id\": 1, \"name\": \"Name1\"}, {\"id\": 2, \"name\": \"Name2\"}]}","[{\"hoge\": 1}]","A",6c3983adda53b2136532dbd5b4d668b9
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby,"{\"events\": [{\"id\": 1, \"name\": \"Name1\"}, {\"id\": 2, \"name\": \"Name2\"}]}","[]","B",NULL
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",NULL,"[]","C",8c3983ad4jsal42010325kld024954ll
-4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,"[]","A",474a3bdb966b40c80acb069f73441f29
-4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,"[]","D",NULL
-4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,NULL,"A",NULL
+id,account,time,purchase,comment,data,data2,type,md5,flag
+1,32864,2015-01-27 19:23:49,20150127,embulk,"{\"foo\": \"bar\", \"events\": [{\"id\": 1, \"name\": \"Name1\"}, {\"id\": 2, \"name\": \"Name2\"}]}","[{\"hoge\": 1}]","A",6c3983adda53b2136532dbd5b4d668b9,true
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby,"{\"events\": [{\"id\": 1, \"name\": \"Name1\"}, {\"id\": 2, \"name\": \"Name2\"}]}","[]","B",NULL,true
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",NULL,"[]","C",8c3983ad4jsal42010325kld024954ll,true
+4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,"[]","A",474a3bdb966b40c80acb069f73441f29,true
+4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,"[]","D",NULL,false
+4,11270,2015-01-29 11:54:36,20150129,NULL,NULL,NULL,"A",NULL,false

data/example/schema.avsc

@@ -28,6 +28,7 @@
       }
     }},
     {"name": "type", "type": {"type": "enum", "name": "type_enum", "symbols": ["A", "B", "C"]}},
-    {"name": "md5", "type": ["null", {"type": "fixed", "name": "md5_value", "size": 32}]}
+    {"name": "md5", "type": ["null", {"type": "fixed", "name": "md5_value", "size": 32}]},
+    {"name": "flag", "type": "boolean"}
   ]
 }

data/src/main/java/org/embulk/formatter/avro/AvroFormatterPlugin.java

@@ -1,5 +1,9 @@
 package org.embulk.formatter.avro;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Optional;
+import org.apache.avro.file.CodecFactory;
 import org.apache.avro.file.DataFileWriter;
 import org.apache.avro.generic.GenericData;
 import org.apache.avro.generic.GenericDatumWriter;
@@ -30,6 +34,7 @@ import org.slf4j.Logger;
 
 import java.io.File;
 import java.io.IOException;
+import java.util.Locale;
 import java.util.Map;
 
 public class AvroFormatterPlugin
@@ -41,6 +46,14 @@ public class AvroFormatterPlugin
         @Config("avsc")
         LocalFile getAvsc();
 
+        @Config("codec")
+        @ConfigDefault("null")
+        Optional<Codec> getCodec();
+
+        @Config("compression_level")
+        @ConfigDefault("null")
+        Optional<Integer> getCompressionLevel();
+
         @Config("column_options")
         @ConfigDefault("{}")
         Map<String, TimestampFormatter.TimestampColumnOption> getColumnOptions();
@@ -95,6 +108,7 @@ public class AvroFormatterPlugin
             avroSchema = new org.apache.avro.Schema.Parser().parse(avsc);
             GenericDatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(avroSchema);
             writer = new DataFileWriter<>(datumWriter);
+            writer.setCodec(task.getCodec().or(Codec.NULL).getCodecFactory(task.getCompressionLevel()));
             stream.nextFile();
             writer.create(avroSchema, stream);
         } catch (IOException e) {
@@ -166,4 +180,56 @@ public class AvroFormatterPlugin
             }
         };
     }
+
+    public enum Codec {
+        NULL {
+            public CodecFactory getCodecFactory(Optional<Integer> compressionLevel) {
+                return CodecFactory.nullCodec();
+            }
+        },
+        DEFLATE {
+            public CodecFactory getCodecFactory(Optional<Integer> compressionLevel) {
+                return CodecFactory.deflateCodec(compressionLevel.or(CodecFactory.DEFAULT_DEFLATE_LEVEL));
+            }
+        },
+        XZ {
+            public CodecFactory getCodecFactory(Optional<Integer> compressionLevel) {
+                return CodecFactory.xzCodec(compressionLevel.or(CodecFactory.DEFAULT_XZ_LEVEL));
+            }
+        },
+        SNAPPY {
+            public CodecFactory getCodecFactory(Optional<Integer> compressionLevel) {
+                return CodecFactory.snappyCodec();
+            }
+        },
+        BZIP2 {
+            public CodecFactory getCodecFactory(Optional<Integer> compressionLevel) {
+                return CodecFactory.bzip2Codec();
+            }
+        };
+
+        @JsonValue
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        abstract public CodecFactory getCodecFactory(Optional<Integer> compressionLevel);
+
+        @JsonCreator
+        public static Codec fromString(String name) {
+            switch (name) {
+                case "deflate":
+                    return DEFLATE;
+                case "xz":
+                    return XZ;
+                case "snappy":
+                    return SNAPPY;
+                case "bzip2":
+                    return BZIP2;
+                default:
+                    throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are single_column, multi_column", name));
+            }
+        }
+    }
 }

data/src/main/java/org/embulk/formatter/avro/converter/AvroBooleanConverter.java

@@ -1,6 +1,7 @@
 package org.embulk.formatter.avro.converter;
 
 import org.apache.avro.Schema;
+import org.msgpack.value.Value;
 
 public class AvroBooleanConverter extends AbstractAvroValueConverter {
     public AvroBooleanConverter(Schema schema) {
@@ -8,7 +9,22 @@ public class AvroBooleanConverter extends AbstractAvroValueConverter {
     }
 
     @Override
-    public Object booleanColumn(boolean value) {
-        throw new RuntimeException("Unsupported typecasting");
+    public Boolean booleanColumn(boolean value) {
+        return value;
+    }
+
+    @Override
+    public Boolean longColumn(long value) {
+        return value != 0;
+    }
+
+    @Override
+    public Boolean doubleColumn(double value) {
+        return value != 0;
+    }
+
+    @Override
+    public Boolean stringColumn(String value) {
+        return Boolean.valueOf(value);
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-formatter-avro
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-04 00:00:00.000000000 Z
+date: 2017-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -76,14 +76,14 @@ files:
 - src/main/java/org/embulk/formatter/avro/converter/AvroStringConverter.java
 - src/main/java/org/embulk/formatter/avro/converter/AvroValueConverterFactory.java
 - src/test/java/org/embulk/formatter/avro/TestAvroFormatterPlugin.java
-- classpath/xz-1.5.jar
-- classpath/jackson-mapper-asl-1.9.13.jar
 - classpath/paranamer-2.7.jar
-- classpath/avro-1.8.1.jar
-- classpath/commons-compress-1.8.1.jar
+- classpath/xz-1.5.jar
 - classpath/snappy-java-1.1.1.3.jar
-- classpath/embulk-formatter-avro-0.1.1.jar
 - classpath/jackson-core-asl-1.9.13.jar
+- classpath/embulk-formatter-avro-0.2.1.jar
+- classpath/avro-1.8.1.jar
+- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/commons-compress-1.8.1.jar
 homepage: https://github.com/joker1007/embulk-formatter-avro
 licenses:
 - MIT