embulk-formatter-avro 0.1.0

data/src/main/java/org/embulk/formatter/avro/converter/AbstractAvroValueConverter.java

@@ -0,0 +1,36 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.msgpack.value.Value;
+
+abstract public class AbstractAvroValueConverter {
+    protected Schema avroSchema;
+
+    public AbstractAvroValueConverter(Schema schema) {
+        this.avroSchema = schema;
+    }
+
+    public Object booleanColumn(boolean value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+
+    public Object longColumn(long value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+
+    public Object doubleColumn(double value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+
+    public Object stringColumn(String value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+
+    public Object timestampColumn(String value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+
+    public Object jsonColumn(Value value) {
+        throw new RuntimeException(String.format("%s cannot convert to %s of %s field", value, avroSchema.getType(), avroSchema.getName()));
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroArrayConverter.java

@@ -0,0 +1,52 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericArray;
+import org.apache.avro.generic.GenericData;
+import org.msgpack.value.Value;
+
+import java.util.List;
+
+public class AvroArrayConverter extends AbstractAvroValueConverter {
+    private AbstractAvroValueConverter elementConverter;
+
+    public AvroArrayConverter(Schema schema, AbstractAvroValueConverter elementConverter) {
+        super(schema);
+        this.elementConverter = elementConverter;
+    }
+
+    @Override
+    public GenericArray<Object> jsonColumn(Value value) {
+        if (!value.isArrayValue())
+            throw new RuntimeException("Support only array type json record");
+
+        List<Value> list = value.asArrayValue().list();
+
+        GenericArray<Object> array = new GenericData.Array<>(list.size(), avroSchema);
+        for (Value val : list) {
+            switch (val.getValueType()) {
+                case STRING:
+                    array.add(elementConverter.stringColumn(val.asStringValue().toString()));
+                    break;
+                case INTEGER:
+                    array.add(elementConverter.longColumn(val.asIntegerValue().toLong()));
+                    break;
+                case FLOAT:
+                    array.add(elementConverter.doubleColumn(val.asFloatValue().toDouble()));
+                    break;
+                case BOOLEAN:
+                    array.add(elementConverter.booleanColumn(val.asBooleanValue().getBoolean()));
+                    break;
+                case ARRAY:
+                    array.add(elementConverter.jsonColumn(val.asArrayValue()));
+                    break;
+                case MAP:
+                    array.add(elementConverter.jsonColumn(val.asMapValue()));
+                    break;
+                default:
+                    throw new RuntimeException("Irregular Messagepack type");
+            }
+        }
+        return array;
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroBooleanConverter.java

@@ -0,0 +1,14 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+
+public class AvroBooleanConverter extends AbstractAvroValueConverter {
+    public AvroBooleanConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Object booleanColumn(boolean value) {
+        throw new RuntimeException("Unsupported typecasting");
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroDoubleConverter.java

@@ -0,0 +1,29 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+
+public class AvroDoubleConverter extends AbstractAvroValueConverter {
+    public AvroDoubleConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Double longColumn(long value) {
+        return Long.valueOf(value).doubleValue();
+    }
+
+    @Override
+    public Double doubleColumn(double value) {
+        return value;
+    }
+
+    @Override
+    public Double stringColumn(String value) {
+        return Double.parseDouble(value);
+    }
+
+    @Override
+    public Double timestampColumn(String value) {
+        return Double.parseDouble(value);
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroEnumConverter.java

@@ -0,0 +1,25 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericEnumSymbol;
+
+import java.util.List;
+
+public class AvroEnumConverter extends AbstractAvroValueConverter {
+    private List<String> enumSymbols;
+
+    AvroEnumConverter(Schema schema, List<String> enumSymbols) {
+        super(schema);
+        this.enumSymbols = enumSymbols;
+    }
+
+    @Override
+    public GenericEnumSymbol stringColumn(String value) {
+        if (enumSymbols.contains(value)) {
+            return new GenericData.EnumSymbol(avroSchema, value);
+        } else {
+            throw new RuntimeException(String.format("%s is not in %s", value, enumSymbols.toString()));
+        }
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroFixedConverter.java

@@ -0,0 +1,16 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericFixed;
+
+public class AvroFixedConverter extends AbstractAvroValueConverter {
+    AvroFixedConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public GenericFixed stringColumn(String value) {
+        return new GenericData.Fixed(avroSchema, value.getBytes());
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroFloatConverter.java

@@ -0,0 +1,29 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+
+public class AvroFloatConverter extends AbstractAvroValueConverter {
+    public AvroFloatConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Float longColumn(long value) {
+        return Long.valueOf(value).floatValue();
+    }
+
+    @Override
+    public Float doubleColumn(double value) {
+        return Double.valueOf(value).floatValue();
+    }
+
+    @Override
+    public Float stringColumn(String value) {
+        return Float.parseFloat(value);
+    }
+
+    @Override
+    public Float timestampColumn(String value) {
+        return Float.parseFloat(value);
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroIntConverter.java

@@ -0,0 +1,29 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+
+public class AvroIntConverter extends AbstractAvroValueConverter {
+    public AvroIntConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Integer longColumn(long value) {
+        return Long.valueOf(value).intValue();
+    }
+
+    @Override
+    public Integer doubleColumn(double value) {
+        return Double.valueOf(value).intValue();
+    }
+
+    @Override
+    public Integer stringColumn(String value) {
+        return Integer.parseInt(value);
+    }
+
+    @Override
+    public Integer timestampColumn(String value) {
+        return Integer.parseInt(value);
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroLongConverter.java

@@ -0,0 +1,29 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+
+public class AvroLongConverter extends AbstractAvroValueConverter {
+    public AvroLongConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Long longColumn(long value) {
+        return value;
+    }
+
+    @Override
+    public Long doubleColumn(double value) {
+        return Math.round(value);
+    }
+
+    @Override
+    public Long stringColumn(String value) {
+        return Long.parseLong(value);
+    }
+
+    @Override
+    public Long timestampColumn(String value) {
+        return Long.parseLong(value);
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroMapConverter.java

@@ -0,0 +1,52 @@
+package org.embulk.formatter.avro.converter;
+
+import avro.shaded.com.google.common.collect.ImmutableMap;
+import org.apache.avro.Schema;
+import org.msgpack.value.Value;
+
+import java.util.Map;
+
+public class AvroMapConverter extends AbstractAvroValueConverter {
+    private AbstractAvroValueConverter elementConverter;
+
+    public AvroMapConverter(Schema schema, AbstractAvroValueConverter elementConverter) {
+        super(schema);
+        this.elementConverter = elementConverter;
+    }
+
+    @Override
+    public Map<String, Object> jsonColumn(Value value) {
+        if (!value.isMapValue())
+            throw new RuntimeException("Support only map type json record");
+
+        Map<Value, Value> map = value.asMapValue().map();
+
+        ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
+
+        for (Map.Entry<Value, Value> entry : map.entrySet()) {
+            switch (entry.getValue().getValueType()) {
+                case STRING:
+                    builder.put(entry.getKey().toString(), elementConverter.stringColumn(entry.getValue().asStringValue().toString()));
+                    break;
+                case INTEGER:
+                    builder.put(entry.getKey().toString(), elementConverter.longColumn(entry.getValue().asIntegerValue().toLong()));
+                    break;
+                case FLOAT:
+                    builder.put(entry.getKey().toString(), elementConverter.doubleColumn(entry.getValue().asFloatValue().toDouble()));
+                    break;
+                case BOOLEAN:
+                    builder.put(entry.getKey().toString(), elementConverter.booleanColumn(entry.getValue().asBooleanValue().getBoolean()));
+                    break;
+                case ARRAY:
+                    builder.put(entry.getKey().toString(), elementConverter.jsonColumn(entry.getValue().asArrayValue()));
+                    break;
+                case MAP:
+                    builder.put(entry.getKey().toString(), elementConverter.jsonColumn(entry.getValue().asMapValue()));
+                    break;
+                default:
+                    throw new RuntimeException("Irregular Messagepack type");
+            }
+        }
+        return builder.build();
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroNullConverter.java

@@ -0,0 +1,40 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.msgpack.value.Value;
+
+public class AvroNullConverter extends AbstractAvroValueConverter {
+    public AvroNullConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public Object booleanColumn(boolean value) {
+        return null;
+    }
+
+    @Override
+    public Object longColumn(long value) {
+        return null;
+    }
+
+    @Override
+    public Object doubleColumn(double value) {
+        return null;
+    }
+
+    @Override
+    public Object stringColumn(String value) {
+        return null;
+    }
+
+    @Override
+    public Object timestampColumn(String value) {
+        return null;
+    }
+
+    @Override
+    public Object jsonColumn(Value value) {
+        return null;
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroRecordConverter.java

@@ -0,0 +1,59 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import java.util.Map;
+
+public class AvroRecordConverter extends AbstractAvroValueConverter {
+    private Map<String, AbstractAvroValueConverter> converterTable;
+
+    public AvroRecordConverter(Schema schema, Map<String, AbstractAvroValueConverter> converterTable) {
+        super(schema);
+        this.converterTable = converterTable;
+    }
+
+    @Override
+    public GenericRecord jsonColumn(Value value) {
+        if (!value.isMapValue())
+            throw new RuntimeException("Support only map type json record");
+
+        Map<Value, Value> map = value.asMapValue().map();
+
+        GenericRecord record = new GenericData.Record(avroSchema);
+        for (Map.Entry<String, AbstractAvroValueConverter> entry : converterTable.entrySet()) {
+            Value key = ValueFactory.newString(entry.getKey());
+            if (!map.containsKey(key)) {
+                record.put(entry.getKey(), null);
+            } else {
+                Value child = map.get(ValueFactory.newString(entry.getKey()));
+                switch (child.getValueType()) {
+                    case STRING:
+                        record.put(entry.getKey(), entry.getValue().stringColumn(child.asStringValue().toString()));
+                        break;
+                    case INTEGER:
+                        record.put(entry.getKey(), entry.getValue().longColumn(child.asIntegerValue().toLong()));
+                        break;
+                    case FLOAT:
+                        record.put(entry.getKey(), entry.getValue().doubleColumn(child.asFloatValue().toDouble()));
+                        break;
+                    case BOOLEAN:
+                        record.put(entry.getKey(), entry.getValue().booleanColumn(child.asBooleanValue().getBoolean()));
+                        break;
+                    case ARRAY:
+                        record.put(entry.getKey(), entry.getValue().jsonColumn(child.asArrayValue()));
+                        break;
+                    case MAP:
+                        record.put(entry.getKey(), entry.getValue().jsonColumn(child.asMapValue()));
+                        break;
+                    default:
+                        throw new RuntimeException("Irregular Messagepack type");
+                }
+            }
+        }
+        return record;
+    }
+}

data/src/main/java/org/embulk/formatter/avro/converter/AvroStringConverter.java

@@ -0,0 +1,41 @@
+package org.embulk.formatter.avro.converter;
+
+import org.apache.avro.Schema;
+import org.msgpack.value.Value;
+
+public class AvroStringConverter extends AbstractAvroValueConverter {
+    public AvroStringConverter(Schema schema) {
+        super(schema);
+    }
+
+    @Override
+    public String booleanColumn(boolean value) {
+        return String.valueOf(value);
+    }
+
+    @Override
+    public String longColumn(long value) {
+        return String.valueOf(value);
+    }
+
+    @Override
+    public String doubleColumn(double value) {
+        return String.valueOf(value);
+    }
+
+    @Override
+    public String stringColumn(String value) {
+        return value;
+    }
+
+    @Override
+    public String timestampColumn(String value) {
+        return value;
+    }
+
+
+    @Override
+    public String jsonColumn(Value value) {
+        return value.toJson();
+    }
+}