embulk-parser-avro 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2ee00ffbaf0cf78dfc2c7bd0861e68603b528565
-  data.tar.gz: 08786ae000735a7f8d2aad1514a02de9c5095ec4
+  metadata.gz: c910b362e790acea208b141982b8d9530d3c29c8
+  data.tar.gz: 450bc2c36228d32575e04fdd2bca4664536ca5f7
 SHA512:
-  metadata.gz: 2697fb6b9cd4fb2cf6a72194450031f11ade7780d2afb050d40d65de39a10a5b56da507125b5580de820ba78e399ff39348726bb0eb82edb0c7aae13de27a555
-  data.tar.gz: 839d69c7a19dfc6eeddff3b4fb9f2bee3172b1d4ab05793753bae1fb466ef1bfa91c5e86e099f3fe44bc5c0e0047ed3a34eb789ae26c7e0507b29c25a58ce4ca
+  metadata.gz: 521f6e18548b1d163c2ecbbc2d2c19be584d3ab72d7530f63302ea16e9b36cbe66297c39ea4ccf08dfe4064708fc7d0a6eddeb2d472341e724687f4cc585f303
+  data.tar.gz: acda88bdc802a4b7c8a64818ab426a48bb39c7bb2d4ee8c1a54d9424fc8e3982ab9f5431b67f9cb90167b5ddb2bead8fa1ec54d2986d09a38e49697be0ef8f19

data/README.md

@@ -5,7 +5,7 @@
 ## Overview
 
 * **Plugin type**: parser
-* **Guess supported**: no
+* **Guess supported**: yes
 
 ## Configuration
 
@@ -102,7 +102,7 @@ out:
 }
 ```
 
-(If guess supported) you don't have to write `parser:` section in the configuration file. After writing `in:` section, you can let embulk guess `parser:` section using this command:
+You don't have to write `parser:` section in the configuration file. After writing `in:` section, you can let embulk guess `parser:` section using this command:
 
 ```
 $ embulk gem install embulk-parser-avro

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.4.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -21,7 +21,12 @@ targetCompatibility = 1.8
 dependencies {
     compile  "org.embulk:embulk-core:0.9.4"
     provided "org.embulk:embulk-core:0.9.4"
-    compile "org.apache.avro:avro:1.8.2"
+    compile ("org.apache.avro:avro:1.10.1") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+    }
+    compile "com.github.luben:zstd-jni:1.4.5-12"
+    compile "org.tukaani:xz:1.8"
+    compile "org.xerial.snappy:snappy-java:1.1.8.1"
     testCompile "junit:junit:4.+"
 
     testCompile  "org.embulk:embulk-core:0.9.4:tests"

data/lib/embulk/guess/avro.rb

@@ -1,61 +1,3 @@
-module Embulk
-  module Guess
-
-    # TODO implement guess plugin to make this command work:
-    #      $ embulk guess -g "avro" partial-config.yml
-    #
-    #      Depending on the file format the plugin uses, you can use choose
-    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
-    #      or line guess (LineGuessPlugin).
-
-    #class Avro < GuessPlugin
-    #  Plugin.register_guess("avro", self)
-    #
-    #  def guess(config, sample_buffer)
-    #    if sample_buffer[0,2] == GZIP_HEADER
-    #      guessed = {}
-    #      guessed["type"] = "avro"
-    #      guessed["property1"] = "guessed-value"
-    #      return {"parser" => guessed}
-    #    else
-    #      return {}
-    #    end
-    #  end
-    #end
-
-    #class Avro < TextGuessPlugin
-    #  Plugin.register_guess("avro", self)
-    #
-    #  def guess_text(config, sample_text)
-    #    js = JSON.parse(sample_text) rescue nil
-    #    if js && js["mykeyword"] == "keyword"
-    #      guessed = {}
-    #      guessed["type"] = "avro"
-    #      guessed["property1"] = "guessed-value"
-    #      return {"parser" => guessed}
-    #    else
-    #      return {}
-    #    end
-    #  end
-    #end
-
-    #class Avro < LineGuessPlugin
-    #  Plugin.register_guess("avro", self)
-    #
-    #  def guess_lines(config, sample_lines)
-    #    all_line_matched = sample_lines.all? do |line|
-    #      line =~ /mypattern/
-    #    end
-    #    if all_line_matched
-    #      guessed = {}
-    #      guessed["type"] = "avro"
-    #      guessed["property1"] = "guessed-value"
-    #      return {"parser" => guessed}
-    #    else
-    #      return {}
-    #    end
-    #  end
-    #end
-
-  end
-end
+Embulk::JavaPlugin.register_guess(
+  "avro", "org.embulk.guess.avro.AvroGuessPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/guess/avro/AvroGuessPlugin.java

@@ -0,0 +1,97 @@
+package org.embulk.guess.avro;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.SeekableByteArrayInput;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericRecord;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.GuessPlugin;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AvroGuessPlugin
+        implements GuessPlugin {
+
+    private static final byte[] AVRO_HEADER = {0x4f, 0x62, 0x6a, 0x01};
+
+    private static final Map<Schema.Type, Type> TYPE_MAP = new EnumMap<>(Schema.Type.class);
+
+    static {
+        TYPE_MAP.put(Schema.Type.STRING, Types.STRING);
+        TYPE_MAP.put(Schema.Type.BYTES, Types.STRING);
+        TYPE_MAP.put(Schema.Type.FIXED, Types.STRING);
+        TYPE_MAP.put(Schema.Type.ENUM, Types.STRING);
+        TYPE_MAP.put(Schema.Type.NULL, Types.STRING);
+        TYPE_MAP.put(Schema.Type.INT, Types.LONG);
+        TYPE_MAP.put(Schema.Type.LONG, Types.LONG);
+        TYPE_MAP.put(Schema.Type.FLOAT, Types.DOUBLE);
+        TYPE_MAP.put(Schema.Type.DOUBLE, Types.DOUBLE);
+        TYPE_MAP.put(Schema.Type.BOOLEAN, Types.BOOLEAN);
+        TYPE_MAP.put(Schema.Type.MAP, Types.JSON);
+        TYPE_MAP.put(Schema.Type.ARRAY, Types.JSON);
+        TYPE_MAP.put(Schema.Type.RECORD, Types.JSON);
+    }
+
+    private Type convertType(Schema.Field field) {
+        Schema.Type type = field.schema().getType();
+        if (type == Schema.Type.UNION) {
+            for (Schema schema : field.schema().getTypes()) {
+                Schema.Type t = schema.getType();
+                if (t != Schema.Type.NULL) {
+                    type = t;
+                    break;
+                }
+            }
+        }
+        return TYPE_MAP.get(type);
+    }
+
+    private byte[] copyBuffer(Buffer buffer, int size) {
+        byte[] bytes = new byte[size];
+        buffer.getBytes(0, bytes, 0, size);
+        return bytes;
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config, Buffer sample) {
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        byte[] bytes = copyBuffer(sample, AVRO_HEADER.length);
+        if (!Arrays.equals(bytes, AVRO_HEADER)) {
+            return configDiff;
+        }
+        ConfigDiff parserConfig = configDiff.set("parser", Collections.emptyMap()).getNested("parser");
+        parserConfig.set("type", "avro");
+
+        bytes = copyBuffer(sample, sample.capacity());
+        DataFileReader<GenericRecord> reader;
+        try {
+            reader = new DataFileReader<>(new SeekableByteArrayInput(bytes), new GenericDatumReader<>());
+        } catch (IOException e) {
+            return configDiff;
+        }
+        List<Map<String, String>> columns = new ArrayList<>();
+        for (Schema.Field field : reader.getSchema().getFields()) {
+            Map<String, String> column = new HashMap<>();
+            column.put("name", field.name());
+            column.put("type", convertType(field).getName());
+            columns.add(column);
+        }
+        parserConfig.set("columns", columns);
+
+        return configDiff;
+    }
+}

data/src/test/java/org/embulk/guess/avro/TestAvroGuessPlugin.java

@@ -0,0 +1,81 @@
+package org.embulk.guess.avro;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.collect.ImmutableMap;
+import org.apache.commons.compress.utils.IOUtils;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Buffer;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestAvroGuessPlugin {
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private AvroGuessPlugin plugin;
+
+    @Before
+    public void setUp() {
+        plugin = new AvroGuessPlugin();
+        config = runtime.getExec().newConfigSource();
+    }
+
+    @Test
+    public void testAvroFile() throws IOException {
+        Map<String, String> expectedColumns = ImmutableMap.<String, String>builder()
+                .put("id", "long")
+                .put("code", "long")
+                .put("name", "string")
+                .put("description", "string")
+                .put("flag", "boolean")
+                .put("created_at", "string")
+                .put("created_at_utc", "double")
+                .put("price", "double")
+                .put("spec", "json")
+                .put("tags", "json")
+                .put("options", "json")
+                .put("item_type", "string")
+                .put("dummy", "string")
+                .build();
+
+        ConfigDiff configDiff = guess("items.avro");
+
+        JsonNode parserNode = configDiff.getObjectNode().get("parser");
+        assertEquals("avro", parserNode.get("type").asText());
+
+        Iterator<JsonNode> it = parserNode.get("columns").elements();
+        while (it.hasNext()) {
+            JsonNode node = it.next();
+            String name = node.get("name").asText();
+            assertTrue(expectedColumns.containsKey(name));
+            assertEquals(expectedColumns.get(name), node.get("type").asText());
+        }
+    }
+
+    @Test
+    public void testNonAvroFile() throws IOException {
+        ConfigDiff configDiff = guess("data.json");
+        ObjectNode objectNode = configDiff.getObjectNode();
+        assertEquals(0, objectNode.size());
+    }
+
+    private ConfigDiff guess(String resource) throws IOException {
+        InputStream is = this.getClass().getResourceAsStream("/org/embulk/parser/avro/" + resource);
+        Buffer sample = Buffer.wrap(IOUtils.toByteArray(is));
+        return plugin.guess(config, sample);
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-avro
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-23 00:00:00.000000000 Z
+date: 2021-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,6 +63,7 @@ files:
 - gradlew.bat
 - lib/embulk/guess/avro.rb
 - lib/embulk/parser/avro.rb
+- src/main/java/org/embulk/guess/avro/AvroGuessPlugin.java
 - src/main/java/org/embulk/parser/avro/AvroParserPlugin.java
 - src/main/java/org/embulk/parser/avro/TimestampUnit.java
 - src/main/java/org/embulk/parser/avro/TimestampUnitDeserializer.java
@@ -76,6 +77,7 @@ files:
 - src/main/java/org/embulk/parser/avro/getter/IntegerColumnGetter.java
 - src/main/java/org/embulk/parser/avro/getter/LongColumnGetter.java
 - src/main/java/org/embulk/parser/avro/getter/StringColumnGetter.java
+- src/test/java/org/embulk/guess/avro/TestAvroGuessPlugin.java
 - src/test/java/org/embulk/parser/avro/TestAvroParserPlugin.java
 - src/test/resources/org/embulk/parser/avro/.gitignore
 - src/test/resources/org/embulk/parser/avro/Gemfile
@@ -87,14 +89,14 @@ files:
 - src/test/resources/org/embulk/parser/avro/item2.avsc
 - src/test/resources/org/embulk/parser/avro/items.avro
 - src/test/resources/org/embulk/parser/avro/items2.avro
-- classpath/avro-1.8.2.jar
-- classpath/paranamer-2.7.jar
-- classpath/xz-1.5.jar
-- classpath/commons-compress-1.8.1.jar
-- classpath/embulk-parser-avro-0.3.0.jar
-- classpath/jackson-core-asl-1.9.13.jar
-- classpath/snappy-java-1.1.1.3.jar
-- classpath/jackson-mapper-asl-1.9.13.jar
+- classpath/avro-1.10.1.jar
+- classpath/zstd-jni-1.4.5-12.jar
+- classpath/xz-1.8.jar
+- classpath/snappy-java-1.1.8.1.jar
+- classpath/commons-compress-1.20.jar
+- classpath/jackson-core-2.11.3.jar
+- classpath/slf4j-api-1.7.30.jar
+- classpath/embulk-parser-avro-0.4.0.jar
 homepage: https://github.com/joker1007/embulk-parser-avro
 licenses:
 - MIT