embulk-output-kafka 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c2c8249b98bf48ab359b21e01dc0f4112ebf800f
-  data.tar.gz: 538847da32ff4ad84f24495c67d790185043ce02
+SHA256:
+  metadata.gz: cb0e6dfc4b8b49b93fb6966948e8930e5785cf5a2a89fc5a8f17c80a4e7865c1
+  data.tar.gz: 18cb578eba9423c490e7416c906054f8401301b977e796c271182f593dc08563
 SHA512:
-  metadata.gz: 0542a0a92e3c8b2a998177c874acf4d0209d0e90889894999daff756840169fe585877af51b5368eaada9ba4eb8bd6c6676fafc8ff5fc84d4edf193df19881de
-  data.tar.gz: 1cd8fb2c55282f082eef56ea983cbc59cbae551db606f55bf21c1fccfbea37cfe633ef713eb66b089f0229ab5b97ae7b9e79ad38eddfc79d9157ca2e68fc869a
+  metadata.gz: 84a080d3cc49d30ad04802162e94de0072aac96d2562649c34cb1623e7734f783e96deeb8082c81ce965b36520c21be122cdf0e292a55697c9d2c89fd767f551
+  data.tar.gz: 4eb1b72cb705b2eb473ae76f02085987b4f55a37d9ba7048789ea3133e8056850813d39cfb86b2d4f7482b31fbbb67f5387b2c3a730dfb0bffd2c5f9015d16d0

data/.circleci/config.yml

@@ -0,0 +1,44 @@
+# Java Gradle CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-java/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/openjdk:8-jdk
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx3200m
+      TERM: dumb
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "build.gradle" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+
+      - run: ./gradlew dependencies
+
+      - save_cache:
+          paths:
+            - ~/.gradle
+          key: v1-dependencies-{{ checksum "build.gradle" }}
+
+      # run tests!
+      - run: ./gradlew test
+      - store_test_results:
+          path: build/test-results

data/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gradle" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

data/README.md

@@ -1,4 +1,5 @@
 # Kafka output plugin for Embulk
+[![CircleCI](https://circleci.com/gh/joker1007/embulk-output-kafka.svg?style=svg)](https://circleci.com/gh/joker1007/embulk-output-kafka)
 
 ## Overview
 
@@ -19,6 +20,7 @@
 - **ignore_columns**: remove columns from output  (array(string), default: `[]`)
 - **key_column_name**: use column value as record key (string, default: `null`, if this parameter is null, set random number as record key, and it can use column in `ignore_columns`)
 - **partition_column_name**: use column value as partition id (string, default: `null`, this value is prefer to `key_column_name`, and if partition_column value is null, use key_column for partitioning)
+- **column_for_deletion**: Determine to delete (string, default: `null`, `column_for_deletion` column must be boolean. If the value of the column is `true`, KafkaProducer sends `null` value to a Kafka Broker.)
 - **record_batch_size**: kafka producer record batch size (integer, default: `1000`)
 - **acks**: kafka producer require acks (string, default: `"1"`)
 - **retries**: kafka producer max retry count (integer, default: `1`)
@@ -28,7 +30,9 @@
 If use `avro_with_schema_registry` format, following configs are required.
 
 - **schema_registry_url**
-- **avsc** or **avsc_file**
+
+If avsc and avsc_file are null, embulk-output-kafka fetch a schema from schema registry.
+But currently, embulk-output-kafka supports only TopicNameStrategy.
 
 ## Example
 

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "1.5.0"
+    id "com.jfrog.bintray" version "1.8.5"
+    id "com.github.jruby-gradle.base" version "1.6.0"
     id "java"
     id "checkstyle"
 }
@@ -17,26 +17,43 @@ configurations {
     provided
 }
 
-version = "0.1.7"
+version = "0.1.8"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.17"
-    provided "org.embulk:embulk-core:0.9.17"
+    compile  "org.embulk:embulk-core:0.9.22"
+    provided "org.embulk:embulk-core:0.9.22"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.12"
-    testCompile "org.embulk:embulk-test:0.9.17"
-    testCompile "org.embulk:embulk-standards:0.9.17"
+    testCompile "junit:junit:4.13"
+    testCompile "org.embulk:embulk-test:0.9.22"
+    testCompile "org.embulk:embulk-standards:0.9.22"
+    testCompile "org.embulk:embulk-deps-buffer:0.9.22"
 
-    compile "org.apache.kafka:kafka-clients:2.3.0"
-    compile("org.apache.avro:avro:1.9.0") {
+    compile("org.apache.kafka:kafka-clients:2.5.1") {
+        exclude group: "org.slf4j", module: "slf4j-api"
     }
-    compile("io.confluent:kafka-avro-serializer:5.3.0") {
+    compile("org.apache.avro:avro:1.10.0") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+        exclude group: "org.slf4j", module: "slf4j-api"
+    }
+    compile("io.confluent:kafka-avro-serializer:5.5.1") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+        exclude group: "org.slf4j", module: "slf4j-api"
     }
 
-    testCompile("com.github.charithe:kafka-junit:4.1.6") {
+    testCompile "com.salesforce.kafka.test:kafka-junit4:3.+"
+    testCompile("org.apache.kafka:kafka_2.12:2.5.+") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+//        exclude group: "com.fasterxml.jackson.dataformat", module: "jackson-dataformat-csv"
+        exclude group: "com.fasterxml.jackson.datatype", module: "jackson-datatype-jdk8"
     }
 }
 

data/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://broker:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
 

data/src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java

@@ -14,7 +14,7 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
+public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor<GenericRecord>
 {
     private Schema avroSchema;
     private GenericRecord genericRecord;
@@ -25,8 +25,13 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
         this.avroSchema = avroSchema;
     }
 
-    GenericRecord getGenericRecord()
+    @Override
+    public GenericRecord getRecord()
     {
+        if (isDeletion()) {
+            return null;
+        }
+
         return genericRecord;
     }
 
@@ -40,6 +45,8 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
+        super.booleanColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -145,9 +152,8 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
         switch (avroSchema.getType()) {
             case ARRAY:
                 if (value.isArrayValue()) {
-                    return value.asArrayValue().list().stream().map(item -> {
-                        return convertMsgPackValueToAvroValue(avroSchema.getElementType(), item);
-                    }).filter(Objects::nonNull).collect(Collectors.toList());
+                    return value.asArrayValue().list().stream().map(item ->
+                            convertMsgPackValueToAvroValue(avroSchema.getElementType(), item)).filter(Objects::nonNull).collect(Collectors.toList());
                 }
                 throw new RuntimeException(String.format("Schema mismatch: avro: %s, msgpack: %s", avroSchema.getType().getName(), value.getValueType().name()));
             case MAP:
@@ -166,9 +172,8 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
                     GenericRecord record = new GenericData.Record(avroSchema);
                     Map<Value, Value> valueMap = value.asMapValue().map();
                     for (org.apache.avro.Schema.Field field : avroSchema.getFields()) {
-                        Optional.ofNullable(valueMap.get(ValueFactory.newString(field.name()))).ifPresent(v -> {
-                            record.put(field.name(), convertMsgPackValueToAvroValue(field.schema(), v));
-                        });
+                        Optional.ofNullable(valueMap.get(ValueFactory.newString(field.name()))).ifPresent(v ->
+                                record.put(field.name(), convertMsgPackValueToAvroValue(field.schema(), v)));
                     }
                     return record;
                 }

data/src/main/java/org/embulk/output/kafka/AvroFormatTransactionalPageOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.output.kafka;
+
+import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.embulk.spi.PageReader;
+
+public class AvroFormatTransactionalPageOutput extends KafkaTransactionalPageOutput<Object, GenericRecord>
+{
+    public AvroFormatTransactionalPageOutput(KafkaProducer<Object, Object> producer, PageReader pageReader, KafkaOutputColumnVisitor<GenericRecord> columnVisitor, String topic, int taskIndex)
+    {
+        super(producer, pageReader, columnVisitor, topic, taskIndex);
+    }
+};

data/src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java

@@ -10,7 +10,7 @@ import org.msgpack.value.Value;
 import java.io.IOException;
 import java.time.format.DateTimeFormatter;
 
-public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
+public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor<ObjectNode>
 {
     private ObjectMapper objectMapper;
     private ObjectNode jsonNode;
@@ -23,8 +23,13 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
         this.objectMapper = objectMapper;
     }
 
-    ObjectNode getJsonNode()
+    @Override
+    public ObjectNode getRecord()
     {
+        if (isDeletion()) {
+            return null;
+        }
+
         return jsonNode;
     }
 
@@ -38,6 +43,8 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
+        super.booleanColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }

data/src/main/java/org/embulk/output/kafka/JsonFormatTransactionalPageOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.output.kafka;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.embulk.spi.PageReader;
+
+public class JsonFormatTransactionalPageOutput extends KafkaTransactionalPageOutput<ObjectNode, ObjectNode>
+{
+    public JsonFormatTransactionalPageOutput(KafkaProducer<Object, ObjectNode> producer, PageReader pageReader, KafkaOutputColumnVisitor<ObjectNode> columnVisitor, String topic, int taskIndex)
+    {
+        super(producer, pageReader, columnVisitor, topic, taskIndex);
+    }
+};

data/src/main/java/org/embulk/output/kafka/KafkaJsonSerializer.java

@@ -12,6 +12,10 @@ public class KafkaJsonSerializer implements Serializer<ObjectNode>
     @Override
     public byte[] serialize(String topic, ObjectNode data)
     {
+        if (data == null) {
+            return null;
+        }
+
         try {
             return objectMapper.writeValueAsBytes(data);
         }

data/src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java

@@ -4,7 +4,7 @@ import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 
-public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
+public abstract class KafkaOutputColumnVisitor<T> implements ColumnVisitor
 {
     private KafkaOutputPlugin.PluginTask task;
     PageReader pageReader;
@@ -13,6 +13,7 @@ public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
     private Object recordKey = null;
     private String topicName = null;
     private Integer partition = null;
+    private boolean deletion = false;
 
     KafkaOutputColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader)
     {
@@ -21,6 +22,8 @@ public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
         this.partitionColumnName = task.getPartitionColumnName().orElse(null);
     }
 
+    public abstract T getRecord();
+
     Object getRecordKey()
     {
         return recordKey;
@@ -50,11 +53,17 @@ public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
         return partition;
     }
 
+    boolean isDeletion()
+    {
+        return deletion;
+    }
+
     void reset()
     {
         this.recordKey = null;
         this.topicName = null;
         this.partition = null;
+        this.deletion = false;
     }
 
     boolean isIgnoreColumn(Column column)
@@ -62,6 +71,21 @@ public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
         return task.getIgnoreColumns().stream().anyMatch(name -> name.equals(column.getName()));
     }
 
+    boolean isColumnForDeletion(Column column)
+    {
+        return task.getColumnForDeletion().map(name -> name.equals(column.getName())).orElse(false);
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (!pageReader.isNull(column)) {
+            if (isColumnForDeletion(column)) {
+                deletion = pageReader.getBoolean(column);
+            }
+        }
+    }
+
     @Override
     public void longColumn(Column column)
     {

data/src/main/java/org/embulk/output/kafka/KafkaOutputPlugin.java

@@ -5,11 +5,16 @@ import com.fasterxml.jackson.annotation.JsonValue;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.ImmutableList;
+import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.SchemaRegistryClient;
+import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException;
+import io.confluent.kafka.schemaregistry.testutil.MockSchemaRegistry;
+import io.confluent.kafka.serializers.subject.TopicNameStrategy;
+import io.confluent.kafka.serializers.subject.strategy.SubjectNameStrategy;
 import org.apache.kafka.clients.admin.AdminClient;
 import org.apache.kafka.clients.admin.AdminClientConfig;
 import org.apache.kafka.clients.admin.DescribeTopicsResult;
 import org.apache.kafka.clients.producer.KafkaProducer;
-import org.apache.kafka.clients.producer.ProducerRecord;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -20,12 +25,9 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.File;
 import java.io.IOException;
@@ -33,13 +35,10 @@ import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
-import java.util.PrimitiveIterator;
 import java.util.Properties;
-import java.util.Random;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
 
 public class KafkaOutputPlugin
         implements OutputPlugin
@@ -107,7 +106,7 @@ public class KafkaOutputPlugin
         public Optional<String> getPartitionColumnName();
 
         @Config("record_batch_size")
-        @ConfigDefault("1000")
+        @ConfigDefault("16384")
         public int getRecordBatchSize();
 
         @Config("acks")
@@ -129,17 +128,21 @@ public class KafkaOutputPlugin
         @Config("value_subject_name_strategy")
         @ConfigDefault("null")
         public Optional<String> getValueSubjectNameStrategy();
+
+        @Config("column_for_deletion")
+        @ConfigDefault("null")
+        public Optional<String> getColumnForDeletion();
     }
 
     private static ObjectMapper objectMapper = new ObjectMapper();
-    private Logger logger = LoggerFactory.getLogger(getClass());
+
+    private static final int SCHEMA_REGISTRY_IDENTITY_MAP_CAPACITY = 1000;
 
     private AdminClient getKafkaAdminClient(PluginTask task)
     {
         Properties properties = new Properties();
         properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, task.getBrokers());
-        AdminClient adminClient = AdminClient.create(properties);
-        return adminClient;
+        return AdminClient.create(properties);
     }
 
     @Override
@@ -189,101 +192,48 @@ public class KafkaOutputPlugin
             case AVRO_WITH_SCHEMA_REGISTRY:
                 return buildPageOutputForAvroWithSchemaRegistry(task, schema, taskIndex);
             default:
-                throw new ConfigException("Unknow serialize format");
+                throw new ConfigException("Unknown serialize format");
         }
     }
 
     private TransactionalPageOutput buildPageOutputForJson(PluginTask task, Schema schema, int taskIndex)
     {
         KafkaProducer<Object, ObjectNode> producer = RecordProducerFactory.getForJson(task, schema, task.getOtherProducerConfigs());
-
         PageReader pageReader = new PageReader(schema);
-        PrimitiveIterator.OfLong randomLong = new Random().longs(1, Long.MAX_VALUE).iterator();
-        AtomicLong counter = new AtomicLong(0);
-        AtomicLong recordLoggingCount = new AtomicLong(1);
-
-        return new TransactionalPageOutput() {
-            private JsonFormatColumnVisitor columnVisitor = new JsonFormatColumnVisitor(task, pageReader, objectMapper);
-
-            @Override
-            public void add(Page page)
-            {
-                pageReader.setPage(page);
-                while (pageReader.nextRecord()) {
-                    columnVisitor.reset();
-
-                    pageReader.getSchema().visitColumns(columnVisitor);
-
-                    Object recordKey = columnVisitor.getRecordKey();
-                    if (recordKey == null) {
-                        recordKey = randomLong.next();
-                    }
-
-                    String targetTopic = columnVisitor.getTopicName() != null ? columnVisitor.getTopicName() : task.getTopic();
-                    ProducerRecord<Object, ObjectNode> producerRecord = new ProducerRecord<>(targetTopic, columnVisitor.getPartition(), recordKey, columnVisitor.getJsonNode());
-                    producer.send(producerRecord, (metadata, exception) -> {
-                        if (exception != null) {
-                            logger.error("produce error", exception);
-                        }
-
-                        logger.debug("sent record: {topic: {}, key: {}, value: {}, partition: {}}",
-                                producerRecord.topic(),
-                                producerRecord.key(),
-                                producerRecord.value(),
-                                producerRecord.partition());
-
-                        long current = counter.incrementAndGet();
-                        if (current >= recordLoggingCount.get()) {
-                            logger.info("[task-{}] Producer sent {} records", String.format("%04d", taskIndex), current);
-                            recordLoggingCount.set(recordLoggingCount.get() * 2);
-                        }
-                    });
-                }
-            }
+        KafkaOutputColumnVisitor<ObjectNode> columnVisitor = new JsonFormatColumnVisitor(task, pageReader, objectMapper);
 
-            @Override
-            public void finish()
-            {
-                producer.flush();
-            }
-
-            @Override
-            public void close()
-            {
-                producer.close();
-            }
-
-            @Override
-            public void abort()
-            {
-                producer.flush();
-                producer.close();
-            }
-
-            @Override
-            public TaskReport commit()
-            {
-                return null;
-            }
-        };
+        return new JsonFormatTransactionalPageOutput(producer, pageReader, columnVisitor, task.getTopic(), taskIndex);
     }
 
     private TransactionalPageOutput buildPageOutputForAvroWithSchemaRegistry(PluginTask task, Schema schema, int taskIndex)
     {
         KafkaProducer<Object, Object> producer = RecordProducerFactory.getForAvroWithSchemaRegistry(task, schema, task.getOtherProducerConfigs());
-
         PageReader pageReader = new PageReader(schema);
+        org.apache.avro.Schema avroSchema = getAvroSchema(task);
+        AvroFormatColumnVisitor avroFormatColumnVisitor = new AvroFormatColumnVisitor(task, pageReader, avroSchema);
+
+        return new AvroFormatTransactionalPageOutput(producer, pageReader, avroFormatColumnVisitor, task.getTopic(), taskIndex);
+    }
 
+    private org.apache.avro.Schema getAvroSchema(PluginTask task)
+    {
         org.apache.avro.Schema avroSchema = null;
-        if (!task.getAvsc().isPresent() && !task.getAvscFile().isPresent() || task.getAvsc().isPresent() == task.getAvscFile().isPresent()) {
+        if (!task.getSchemaRegistryUrl().isPresent()) {
+            throw new ConfigException("avro_with_schema_registry format needs schema_registry_url");
+        }
+
+        if (task.getAvsc().isPresent() && task.getAvscFile().isPresent()) {
             throw new ConfigException("avro_with_schema_registry format needs either one of avsc and avsc_file");
         }
+
         if (task.getAvsc().isPresent()) {
             avroSchema = new org.apache.avro.Schema.Parser().parse(task.getAvsc().get().toString());
+            return avroSchema;
         }
         if (task.getAvscFile().isPresent()) {
             try {
                 avroSchema = new org.apache.avro.Schema.Parser().parse(task.getAvscFile().get());
+                return avroSchema;
             }
             catch (IOException e) {
                 e.printStackTrace();
@@ -291,77 +241,28 @@ public class KafkaOutputPlugin
             }
         }
 
-        PrimitiveIterator.OfLong randomLong = new Random().longs(1, Long.MAX_VALUE).iterator();
-
-        AtomicLong counter = new AtomicLong(0);
-        AtomicLong recordLoggingCount = new AtomicLong(1);
-
-        final org.apache.avro.Schema finalAvroSchema = avroSchema;
-        return new TransactionalPageOutput()
-        {
-            private AvroFormatColumnVisitor columnVisitor = new AvroFormatColumnVisitor(task, pageReader, finalAvroSchema);
-
-            @Override
-            public void add(Page page)
-            {
-                pageReader.setPage(page);
-                while (pageReader.nextRecord()) {
-                    columnVisitor.reset();
-
-                    pageReader.getSchema().visitColumns(columnVisitor);
-
-                    Object recordKey = columnVisitor.getRecordKey();
-                    if (recordKey == null) {
-                        recordKey = randomLong.next();
-                    }
-
-                    String targetTopic = columnVisitor.getTopicName() != null ? columnVisitor.getTopicName() : task.getTopic();
-
-                    ProducerRecord<Object, Object> producerRecord = new ProducerRecord<>(targetTopic, columnVisitor.getPartition(), recordKey, columnVisitor.getGenericRecord());
-                    producer.send(producerRecord, (metadata, exception) -> {
-                        if (exception != null) {
-                            logger.error("produce error", exception);
-                        }
-
-                        logger.debug("sent record: {topic: {}, key: {}, value: {}, partition: {}}",
-                                producerRecord.topic(),
-                                producerRecord.key(),
-                                producerRecord.value(),
-                                producerRecord.partition());
-
-                        long current = counter.incrementAndGet();
-                        if (current >= recordLoggingCount.get()) {
-                            logger.info("[task-{}] Producer sent {} records", String.format("%04d", taskIndex), current);
-                            recordLoggingCount.set(recordLoggingCount.get() * 2);
-                        }
-                    });
-                }
-            }
-
-            @Override
-            public void finish()
-            {
-                producer.flush();
-            }
-
-            @Override
-            public void close()
-            {
-                producer.close();
-            }
-
-            @Override
-            public void abort()
-            {
-                producer.flush();
-                producer.close();
-            }
+        SchemaRegistryClient schemaRegistryClient = getSchemaRegistryClient(task.getSchemaRegistryUrl().get());
+        SubjectNameStrategy subjectNameStrategy = new TopicNameStrategy();
+        String subjectName = subjectNameStrategy.subjectName(task.getTopic(), false, null);
+        try {
+            String schema = schemaRegistryClient.getLatestSchemaMetadata(subjectName).getSchema();
+            avroSchema = new org.apache.avro.Schema.Parser().parse(schema);
+            return avroSchema;
+        }
+        catch (IOException | RestClientException e) {
+            throw new ConfigException("cannot fetch latest schema from schema registry.", e);
+        }
+    }
 
-            @Override
-            public TaskReport commit()
-            {
-                return null;
-            }
-        };
+    private static final String MOCK_SCHEMA_REGISTRY_PREFIX = "mock://";
+    private SchemaRegistryClient getSchemaRegistryClient(String url)
+    {
+        if (url.startsWith(MOCK_SCHEMA_REGISTRY_PREFIX)) {
+            String mockScope = url.substring(MOCK_SCHEMA_REGISTRY_PREFIX.length());
+            return MockSchemaRegistry.getClientForScope(mockScope);
+        }
+        else {
+            return new CachedSchemaRegistryClient(url, SCHEMA_REGISTRY_IDENTITY_MAP_CAPACITY);
+        }
     }
 }