embulk-output-kafka 0.1.3 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fb1edf4ce79bd490f0f01662547d36c2e6d2dd6e
-  data.tar.gz: 44b9affb8e9d1385314274b5d1ac083fd5a4848e
+SHA256:
+  metadata.gz: cb0e6dfc4b8b49b93fb6966948e8930e5785cf5a2a89fc5a8f17c80a4e7865c1
+  data.tar.gz: 18cb578eba9423c490e7416c906054f8401301b977e796c271182f593dc08563
 SHA512:
-  metadata.gz: 5c76d2cca3d141b7f44208449ba092ee732993183de6e6f19ec4e5280f302f0e3a4b5746414332103aa492b0cb7ea7df55159fa936248839bb68a8280ecfc060
-  data.tar.gz: 28dcd4f04aceec97e78c427b2b859266b9adb5147ae8cd28a63c5df86840d5c4051af049b225eefe01d410007afdd17222ec96fc3525cf0cd28e60c30414f6a2
+  metadata.gz: 84a080d3cc49d30ad04802162e94de0072aac96d2562649c34cb1623e7734f783e96deeb8082c81ce965b36520c21be122cdf0e292a55697c9d2c89fd767f551
+  data.tar.gz: 4eb1b72cb705b2eb473ae76f02085987b4f55a37d9ba7048789ea3133e8056850813d39cfb86b2d4f7482b31fbbb67f5387b2c3a730dfb0bffd2c5f9015d16d0

data/.circleci/config.yml

@@ -0,0 +1,44 @@
+# Java Gradle CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-java/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/openjdk:8-jdk
+
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx3200m
+      TERM: dumb
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "build.gradle" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+
+      - run: ./gradlew dependencies
+
+      - save_cache:
+          paths:
+            - ~/.gradle
+          key: v1-dependencies-{{ checksum "build.gradle" }}
+
+      # run tests!
+      - run: ./gradlew test
+      - store_test_results:
+          path: build/test-results

data/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gradle" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

data/README.md

@@ -1,4 +1,5 @@
 # Kafka output plugin for Embulk
+[![CircleCI](https://circleci.com/gh/joker1007/embulk-output-kafka.svg?style=svg)](https://circleci.com/gh/joker1007/embulk-output-kafka)
 
 ## Overview
 
@@ -18,6 +19,8 @@
 - **avsc**: inline avro schema config (json, default: `null`)
 - **ignore_columns**: remove columns from output  (array(string), default: `[]`)
 - **key_column_name**: use column value as record key (string, default: `null`, if this parameter is null, set random number as record key, and it can use column in `ignore_columns`)
+- **partition_column_name**: use column value as partition id (string, default: `null`, this value is prefer to `key_column_name`, and if partition_column value is null, use key_column for partitioning)
+- **column_for_deletion**: Determine to delete (string, default: `null`, `column_for_deletion` column must be boolean. If the value of the column is `true`, KafkaProducer sends `null` value to a Kafka Broker.)
 - **record_batch_size**: kafka producer record batch size (integer, default: `1000`)
 - **acks**: kafka producer require acks (string, default: `"1"`)
 - **retries**: kafka producer max retry count (integer, default: `1`)
@@ -27,7 +30,9 @@
 If use `avro_with_schema_registry` format, following configs are required.
 
 - **schema_registry_url**
-- **avsc** or **avsc_file**
+
+If avsc and avsc_file are null, embulk-output-kafka fetch a schema from schema registry.
+But currently, embulk-output-kafka supports only TopicNameStrategy.
 
 ## Example
 

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "1.5.0"
+    id "com.jfrog.bintray" version "1.8.5"
+    id "com.github.jruby-gradle.base" version "1.6.0"
     id "java"
     id "checkstyle"
 }
@@ -17,26 +17,43 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.1.8"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.9.17"
-    provided "org.embulk:embulk-core:0.9.17"
+    compile  "org.embulk:embulk-core:0.9.22"
+    provided "org.embulk:embulk-core:0.9.22"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
-    testCompile "junit:junit:4.12"
-    testCompile "org.embulk:embulk-test:0.9.17"
-    testCompile "org.embulk:embulk-standards:0.9.17"
+    testCompile "junit:junit:4.13"
+    testCompile "org.embulk:embulk-test:0.9.22"
+    testCompile "org.embulk:embulk-standards:0.9.22"
+    testCompile "org.embulk:embulk-deps-buffer:0.9.22"
 
-    compile "org.apache.kafka:kafka-clients:2.3.0"
-    compile("org.apache.avro:avro:1.9.0") {
+    compile("org.apache.kafka:kafka-clients:2.5.1") {
+        exclude group: "org.slf4j", module: "slf4j-api"
     }
-    compile("io.confluent:kafka-avro-serializer:5.3.0") {
+    compile("org.apache.avro:avro:1.10.0") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+        exclude group: "org.slf4j", module: "slf4j-api"
+    }
+    compile("io.confluent:kafka-avro-serializer:5.5.1") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+        exclude group: "org.slf4j", module: "slf4j-api"
     }
 
-    testCompile("com.github.charithe:kafka-junit:4.1.6") {
+    testCompile "com.salesforce.kafka.test:kafka-junit4:3.+"
+    testCompile("org.apache.kafka:kafka_2.12:2.5.+") {
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-databind"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-annotations"
+        exclude group: "com.fasterxml.jackson.core", module: "jackson-core"
+//        exclude group: "com.fasterxml.jackson.dataformat", module: "jackson-dataformat-csv"
+        exclude group: "com.fasterxml.jackson.datatype", module: "jackson-datatype-jdk8"
     }
 }
 

data/docker-compose.yml

@@ -23,7 +23,7 @@ services:
       KAFKA_BROKER_ID: 1
       KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
       KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://broker:9092
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
 

data/src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java

@@ -14,21 +14,39 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 
-public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
+public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor<GenericRecord>
 {
     private Schema avroSchema;
-    public GenericRecord genericRecord;
+    private GenericRecord genericRecord;
 
-    public AvroFormatColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader, Schema avroSchema, GenericRecord genericRecord)
+    AvroFormatColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader, Schema avroSchema)
     {
         super(task, pageReader);
         this.avroSchema = avroSchema;
-        this.genericRecord = genericRecord;
+    }
+
+    @Override
+    public GenericRecord getRecord()
+    {
+        if (isDeletion()) {
+            return null;
+        }
+
+        return genericRecord;
+    }
+
+    @Override
+    void reset()
+    {
+        super.reset();
+        this.genericRecord = new GenericData.Record(avroSchema);
     }
 
     @Override
     public void booleanColumn(Column column)
     {
+        super.booleanColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -134,9 +152,8 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
         switch (avroSchema.getType()) {
             case ARRAY:
                 if (value.isArrayValue()) {
-                    return value.asArrayValue().list().stream().map(item -> {
-                        return convertMsgPackValueToAvroValue(avroSchema.getElementType(), item);
-                    }).filter(Objects::nonNull).collect(Collectors.toList());
+                    return value.asArrayValue().list().stream().map(item ->
+                            convertMsgPackValueToAvroValue(avroSchema.getElementType(), item)).filter(Objects::nonNull).collect(Collectors.toList());
                 }
                 throw new RuntimeException(String.format("Schema mismatch: avro: %s, msgpack: %s", avroSchema.getType().getName(), value.getValueType().name()));
             case MAP:
@@ -155,9 +172,8 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
                     GenericRecord record = new GenericData.Record(avroSchema);
                     Map<Value, Value> valueMap = value.asMapValue().map();
                     for (org.apache.avro.Schema.Field field : avroSchema.getFields()) {
-                        Optional.ofNullable(valueMap.get(ValueFactory.newString(field.name()))).ifPresent(v -> {
-                            record.put(field.name(), convertMsgPackValueToAvroValue(field.schema(), v));
-                        });
+                        Optional.ofNullable(valueMap.get(ValueFactory.newString(field.name()))).ifPresent(v ->
+                                record.put(field.name(), convertMsgPackValueToAvroValue(field.schema(), v)));
                     }
                     return record;
                 }

data/src/main/java/org/embulk/output/kafka/AvroFormatTransactionalPageOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.output.kafka;
+
+import org.apache.avro.generic.GenericRecord;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.embulk.spi.PageReader;
+
+public class AvroFormatTransactionalPageOutput extends KafkaTransactionalPageOutput<Object, GenericRecord>
+{
+    public AvroFormatTransactionalPageOutput(KafkaProducer<Object, Object> producer, PageReader pageReader, KafkaOutputColumnVisitor<GenericRecord> columnVisitor, String topic, int taskIndex)
+    {
+        super(producer, pageReader, columnVisitor, topic, taskIndex);
+    }
+};

data/src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java

@@ -10,23 +10,41 @@ import org.msgpack.value.Value;
 import java.io.IOException;
 import java.time.format.DateTimeFormatter;
 
-public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
+public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor<ObjectNode>
 {
     private ObjectMapper objectMapper;
-    public ObjectNode jsonNode;
+    private ObjectNode jsonNode;
 
     private static DateTimeFormatter timestampFormatter = DateTimeFormatter.ISO_INSTANT;
 
-    public JsonFormatColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader, ObjectMapper objectMapper)
+    JsonFormatColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader, ObjectMapper objectMapper)
     {
         super(task, pageReader);
         this.objectMapper = objectMapper;
+    }
+
+    @Override
+    public ObjectNode getRecord()
+    {
+        if (isDeletion()) {
+            return null;
+        }
+
+        return jsonNode;
+    }
+
+    @Override
+    void reset()
+    {
+        super.reset();
         this.jsonNode = objectMapper.createObjectNode();
     }
 
     @Override
     public void booleanColumn(Column column)
     {
+        super.booleanColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -42,6 +60,8 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void longColumn(Column column)
     {
+        super.longColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -58,6 +78,8 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void doubleColumn(Column column)
     {
+        super.doubleColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -68,12 +90,13 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
         }
 
         jsonNode.put(column.getName(), pageReader.getDouble(column));
-        super.doubleColumn(column);
     }
 
     @Override
     public void stringColumn(Column column)
     {
+        super.stringColumn(column);
+
         if (isIgnoreColumn(column)) {
             return;
         }
@@ -84,7 +107,6 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
         }
 
         jsonNode.put(column.getName(), pageReader.getString(column));
-        super.stringColumn(column);
     }
 
     @Override

data/src/main/java/org/embulk/output/kafka/JsonFormatTransactionalPageOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.output.kafka;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.apache.kafka.clients.producer.KafkaProducer;
+import org.embulk.spi.PageReader;
+
+public class JsonFormatTransactionalPageOutput extends KafkaTransactionalPageOutput<ObjectNode, ObjectNode>
+{
+    public JsonFormatTransactionalPageOutput(KafkaProducer<Object, ObjectNode> producer, PageReader pageReader, KafkaOutputColumnVisitor<ObjectNode> columnVisitor, String topic, int taskIndex)
+    {
+        super(producer, pageReader, columnVisitor, topic, taskIndex);
+    }
+};

data/src/main/java/org/embulk/output/kafka/KafkaJsonSerializer.java

@@ -12,6 +12,10 @@ public class KafkaJsonSerializer implements Serializer<ObjectNode>
     @Override
     public byte[] serialize(String topic, ObjectNode data)
     {
+        if (data == null) {
+            return null;
+        }
+
         try {
             return objectMapper.writeValueAsBytes(data);
         }

data/src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java

@@ -4,44 +4,98 @@ import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageReader;
 
-public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
+public abstract class KafkaOutputColumnVisitor<T> implements ColumnVisitor
 {
-    KafkaOutputPlugin.PluginTask task;
+    private KafkaOutputPlugin.PluginTask task;
     PageReader pageReader;
+    private String partitionColumnName;
 
-    public Object recordKey = null;
-    public String topicName = null;
+    private Object recordKey = null;
+    private String topicName = null;
+    private Integer partition = null;
+    private boolean deletion = false;
 
-    public KafkaOutputColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader)
+    KafkaOutputColumnVisitor(KafkaOutputPlugin.PluginTask task, PageReader pageReader)
     {
         this.task = task;
         this.pageReader = pageReader;
+        this.partitionColumnName = task.getPartitionColumnName().orElse(null);
     }
 
-    void setRecordKey(Column column, Object value)
+    public abstract T getRecord();
+
+    Object getRecordKey()
+    {
+        return recordKey;
+    }
+
+    private void setRecordKey(Column column, Object value)
     {
         if (task.getKeyColumnName().isPresent() && task.getKeyColumnName().get().equals(column.getName())) {
             recordKey = value;
         }
     }
 
-    void setTopicName(Column column, String value)
+    String getTopicName()
+    {
+        return topicName;
+    }
+
+    private void setTopicName(Column column, String value)
     {
         if (task.getTopicColumn().isPresent() && task.getTopicColumn().get().equals(column.getName())) {
             topicName = value;
         }
     }
 
+    Integer getPartition()
+    {
+        return partition;
+    }
+
+    boolean isDeletion()
+    {
+        return deletion;
+    }
+
+    void reset()
+    {
+        this.recordKey = null;
+        this.topicName = null;
+        this.partition = null;
+        this.deletion = false;
+    }
+
     boolean isIgnoreColumn(Column column)
     {
         return task.getIgnoreColumns().stream().anyMatch(name -> name.equals(column.getName()));
     }
 
+    boolean isColumnForDeletion(Column column)
+    {
+        return task.getColumnForDeletion().map(name -> name.equals(column.getName())).orElse(false);
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (!pageReader.isNull(column)) {
+            if (isColumnForDeletion(column)) {
+                deletion = pageReader.getBoolean(column);
+            }
+        }
+    }
+
     @Override
     public void longColumn(Column column)
     {
         if (!pageReader.isNull(column)) {
-            setRecordKey(column, pageReader.getLong(column));
+            long value = pageReader.getLong(column);
+            setRecordKey(column, value);
+
+            if (partitionColumnName != null && partitionColumnName.equals(column.getName())) {
+                partition = Long.valueOf(value).intValue();
+            }
         }
     }