embulk-output-kafka 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 55e78aa9c5aec3c15768aed48208cd4b2023c1cd
-  data.tar.gz: fbdac9757f935083ab553d44fd7bfc82a59e84a2
+  metadata.gz: feda9642eebe84a68dc4ff9a3f3e2b297e2e94f0
+  data.tar.gz: 4cdb1d310d6e1ae9f4caa5102465311c963f2656
 SHA512:
-  metadata.gz: 87a3889c7e9bcd881301585269086656c479ea8c6c8dc60a03a79f4a13a4d18d186a1f7e0eca5f0a9de002e0adfb20262306f0782a478f8d888642f25a12e33d
-  data.tar.gz: a6e44be4d25da3fdf8bd5e836d6ce4e64effaf07dc5d1f860999a1aad661d6edbf0dcc66faa69666b66e988c3c57541cf8d4c8d10396b364641f7bd2591bd265
+  metadata.gz: ab050ce4361ef325f2dcb9a07dce6135c3735d9b65c3820ea84385bbec6f8d3f7ea810f155df4477af43f56bf75b0bb5dd82d90ef5cd911da1932ab5a9637328
+  data.tar.gz: a661ff3eb73bb171f3d04a5699aaf390fa7b0ea8b2dad5feb9cf5420237ae6f5b692011d993bb002d77981544abff890f37a7fc40308a3708e1cbfd0d57b1a1e

data/README.md

@@ -9,14 +9,15 @@
 
 ## Configuration
 
-- **broker**: kafka broker host and port (array<string>, required)
+- **broker**: kafka broker host and port (array(string), required)
 - **topic**: target topic name (string, required)
 - **topic_column**: use column value as target topic (string, default: `null`)
 - **schema_registry_url**: Schema Registy URL that is needed for avro format (string, default: `null`)
 - **serialize_format**: use column value as target topic (enum, required, `json` or `avro_with_schema_registry`)
 - **avsc_file**: avro schema file path (string, default: `null`)
 - **avsc**: inline avro schema config (json, default: `null`)
-- **key_column_name**: use column value as record key (string, default: `null`, if this parameter is null, set random number as record key)
+- **ignore_columns**: remove columns from output  (array(string), default: `[]`)
+- **key_column_name**: use column value as record key (string, default: `null`, if this parameter is null, set random number as record key, and it can use column in `ignore_columns`)
 - **record_batch_size**: kafka producer record batch size (integer, default: `1000`)
 - **acks**: kafka producer require acks (string, default: `"1"`)
 - **retries**: kafka producer max retry count (integer, default: `1`)

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/docker-compose.yml

@@ -0,0 +1,41 @@
+version: "3"
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:5.3.0
+    hostname: zookeeper
+    container_name: zookeeper
+    ports:
+      - "2181:2181"
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+
+  broker:
+    image: confluentinc/cp-kafka:5.3.0
+    hostname: broker
+    container_name: broker
+    depends_on:
+      - zookeeper
+    ports:
+      - "29092:29092"
+      - "9092:9092"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://broker:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+
+  schema-registry:
+    image: confluentinc/cp-schema-registry:5.3.0
+    hostname: schema-registry
+    container_name: schema-registry
+    depends_on:
+      - zookeeper
+      - broker
+    ports:
+      - "48081:8081"
+    environment:
+      SCHEMA_REGISTRY_HOST_NAME: schema-registry
+      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'

data/src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java

@@ -29,6 +29,10 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;
@@ -40,42 +44,61 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void longColumn(Column column)
     {
+        super.longColumn(column);
+
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;
         }
 
         genericRecord.put(column.getName(), pageReader.getLong(column));
-        super.longColumn(column);
     }
 
     @Override
     public void doubleColumn(Column column)
     {
+        super.doubleColumn(column);
+
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;
         }
 
         genericRecord.put(column.getName(), pageReader.getDouble(column));
-        super.doubleColumn(column);
     }
 
     @Override
     public void stringColumn(Column column)
     {
+        super.stringColumn(column);
+
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;
         }
 
         genericRecord.put(column.getName(), pageReader.getString(column));
-        super.stringColumn(column);
     }
 
     @Override
     public void timestampColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;
@@ -87,6 +110,10 @@ public class AvroFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void jsonColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             genericRecord.put(column.getName(), null);
             return;

data/src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java

@@ -27,6 +27,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void booleanColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;
@@ -38,6 +42,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void longColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;
@@ -50,6 +58,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void doubleColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;
@@ -62,6 +74,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void stringColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;
@@ -74,6 +90,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void timestampColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;
@@ -85,6 +105,10 @@ public class JsonFormatColumnVisitor extends KafkaOutputColumnVisitor
     @Override
     public void jsonColumn(Column column)
     {
+        if (isIgnoreColumn(column)) {
+            return;
+        }
+
         if (pageReader.isNull(column)) {
             jsonNode.putNull(column.getName());
             return;

data/src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java

@@ -32,22 +32,33 @@ public abstract class KafkaOutputColumnVisitor implements ColumnVisitor
         }
     }
 
+    boolean isIgnoreColumn(Column column)
+    {
+        return task.getIgnoreColumns().stream().anyMatch(name -> name.equals(column.getName()));
+    }
+
     @Override
     public void longColumn(Column column)
     {
-        setRecordKey(column, pageReader.getLong(column));
+        if (!pageReader.isNull(column)) {
+            setRecordKey(column, pageReader.getLong(column));
+        }
     }
 
     @Override
     public void doubleColumn(Column column)
     {
-        setRecordKey(column, pageReader.getDouble(column));
+        if (!pageReader.isNull(column)) {
+            setRecordKey(column, pageReader.getDouble(column));
+        }
     }
 
     @Override
     public void stringColumn(Column column)
     {
-        setRecordKey(column, pageReader.getString(column));
-        setTopicName(column, pageReader.getString(column));
+        if (!pageReader.isNull(column)) {
+            setRecordKey(column, pageReader.getString(column));
+            setTopicName(column, pageReader.getString(column));
+        }
     }
 }

data/src/main/java/org/embulk/output/kafka/KafkaOutputPlugin.java

@@ -15,6 +15,7 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
@@ -110,11 +111,14 @@ public class KafkaOutputPlugin
         @Config("other_producer_configs")
         @ConfigDefault("{}")
         public Map<String, String> getOtherProducerConfigs();
+
+        @Config("ignore_columns")
+        @ConfigDefault("[]")
+        public List<String> getIgnoreColumns();
     }
 
     private static ObjectMapper objectMapper = new ObjectMapper();
     private Logger logger = LoggerFactory.getLogger(getClass());
-    private int recordLoggingCount = 1;
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
@@ -168,6 +172,7 @@ public class KafkaOutputPlugin
         PageReader pageReader = new PageReader(schema);
         PrimitiveIterator.OfLong randomLong = new Random().longs(1, Long.MAX_VALUE).iterator();
         AtomicInteger counter = new AtomicInteger(0);
+        AtomicInteger recordLoggingCount = new AtomicInteger(1);
 
         return new TransactionalPageOutput() {
             @Override
@@ -194,9 +199,9 @@ public class KafkaOutputPlugin
                         logger.debug("sent record: {key: {}, value: {}}", producerRecord.key(), producerRecord.value());
 
                         int current = counter.incrementAndGet();
-                        if (current >= recordLoggingCount) {
+                        if (current >= recordLoggingCount.get()) {
                             logger.info("[task-{}] Producer sent {} records", String.format("%04d", taskIndex), current);
-                            recordLoggingCount = recordLoggingCount * 2;
+                            recordLoggingCount.set(recordLoggingCount.get() * 2);
                         }
                     });
                 }
@@ -255,6 +260,7 @@ public class KafkaOutputPlugin
         PrimitiveIterator.OfLong randomLong = new Random().longs(1, Long.MAX_VALUE).iterator();
 
         AtomicInteger counter = new AtomicInteger(0);
+        AtomicInteger recordLoggingCount = new AtomicInteger(1);
 
         final org.apache.avro.Schema finalAvroSchema = avroSchema;
         return new TransactionalPageOutput()
@@ -284,9 +290,9 @@ public class KafkaOutputPlugin
                         logger.debug("sent record: {key: {}, value: {}}", producerRecord.key(), producerRecord.value());
 
                         int current = counter.incrementAndGet();
-                        if (current >= recordLoggingCount) {
+                        if (current >= recordLoggingCount.get()) {
                             logger.info("[task-{}] Producer sent {} records", String.format("%04d", taskIndex), current);
-                            recordLoggingCount = recordLoggingCount * 2;
+                            recordLoggingCount.set(recordLoggingCount.get() * 2);
                         }
                     });
                 }

data/src/test/resources/config_complex.yml

@@ -24,3 +24,5 @@ out:
     - "localhost:9092"
   other_producer_configs:
     buffer.memory: "67108864"
+  ignore_columns:
+    - time

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-12 00:00:00.000000000 Z
+date: 2019-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -54,7 +54,7 @@ files:
 - classpath/common-config-5.3.0.jar
 - classpath/common-utils-5.3.0.jar
 - classpath/commons-compress-1.18.jar
-- classpath/embulk-output-kafka-0.1.1.jar
+- classpath/embulk-output-kafka-0.1.2.jar
 - classpath/jackson-annotations-2.9.0.jar
 - classpath/jackson-core-2.9.9.jar
 - classpath/jackson-databind-2.9.9.jar
@@ -73,6 +73,7 @@ files:
 - classpath/zstd-jni-1.4.0-1.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- docker-compose.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew