embulk-output-kafka 0.1.3 → 0.1.8

data/src/test/resources/config_complex.yml

@@ -1,28 +1,9 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in_complex
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: "\t"
-    quote: "\0"
-    escape: "\0"
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'time', type: timestamp, format: "%Y-%m-%dT%H:%M:%S"}
-    - {name: 'array', type: json}
-    - {name: 'data', type: json}
-out:
-  type: kafka
-  topic: "json-topic"
-  serialize_format: json
-  brokers:
-    - "localhost:9092"
-  other_producer_configs:
-    buffer.memory: "67108864"
-  ignore_columns:
-    - time
+type: kafka
+topic: "json-complex-topic"
+serialize_format: json
+brokers:
+  - "localhost:9092"
+other_producer_configs:
+  buffer.memory: "67108864"
+ignore_columns:
+  - time

data/src/test/resources/config_complex_avro.yml

@@ -1,43 +1,24 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in_complex
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: "\t"
-    quote: "\0"
-    escape: "\0"
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'time', type: timestamp, format: "%Y-%m-%dT%H:%M:%S"}
-    - {name: 'array', type: json}
-    - {name: 'data', type: json}
-out:
-  type: kafka
-  topic: "avro-complex-topic"
-  acks: all
-  retries: 3
-  brokers:
-    - "localhost:9092"
-  schema_registry_url: "http://localhost:48081/"
-  serialize_format: avro_with_schema_registry
-  other_producer_configs:
-    buffer.memory: "67108864"
-  avsc:
-    type: record
-    name: ComplexRecord
-    fields: [
-      {name: "id", type: "string"},
-      {name: "int_item", type: "long"},
-      {name: "time", type: "long", logicalType: "timestamp-milli"},
+type: kafka
+topic: "avro-complex-topic"
+acks: all
+retries: 3
+brokers:
+  - "localhost:9092"
+schema_registry_url: "mock://embulk-output-kafka"
+serialize_format: avro_with_schema_registry
+other_producer_configs:
+  buffer.memory: "67108864"
+avsc:
+  type: record
+  name: ComplexRecord
+  fields: [
+    {name: "id", type: "string"},
+    {name: "int_item", type: "long"},
+    {name: "time", type: "long", logicalType: "timestamp-milli"},
+    {name: "array", type: {type: "array", items: "long"}},
+    {name: "data", type: {type: "record", name: "InnerData", fields: [
+      {name: "hoge", type: "string"},
+      {name: "aaa", type: ["null", "string"]},
       {name: "array", type: {type: "array", items: "long"}},
-      {name: "data", type: {type: "record", name: "InnerData", fields: [
-        {name: "hoge", type: "string"},
-        {name: "aaa", type: ["null", "string"]},
-        {name: "array", type: {type: "array", items: "long"}},
-      ]}},
-    ]
+    ]}},
+  ]

data/src/test/resources/config_simple.yml

@@ -1,22 +1,5 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in1
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: ','
-    quote: '"'
-    escape: '"'
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'varchar_item', type: string}
-out:
-  type: kafka
-  topic: "json-topic"
-  serialize_format: json
-  brokers:
-    - "localhost:9092"
+type: kafka
+topic: "json-topic"
+serialize_format: json
+brokers:
+  - "localhost:9092"

data/src/test/resources/config_simple_avro.yml

@@ -1,32 +1,14 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in1
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: ','
-    quote: '"'
-    escape: '"'
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'varchar_item', type: string}
-
-out:
-  type: kafka
-  topic: "avro-simple-topic"
-  brokers:
-    - "localhost:9092"
-  schema_registry_url: "http://localhost:48081/"
-  serialize_format: avro_with_schema_registry
-  avsc:
-    type: record
-    name: SimpleRecord
-    fields: [
-      {name: "id", type: "string"},
-      {name: "int_item", type: "long"},
-      {name: "varchar_item", type: "string"},
-    ]
+type: kafka
+topic: "avro-simple-topic"
+brokers:
+  - "localhost:9092"
+schema_registry_url: "mock://embulk-output-kafka"
+serialize_format: avro_with_schema_registry
+avsc:
+  type: record
+  name: SimpleRecord
+  fields: [
+    {name: "id", type: "string"},
+    {name: "int_item", type: "long"},
+    {name: "varchar_item", type: "string"},
+  ]

data/src/test/resources/config_simple_avro_avsc_file.yml

@@ -1,25 +1,7 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in1
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: ','
-    quote: '"'
-    escape: '"'
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'varchar_item', type: string}
-
-out:
-  type: kafka
-  topic: "avro-simple-topic"
-  brokers:
-    - "localhost:9092"
-  schema_registry_url: "http://localhost:48081/"
-  serialize_format: avro_with_schema_registry
-  avsc_file: ./src/test/resources/SimpleRecord.avsc
+type: kafka
+topic: "avro-simple-topic"
+brokers:
+  - "localhost:9092"
+schema_registry_url: "mock://embulk-output-kafka"
+serialize_format: avro_with_schema_registry
+avsc_file: ./src/test/resources/SimpleRecord.avsc

data/src/test/resources/config_with_column_for_deletion.yml

@@ -0,0 +1,7 @@
+type: kafka
+topic: "json-topic"
+serialize_format: json
+brokers:
+  - "localhost:9092"
+column_for_deletion: "_delete"
+key_column_name: id

data/src/test/resources/config_with_column_for_deletion_avro.yml

@@ -0,0 +1,18 @@
+type: kafka
+topic: "avro-simple-topic"
+brokers:
+  - "localhost:9092"
+schema_registry_url: "mock://embulk-output-kafka"
+serialize_format: avro_with_schema_registry
+avsc:
+  type: record
+  name: SimpleRecord
+  fields: [
+    {name: "id", type: "string"},
+    {name: "int_item", type: "long"},
+    {name: "varchar_item", type: "string"},
+  ]
+key_column_name: id
+column_for_deletion: _delete
+ignore_columns:
+  - _delete

data/src/test/resources/config_with_key_column.yml

@@ -1,23 +1,6 @@
-in:
-  type: file
-  path_prefix: ./src/test/resources/in1
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: ','
-    quote: '"'
-    escape: '"'
-    null_string: 'NULL'
-    skip_header_lines: 1
-    columns:
-    - {name: 'id', type: string}
-    - {name: 'int_item', type: long}
-    - {name: 'varchar_item', type: string}
-out:
-  type: kafka
-  topic: "json-topic"
-  serialize_format: json
-  brokers:
-    - "localhost:9092"
-  key_column_name: id
+type: kafka
+topic: "json-topic"
+serialize_format: json
+brokers:
+  - "localhost:9092"
+key_column_name: id

data/src/test/resources/config_with_partition_column.yml

@@ -0,0 +1,6 @@
+type: kafka
+topic: "json-topic"
+serialize_format: json
+brokers:
+  - "localhost:9092"
+partition_column_name: int_item

data/src/test/resources/in1.csv

@@ -1,4 +1,4 @@
-id,int_item,varchar_item
-A001,9,a
-A002,0,b
-A003,9,c
+id:string,int_item:long,varchar_item:string
+"A001","1","a"
+"A002","2","b"
+"A003","3","c"

data/src/test/resources/in_complex.csv

@@ -1,5 +1,5 @@
-id	int_item	time	array	data
-A001	9	2018-02-01T12:15:18	[1,2,3]	{"hoge": "fuga1", "aaa": "bbb1", "array": [1,2,3]}
-A002	0	2018-02-02T12:15:18	[1,2,3]	{"hoge": "fuga2", "aaa": null, "array": [4,5,6]}
-A003	9	2018-02-03T12:15:18	[1,2,3]	{"hoge": "fuga3", "aaa": "bbb3", "array": [7,8,9]}
+id:string,int_item:long,time:timestamp,array:json,data:json
+"A001","9","2018-02-01 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga1"", ""aaa"": ""bbb1"", ""array"": [1,2,3]}"
+"A002","0","2018-02-02 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga2"", ""aaa"": null, ""array"": [4,5,6]}"
+"A003","9","2018-02-03 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga3"", ""aaa"": ""bbb3"", ""array"": [7,8,9]}"
 

data/src/test/resources/in_with_deletion.csv

@@ -0,0 +1,4 @@
+id:string,int_item:long,varchar_item:string,_delete:boolean
+"A001","1","a","false"
+"A002","2","b","true"
+"A003","3","c","false"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.8
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-13 00:00:00.000000000 Z
+date: 2020-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -45,32 +45,31 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".circleci/config.yml"
+- ".github/dependabot.yml"
 - ".gitignore"
 - LICENSE.txt
 - README.md
 - build.gradle
-- classpath/audience-annotations-0.5.0.jar
-- classpath/avro-1.9.0.jar
-- classpath/common-config-5.3.0.jar
-- classpath/common-utils-5.3.0.jar
-- classpath/commons-compress-1.18.jar
-- classpath/embulk-output-kafka-0.1.3.jar
-- classpath/jackson-annotations-2.9.0.jar
-- classpath/jackson-core-2.9.9.jar
-- classpath/jackson-databind-2.9.9.jar
-- classpath/jline-0.9.94.jar
-- classpath/jsr305-3.0.2.jar
-- classpath/kafka-avro-serializer-5.3.0.jar
-- classpath/kafka-clients-5.3.0-ccs.jar
-- classpath/kafka-schema-registry-client-5.3.0.jar
-- classpath/lz4-java-1.6.0.jar
-- classpath/netty-3.10.6.Final.jar
-- classpath/slf4j-api-1.7.26.jar
+- classpath/avro-1.10.0.jar
+- classpath/common-config-5.5.1.jar
+- classpath/common-utils-5.5.1.jar
+- classpath/commons-compress-1.20.jar
+- classpath/embulk-output-kafka-0.1.8.jar
+- classpath/jakarta.annotation-api-1.3.5.jar
+- classpath/jakarta.inject-2.6.1.jar
+- classpath/jakarta.ws.rs-api-2.1.6.jar
+- classpath/javax.ws.rs-api-2.1.1.jar
+- classpath/jersey-common-2.30.jar
+- classpath/kafka-avro-serializer-5.5.1.jar
+- classpath/kafka-clients-5.5.1-ccs.jar
+- classpath/kafka-schema-registry-client-5.5.1.jar
+- classpath/kafka-schema-serializer-5.5.1.jar
+- classpath/lz4-java-1.7.1.jar
+- classpath/osgi-resource-locator-1.0.3.jar
 - classpath/snappy-java-1.1.7.3.jar
-- classpath/spotbugs-annotations-3.1.9.jar
-- classpath/zkclient-0.10.jar
-- classpath/zookeeper-3.4.14.jar
-- classpath/zstd-jni-1.4.0-1.jar
+- classpath/swagger-annotations-1.6.0.jar
+- classpath/zstd-jni-1.4.4-7.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - docker-compose.yml
@@ -80,10 +79,13 @@ files:
 - gradlew.bat
 - lib/embulk/output/kafka.rb
 - src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java
+- src/main/java/org/embulk/output/kafka/AvroFormatTransactionalPageOutput.java
 - src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java
+- src/main/java/org/embulk/output/kafka/JsonFormatTransactionalPageOutput.java
 - src/main/java/org/embulk/output/kafka/KafkaJsonSerializer.java
 - src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java
 - src/main/java/org/embulk/output/kafka/KafkaOutputPlugin.java
+- src/main/java/org/embulk/output/kafka/KafkaTransactionalPageOutput.java
 - src/main/java/org/embulk/output/kafka/RecordProducerFactory.java
 - src/test/java/org/embulk/output/kafka/TestKafkaOutputPlugin.java
 - src/test/resources/SimpleRecord.avsc
@@ -92,9 +94,13 @@ files:
 - src/test/resources/config_simple.yml
 - src/test/resources/config_simple_avro.yml
 - src/test/resources/config_simple_avro_avsc_file.yml
+- src/test/resources/config_with_column_for_deletion.yml
+- src/test/resources/config_with_column_for_deletion_avro.yml
 - src/test/resources/config_with_key_column.yml
+- src/test/resources/config_with_partition_column.yml
 - src/test/resources/in1.csv
 - src/test/resources/in_complex.csv
+- src/test/resources/in_with_deletion.csv
 homepage: https://github.com/joker1007/embulk-output-kafka
 licenses:
 - MIT
@@ -115,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Kafka output plugin for Embulk