embulk-output-kafka 0.1.3 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. checksums.yaml +5 -5
  2. data/.circleci/config.yml +44 -0
  3. data/.github/dependabot.yml +11 -0
  4. data/README.md +6 -1
  5. data/build.gradle +29 -12
  6. data/docker-compose.yml +1 -1
  7. data/src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java +26 -10
  8. data/src/main/java/org/embulk/output/kafka/AvroFormatTransactionalPageOutput.java +13 -0
  9. data/src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java +27 -5
  10. data/src/main/java/org/embulk/output/kafka/JsonFormatTransactionalPageOutput.java +13 -0
  11. data/src/main/java/org/embulk/output/kafka/KafkaJsonSerializer.java +4 -0
  12. data/src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java +62 -8
  13. data/src/main/java/org/embulk/output/kafka/KafkaOutputPlugin.java +82 -145
  14. data/src/main/java/org/embulk/output/kafka/KafkaTransactionalPageOutput.java +104 -0
  15. data/src/main/java/org/embulk/output/kafka/RecordProducerFactory.java +3 -3
  16. data/src/test/java/org/embulk/output/kafka/TestKafkaOutputPlugin.java +384 -0
  17. data/src/test/resources/config_complex.yml +9 -28
  18. data/src/test/resources/config_complex_avro.yml +23 -42
  19. data/src/test/resources/config_simple.yml +5 -22
  20. data/src/test/resources/config_simple_avro.yml +14 -32
  21. data/src/test/resources/config_simple_avro_avsc_file.yml +7 -25
  22. data/src/test/resources/config_with_column_for_deletion.yml +7 -0
  23. data/src/test/resources/config_with_column_for_deletion_avro.yml +18 -0
  24. data/src/test/resources/config_with_key_column.yml +6 -23
  25. data/src/test/resources/config_with_partition_column.yml +6 -0
  26. data/src/test/resources/in1.csv +4 -4
  27. data/src/test/resources/in_complex.csv +4 -4
  28. data/src/test/resources/in_with_deletion.csv +4 -0
  29. metadata +30 -24
data/src/test/resources/config_complex.yml CHANGED
@@ -1,28 +1,9 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in_complex
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: "\t"
9
- quote: "\0"
10
- escape: "\0"
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'time', type: timestamp, format: "%Y-%m-%dT%H:%M:%S"}
17
- - {name: 'array', type: json}
18
- - {name: 'data', type: json}
19
- out:
20
- type: kafka
21
- topic: "json-topic"
22
- serialize_format: json
23
- brokers:
24
- - "localhost:9092"
25
- other_producer_configs:
26
- buffer.memory: "67108864"
27
- ignore_columns:
28
- - time
1
+ type: kafka
2
+ topic: "json-complex-topic"
3
+ serialize_format: json
4
+ brokers:
5
+ - "localhost:9092"
6
+ other_producer_configs:
7
+ buffer.memory: "67108864"
8
+ ignore_columns:
9
+ - time
data/src/test/resources/config_complex_avro.yml CHANGED
@@ -1,43 +1,24 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in_complex
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: "\t"
9
- quote: "\0"
10
- escape: "\0"
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'time', type: timestamp, format: "%Y-%m-%dT%H:%M:%S"}
17
- - {name: 'array', type: json}
18
- - {name: 'data', type: json}
19
- out:
20
- type: kafka
21
- topic: "avro-complex-topic"
22
- acks: all
23
- retries: 3
24
- brokers:
25
- - "localhost:9092"
26
- schema_registry_url: "http://localhost:48081/"
27
- serialize_format: avro_with_schema_registry
28
- other_producer_configs:
29
- buffer.memory: "67108864"
30
- avsc:
31
- type: record
32
- name: ComplexRecord
33
- fields: [
34
- {name: "id", type: "string"},
35
- {name: "int_item", type: "long"},
36
- {name: "time", type: "long", logicalType: "timestamp-milli"},
1
+ type: kafka
2
+ topic: "avro-complex-topic"
3
+ acks: all
4
+ retries: 3
5
+ brokers:
6
+ - "localhost:9092"
7
+ schema_registry_url: "mock://embulk-output-kafka"
8
+ serialize_format: avro_with_schema_registry
9
+ other_producer_configs:
10
+ buffer.memory: "67108864"
11
+ avsc:
12
+ type: record
13
+ name: ComplexRecord
14
+ fields: [
15
+ {name: "id", type: "string"},
16
+ {name: "int_item", type: "long"},
17
+ {name: "time", type: "long", logicalType: "timestamp-milli"},
18
+ {name: "array", type: {type: "array", items: "long"}},
19
+ {name: "data", type: {type: "record", name: "InnerData", fields: [
20
+ {name: "hoge", type: "string"},
21
+ {name: "aaa", type: ["null", "string"]},
37
22
  {name: "array", type: {type: "array", items: "long"}},
38
- {name: "data", type: {type: "record", name: "InnerData", fields: [
39
- {name: "hoge", type: "string"},
40
- {name: "aaa", type: ["null", "string"]},
41
- {name: "array", type: {type: "array", items: "long"}},
42
- ]}},
43
- ]
23
+ ]}},
24
+ ]
data/src/test/resources/config_simple.yml CHANGED
@@ -1,22 +1,5 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in1
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: ','
9
- quote: '"'
10
- escape: '"'
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'varchar_item', type: string}
17
- out:
18
- type: kafka
19
- topic: "json-topic"
20
- serialize_format: json
21
- brokers:
22
- - "localhost:9092"
1
+ type: kafka
2
+ topic: "json-topic"
3
+ serialize_format: json
4
+ brokers:
5
+ - "localhost:9092"
data/src/test/resources/config_simple_avro.yml CHANGED
@@ -1,32 +1,14 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in1
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: ','
9
- quote: '"'
10
- escape: '"'
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'varchar_item', type: string}
17
-
18
- out:
19
- type: kafka
20
- topic: "avro-simple-topic"
21
- brokers:
22
- - "localhost:9092"
23
- schema_registry_url: "http://localhost:48081/"
24
- serialize_format: avro_with_schema_registry
25
- avsc:
26
- type: record
27
- name: SimpleRecord
28
- fields: [
29
- {name: "id", type: "string"},
30
- {name: "int_item", type: "long"},
31
- {name: "varchar_item", type: "string"},
32
- ]
1
+ type: kafka
2
+ topic: "avro-simple-topic"
3
+ brokers:
4
+ - "localhost:9092"
5
+ schema_registry_url: "mock://embulk-output-kafka"
6
+ serialize_format: avro_with_schema_registry
7
+ avsc:
8
+ type: record
9
+ name: SimpleRecord
10
+ fields: [
11
+ {name: "id", type: "string"},
12
+ {name: "int_item", type: "long"},
13
+ {name: "varchar_item", type: "string"},
14
+ ]
data/src/test/resources/config_simple_avro_avsc_file.yml CHANGED
@@ -1,25 +1,7 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in1
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: ','
9
- quote: '"'
10
- escape: '"'
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'varchar_item', type: string}
17
-
18
- out:
19
- type: kafka
20
- topic: "avro-simple-topic"
21
- brokers:
22
- - "localhost:9092"
23
- schema_registry_url: "http://localhost:48081/"
24
- serialize_format: avro_with_schema_registry
25
- avsc_file: ./src/test/resources/SimpleRecord.avsc
1
+ type: kafka
2
+ topic: "avro-simple-topic"
3
+ brokers:
4
+ - "localhost:9092"
5
+ schema_registry_url: "mock://embulk-output-kafka"
6
+ serialize_format: avro_with_schema_registry
7
+ avsc_file: ./src/test/resources/SimpleRecord.avsc
data/src/test/resources/config_with_column_for_deletion.yml ADDED
@@ -0,0 +1,7 @@
1
+ type: kafka
2
+ topic: "json-topic"
3
+ serialize_format: json
4
+ brokers:
5
+ - "localhost:9092"
6
+ column_for_deletion: "_delete"
7
+ key_column_name: id
data/src/test/resources/config_with_column_for_deletion_avro.yml ADDED
@@ -0,0 +1,18 @@
1
+ type: kafka
2
+ topic: "avro-simple-topic"
3
+ brokers:
4
+ - "localhost:9092"
5
+ schema_registry_url: "mock://embulk-output-kafka"
6
+ serialize_format: avro_with_schema_registry
7
+ avsc:
8
+ type: record
9
+ name: SimpleRecord
10
+ fields: [
11
+ {name: "id", type: "string"},
12
+ {name: "int_item", type: "long"},
13
+ {name: "varchar_item", type: "string"},
14
+ ]
15
+ key_column_name: id
16
+ column_for_deletion: _delete
17
+ ignore_columns:
18
+ - _delete
data/src/test/resources/config_with_key_column.yml CHANGED
@@ -1,23 +1,6 @@
1
- in:
2
- type: file
3
- path_prefix: ./src/test/resources/in1
4
- parser:
5
- charset: UTF-8
6
- newline: CRLF
7
- type: csv
8
- delimiter: ','
9
- quote: '"'
10
- escape: '"'
11
- null_string: 'NULL'
12
- skip_header_lines: 1
13
- columns:
14
- - {name: 'id', type: string}
15
- - {name: 'int_item', type: long}
16
- - {name: 'varchar_item', type: string}
17
- out:
18
- type: kafka
19
- topic: "json-topic"
20
- serialize_format: json
21
- brokers:
22
- - "localhost:9092"
23
- key_column_name: id
1
+ type: kafka
2
+ topic: "json-topic"
3
+ serialize_format: json
4
+ brokers:
5
+ - "localhost:9092"
6
+ key_column_name: id
data/src/test/resources/config_with_partition_column.yml ADDED
@@ -0,0 +1,6 @@
1
+ type: kafka
2
+ topic: "json-topic"
3
+ serialize_format: json
4
+ brokers:
5
+ - "localhost:9092"
6
+ partition_column_name: int_item
data/src/test/resources/in1.csv CHANGED
@@ -1,4 +1,4 @@
1
- id,int_item,varchar_item
2
- A001,9,a
3
- A002,0,b
4
- A003,9,c
1
+ id:string,int_item:long,varchar_item:string
2
+ "A001","1","a"
3
+ "A002","2","b"
4
+ "A003","3","c"
data/src/test/resources/in_complex.csv CHANGED
@@ -1,5 +1,5 @@
1
- id int_item time array data
2
- A001 9 2018-02-01T12:15:18 [1,2,3] {"hoge": "fuga1", "aaa": "bbb1", "array": [1,2,3]}
3
- A002 0 2018-02-02T12:15:18 [1,2,3] {"hoge": "fuga2", "aaa": null, "array": [4,5,6]}
4
- A003 9 2018-02-03T12:15:18 [1,2,3] {"hoge": "fuga3", "aaa": "bbb3", "array": [7,8,9]}
1
+ id:string,int_item:long,time:timestamp,array:json,data:json
2
+ "A001","9","2018-02-01 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga1"", ""aaa"": ""bbb1"", ""array"": [1,2,3]}"
3
+ "A002","0","2018-02-02 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga2"", ""aaa"": null, ""array"": [4,5,6]}"
4
+ "A003","9","2018-02-03 12:15:18.000 +0000","[1,2,3]","{""hoge"": ""fuga3"", ""aaa"": ""bbb3"", ""array"": [7,8,9]}"
5
5
 
data/src/test/resources/in_with_deletion.csv ADDED
@@ -0,0 +1,4 @@
1
+ id:string,int_item:long,varchar_item:string,_delete:boolean
2
+ "A001","1","a","false"
3
+ "A002","2","b","true"
4
+ "A003","3","c","false"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-kafka
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.1.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - joker1007
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-09-13 00:00:00.000000000 Z
11
+ date: 2020-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -45,32 +45,31 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
+ - ".circleci/config.yml"
49
+ - ".github/dependabot.yml"
48
50
  - ".gitignore"
49
51
  - LICENSE.txt
50
52
  - README.md
51
53
  - build.gradle
52
- - classpath/audience-annotations-0.5.0.jar
53
- - classpath/avro-1.9.0.jar
54
- - classpath/common-config-5.3.0.jar
55
- - classpath/common-utils-5.3.0.jar
56
- - classpath/commons-compress-1.18.jar
57
- - classpath/embulk-output-kafka-0.1.3.jar
58
- - classpath/jackson-annotations-2.9.0.jar
59
- - classpath/jackson-core-2.9.9.jar
60
- - classpath/jackson-databind-2.9.9.jar
61
- - classpath/jline-0.9.94.jar
62
- - classpath/jsr305-3.0.2.jar
63
- - classpath/kafka-avro-serializer-5.3.0.jar
64
- - classpath/kafka-clients-5.3.0-ccs.jar
65
- - classpath/kafka-schema-registry-client-5.3.0.jar
66
- - classpath/lz4-java-1.6.0.jar
67
- - classpath/netty-3.10.6.Final.jar
68
- - classpath/slf4j-api-1.7.26.jar
54
+ - classpath/avro-1.10.0.jar
55
+ - classpath/common-config-5.5.1.jar
56
+ - classpath/common-utils-5.5.1.jar
57
+ - classpath/commons-compress-1.20.jar
58
+ - classpath/embulk-output-kafka-0.1.8.jar
59
+ - classpath/jakarta.annotation-api-1.3.5.jar
60
+ - classpath/jakarta.inject-2.6.1.jar
61
+ - classpath/jakarta.ws.rs-api-2.1.6.jar
62
+ - classpath/javax.ws.rs-api-2.1.1.jar
63
+ - classpath/jersey-common-2.30.jar
64
+ - classpath/kafka-avro-serializer-5.5.1.jar
65
+ - classpath/kafka-clients-5.5.1-ccs.jar
66
+ - classpath/kafka-schema-registry-client-5.5.1.jar
67
+ - classpath/kafka-schema-serializer-5.5.1.jar
68
+ - classpath/lz4-java-1.7.1.jar
69
+ - classpath/osgi-resource-locator-1.0.3.jar
69
70
  - classpath/snappy-java-1.1.7.3.jar
70
- - classpath/spotbugs-annotations-3.1.9.jar
71
- - classpath/zkclient-0.10.jar
72
- - classpath/zookeeper-3.4.14.jar
73
- - classpath/zstd-jni-1.4.0-1.jar
71
+ - classpath/swagger-annotations-1.6.0.jar
72
+ - classpath/zstd-jni-1.4.4-7.jar
74
73
  - config/checkstyle/checkstyle.xml
75
74
  - config/checkstyle/default.xml
76
75
  - docker-compose.yml
@@ -80,10 +79,13 @@ files:
80
79
  - gradlew.bat
81
80
  - lib/embulk/output/kafka.rb
82
81
  - src/main/java/org/embulk/output/kafka/AvroFormatColumnVisitor.java
82
+ - src/main/java/org/embulk/output/kafka/AvroFormatTransactionalPageOutput.java
83
83
  - src/main/java/org/embulk/output/kafka/JsonFormatColumnVisitor.java
84
+ - src/main/java/org/embulk/output/kafka/JsonFormatTransactionalPageOutput.java
84
85
  - src/main/java/org/embulk/output/kafka/KafkaJsonSerializer.java
85
86
  - src/main/java/org/embulk/output/kafka/KafkaOutputColumnVisitor.java
86
87
  - src/main/java/org/embulk/output/kafka/KafkaOutputPlugin.java
88
+ - src/main/java/org/embulk/output/kafka/KafkaTransactionalPageOutput.java
87
89
  - src/main/java/org/embulk/output/kafka/RecordProducerFactory.java
88
90
  - src/test/java/org/embulk/output/kafka/TestKafkaOutputPlugin.java
89
91
  - src/test/resources/SimpleRecord.avsc
@@ -92,9 +94,13 @@ files:
92
94
  - src/test/resources/config_simple.yml
93
95
  - src/test/resources/config_simple_avro.yml
94
96
  - src/test/resources/config_simple_avro_avsc_file.yml
97
+ - src/test/resources/config_with_column_for_deletion.yml
98
+ - src/test/resources/config_with_column_for_deletion_avro.yml
95
99
  - src/test/resources/config_with_key_column.yml
100
+ - src/test/resources/config_with_partition_column.yml
96
101
  - src/test/resources/in1.csv
97
102
  - src/test/resources/in_complex.csv
103
+ - src/test/resources/in_with_deletion.csv
98
104
  homepage: https://github.com/joker1007/embulk-output-kafka
99
105
  licenses:
100
106
  - MIT
@@ -115,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
115
121
  version: '0'
116
122
  requirements: []
117
123
  rubyforge_project:
118
- rubygems_version: 2.6.8
124
+ rubygems_version: 2.6.13
119
125
  signing_key:
120
126
  specification_version: 4
121
127
  summary: Kafka output plugin for Embulk