ruby-kafka 0.5.2.beta1 → 0.5.2.beta2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +78 -3
- data/CHANGELOG.md +1 -0
- data/lib/kafka/compression.rb +5 -1
- data/lib/kafka/gzip_codec.rb +4 -4
- data/lib/kafka/lz4_codec.rb +5 -5
- data/lib/kafka/protocol/message.rb +6 -7
- data/lib/kafka/protocol/message_set.rb +1 -2
- data/lib/kafka/snappy_codec.rb +5 -5
- data/lib/kafka/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eed57dde0ac66ebd2434a84f198ae88654b0a792
|
4
|
+
data.tar.gz: 6da001e90557c6633c930db1e8c8c27eda062556
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7dc72dec0637c11e5f18f4fb4b90b2384c6ac0150069fb7c2c03e56a43b27e023d07e6bd7f01141131dabe83735a8d3788de765bab3dc59d6db263f23e39859f
|
7
|
+
data.tar.gz: 96895ff990a1dfa40a0487811b8abee3ff00a560b3a1beadd6e3835ba6343b2a7c70305df74da002476f2d0dad775304e75ddd6587121ea1cfd90f53a82fb412
|
data/.circleci/config.yml
CHANGED
@@ -1,6 +1,17 @@
|
|
1
1
|
version: 2
|
2
2
|
jobs:
|
3
|
-
|
3
|
+
unit:
|
4
|
+
docker:
|
5
|
+
- image: circleci/ruby:2.4.1-node
|
6
|
+
environment:
|
7
|
+
LOG_LEVEL: DEBUG
|
8
|
+
steps:
|
9
|
+
- checkout
|
10
|
+
- run: bundle install --path vendor/bundle
|
11
|
+
- run: bundle exec rspec
|
12
|
+
- run: bundle exec rubocop
|
13
|
+
|
14
|
+
kafka-0.10:
|
4
15
|
docker:
|
5
16
|
- image: circleci/ruby:2.4.1-node
|
6
17
|
environment:
|
@@ -24,10 +35,74 @@ jobs:
|
|
24
35
|
KAFKA_ADVERTISED_PORT: 9094
|
25
36
|
KAFKA_PORT: 9094
|
26
37
|
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
38
|
+
steps:
|
39
|
+
- checkout
|
40
|
+
- run: bundle install --path vendor/bundle
|
41
|
+
- run: bundle exec rspec --profile --tag functional spec/functional
|
42
|
+
|
43
|
+
kafka-0.11:
|
44
|
+
docker:
|
45
|
+
- image: circleci/ruby:2.4.1-node
|
46
|
+
environment:
|
47
|
+
LOG_LEVEL: DEBUG
|
48
|
+
- image: wurstmeister/zookeeper
|
49
|
+
- image: wurstmeister/kafka:0.11.0.1
|
50
|
+
environment:
|
51
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
52
|
+
KAFKA_ADVERTISED_PORT: 9092
|
53
|
+
KAFKA_PORT: 9092
|
54
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
55
|
+
- image: wurstmeister/kafka:0.11.0.1
|
56
|
+
environment:
|
57
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
58
|
+
KAFKA_ADVERTISED_PORT: 9093
|
59
|
+
KAFKA_PORT: 9093
|
60
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
61
|
+
- image: wurstmeister/kafka:0.11.0.1
|
62
|
+
environment:
|
63
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
64
|
+
KAFKA_ADVERTISED_PORT: 9094
|
65
|
+
KAFKA_PORT: 9094
|
66
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
67
|
+
steps:
|
68
|
+
- checkout
|
69
|
+
- run: bundle install --path vendor/bundle
|
70
|
+
- run: bundle exec rspec --profile --tag functional spec/functional
|
27
71
|
|
72
|
+
kafka-1.0:
|
73
|
+
docker:
|
74
|
+
- image: circleci/ruby:2.4.1-node
|
75
|
+
environment:
|
76
|
+
LOG_LEVEL: DEBUG
|
77
|
+
- image: wurstmeister/zookeeper
|
78
|
+
- image: wurstmeister/kafka:1.0.0
|
79
|
+
environment:
|
80
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
81
|
+
KAFKA_ADVERTISED_PORT: 9092
|
82
|
+
KAFKA_PORT: 9092
|
83
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
84
|
+
- image: wurstmeister/kafka:1.0.0
|
85
|
+
environment:
|
86
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
87
|
+
KAFKA_ADVERTISED_PORT: 9093
|
88
|
+
KAFKA_PORT: 9093
|
89
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
90
|
+
- image: wurstmeister/kafka:1.0.0
|
91
|
+
environment:
|
92
|
+
KAFKA_ADVERTISED_HOST_NAME: localhost
|
93
|
+
KAFKA_ADVERTISED_PORT: 9094
|
94
|
+
KAFKA_PORT: 9094
|
95
|
+
KAFKA_ZOOKEEPER_CONNECT: localhost:2181
|
28
96
|
steps:
|
29
97
|
- checkout
|
30
98
|
- run: bundle install --path vendor/bundle
|
31
|
-
- run: bundle exec rspec
|
32
99
|
- run: bundle exec rspec --profile --tag functional spec/functional
|
33
|
-
|
100
|
+
|
101
|
+
workflows:
|
102
|
+
version: 2
|
103
|
+
test:
|
104
|
+
jobs:
|
105
|
+
- unit
|
106
|
+
- kafka-0.10
|
107
|
+
- kafka-0.11
|
108
|
+
- kafka-1.0
|
data/CHANGELOG.md
CHANGED
@@ -10,6 +10,7 @@ Changes and additions to the library will be listed here.
|
|
10
10
|
- Add support for LZ4 compression (#499).
|
11
11
|
- Refactor compression codec lookup (#509).
|
12
12
|
- Fix compressed message set offset bug (#506).
|
13
|
+
- Test against multiple versions of Kafka.
|
13
14
|
|
14
15
|
## v0.5.1
|
15
16
|
|
data/lib/kafka/compression.rb
CHANGED
data/lib/kafka/gzip_codec.rb
CHANGED
data/lib/kafka/lz4_codec.rb
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
module Kafka
|
2
2
|
class LZ4Codec
|
3
|
-
def
|
3
|
+
def codec_id
|
4
|
+
3
|
5
|
+
end
|
6
|
+
|
7
|
+
def load
|
4
8
|
require "extlz4"
|
5
9
|
rescue LoadError
|
6
10
|
raise LoadError, "using lz4 compression requires adding a dependency on the `extlz4` gem to your Gemfile."
|
7
11
|
end
|
8
12
|
|
9
|
-
def codec_id
|
10
|
-
3
|
11
|
-
end
|
12
|
-
|
13
13
|
def compress(data)
|
14
14
|
LZ4.encode(data)
|
15
15
|
end
|
@@ -58,7 +58,7 @@ module Kafka
|
|
58
58
|
message_set_decoder = Decoder.from_string(data)
|
59
59
|
message_set = MessageSet.decode(message_set_decoder)
|
60
60
|
|
61
|
-
correct_offsets(message_set)
|
61
|
+
correct_offsets(message_set.messages)
|
62
62
|
end
|
63
63
|
|
64
64
|
def self.decode(decoder)
|
@@ -111,15 +111,16 @@ module Kafka
|
|
111
111
|
# All other cases, compressed inner messages should have relative offset, with below attributes:
|
112
112
|
# - The container message should have the 'real' offset
|
113
113
|
# - The container message's offset should be the 'real' offset of the last message in the compressed batch
|
114
|
-
def correct_offsets(
|
115
|
-
max_relative_offset =
|
114
|
+
def correct_offsets(messages)
|
115
|
+
max_relative_offset = messages.last.offset
|
116
116
|
|
117
117
|
# The offsets are already correct, do nothing.
|
118
|
-
return
|
118
|
+
return messages if max_relative_offset == offset
|
119
119
|
|
120
120
|
# The contained messages have relative offsets, and needs to be corrected.
|
121
121
|
base_offset = offset - max_relative_offset
|
122
|
-
|
122
|
+
|
123
|
+
messages.map do |message|
|
123
124
|
Message.new(
|
124
125
|
offset: message.offset + base_offset,
|
125
126
|
value: message.value,
|
@@ -128,8 +129,6 @@ module Kafka
|
|
128
129
|
codec_id: message.codec_id
|
129
130
|
)
|
130
131
|
end
|
131
|
-
|
132
|
-
MessageSet.new(messages: messages)
|
133
132
|
end
|
134
133
|
|
135
134
|
def encode_with_crc
|
@@ -31,8 +31,7 @@ module Kafka
|
|
31
31
|
message = Message.decode(decoder)
|
32
32
|
|
33
33
|
if message.compressed?
|
34
|
-
|
35
|
-
fetched_messages.concat(wrapped_message_set.messages)
|
34
|
+
fetched_messages.concat(message.decompress)
|
36
35
|
else
|
37
36
|
fetched_messages << message
|
38
37
|
end
|
data/lib/kafka/snappy_codec.rb
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
module Kafka
|
2
2
|
class SnappyCodec
|
3
|
-
def
|
3
|
+
def codec_id
|
4
|
+
2
|
5
|
+
end
|
6
|
+
|
7
|
+
def load
|
4
8
|
require "snappy"
|
5
9
|
rescue LoadError
|
6
10
|
raise LoadError,
|
7
11
|
"Using snappy compression requires adding a dependency on the `snappy` gem to your Gemfile."
|
8
12
|
end
|
9
13
|
|
10
|
-
def codec_id
|
11
|
-
2
|
12
|
-
end
|
13
|
-
|
14
14
|
def compress(data)
|
15
15
|
Snappy.deflate(data)
|
16
16
|
end
|
data/lib/kafka/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ruby-kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.2.
|
4
|
+
version: 0.5.2.beta2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Daniel Schierbeck
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-12-
|
11
|
+
date: 2017-12-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|