rafka 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: bd21afb5b92c9e7c0e712459ca3f3af0ce3fe823
4
+ data.tar.gz: 1c5868fa247da36d91d14b176586620e6c7a1298
5
+ SHA512:
6
+ metadata.gz: 3fe0b6acbf267039e172a49f8022e9d1a90ea8f33b7e472f7fb442b645a9bfe8105bc5151943fc7fb309fd6abcbf0ce6c9d066658190a7d590cdb5e2f49bdead
7
+ data.tar.gz: 07cc6ca0d115cf69cd8e189756583dbe2325a74ac90bfc7cc27c9326df41c1f079f73a267e4f2b7d637a9e870000a5d7a683a72f54efccaf8e3231193de015e5
@@ -0,0 +1 @@
1
+ TODO
@@ -0,0 +1,73 @@
1
+ rafka-rb: Ruby driver for Rafka
2
+ ===============================================================================
3
+ [![Gem Version](https://badge.fury.io/rb/rafka.svg)](https://badge.fury.io/rb/rafka-rb)
4
+ [![Documentation](http://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/github/skroutz/rafka-rb)
5
+
6
+ Rafka is a thin Ruby client library for [Rafka](https://github.com/skroutz/rafka-rb),
7
+ providing a consumer and a producer with simple semantics. It is backed by
8
+ [redis-rb](https://github.com/redis/redis-rb).
9
+
10
+ View the [API documentation](http://www.rubydoc.info/github/skroutz/rafka-rb).
11
+
12
+ Status
13
+ -------------------------------------------------------------------------------
14
+
15
+ Rafka is not yet stable and therefore is _not_ recommended for production use.
16
+
17
+
18
+
19
+
20
+
21
+
22
+
23
+
24
+
25
+ Getting started
26
+ -------------------------------------------------------------------------------
27
+ Install rafka-rb:
28
+
29
+ ```shell
30
+ $ gem install rafka
31
+ ```
32
+
33
+ If you're using Bundler, add it to your Gemfile:
34
+ ```ruby
35
+ gem "rafka"
36
+ ```
37
+ and run `bundle install`.
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+ Usage
46
+ -------------------------------------------------------------------------------
47
+
48
+ ### Producer
49
+
50
+ ```ruby
51
+ require "rafka"
52
+
53
+ prod = Rafka::Producer.new(host: "localhost", port: 6380)
54
+ prod.produce("greetings", "Hello there!") # produce to topic "greetings"
55
+ ```
56
+
57
+
58
+
59
+
60
+ ### Consumer
61
+
62
+ ```ruby
63
+ require "rafka"
64
+
65
+ cons = Rafka::Consumer.new(topic: "greetings", group: "myapp", id: "greeter1")
66
+ cons.consume # => "Hello there!"
67
+
68
+ # with a block
69
+ cons.consume { |msg| puts "Received: #{msg.value}" } # => "Hello there!"
70
+ ```
71
+
72
+ `Rafka::Consumer#consume` automatically commits the offsets when the given block
73
+ is executed without raising any exceptions.
@@ -0,0 +1,18 @@
1
+ require "redis"
2
+
3
+ require "rafka/errors"
4
+ require "rafka/generic_commands"
5
+ require "rafka/message"
6
+ require "rafka/version"
7
+
8
+ require "rafka/consumer"
9
+ require "rafka/producer"
10
+
11
+ module Rafka
12
+ DEFAULTS = {
13
+ host: "localhost",
14
+ port: 6380,
15
+ reconnect_attempts: 0,
16
+ }
17
+ end
18
+
@@ -0,0 +1,75 @@
1
+ require 'securerandom'
2
+
3
+ module Rafka
4
+ class Consumer
5
+ include GenericCommands
6
+
7
+ REQUIRED = [:group, :topic]
8
+
9
+ # The underlying Redis client object
10
+ attr_reader :redis
11
+
12
+ # Create a new client instance.
13
+ #
14
+ # @param [Hash] opts
15
+ # @option opts [String] :host ("localhost") server hostname
16
+ # @option opts [Fixnum] :port (6380) server port
17
+ # @option opts [String] :topic Kafka topic to consume (required)
18
+ # @option opts [String] :group Kafka consumer group name (required)
19
+ # @option opts [String] :id (random) Kafka consumer id
20
+ # @option opts [Hash] :redis_opts ({}) Optional configuration for the
21
+ # underlying Redis client
22
+ def initialize(opts={})
23
+ opts[:redis_opts] = {} if !opts[:redis_opts]
24
+ opts = parse_opts(opts)
25
+ client_id = "#{opts[:group]}:#{opts[:id]}"
26
+ @redis = Redis.new(host: opts[:host], port: opts[:port], id: client_id)
27
+ @topic = "topics:#{opts[:topic]}"
28
+ end
29
+
30
+ # Fetch the next message.
31
+ #
32
+ # @param timeout [Fixnum] the time in seconds to wait for a message
33
+ #
34
+ # @raise [MalformedMessage] if the message from Rafka cannot be parsed
35
+ #
36
+ # @return [nil, Message] the message, if any
37
+ #
38
+ # @example
39
+ # consume(5) { |msg| puts "I received #{msg.value}" }
40
+ def consume(timeout=5)
41
+ msg = @redis.blpop(@topic, timeout: timeout)
42
+
43
+ return if !msg
44
+
45
+ msg = Message.new(msg)
46
+
47
+ begin
48
+ raised = false
49
+ yield(msg) if block_given?
50
+ rescue => e
51
+ raised = true
52
+ raise e
53
+ end
54
+
55
+ msg
56
+ ensure
57
+ if msg && !raised
58
+ @redis.rpush("acks", "#{msg.topic}:#{msg.partition}:#{msg.offset}")
59
+ end
60
+ end
61
+
62
+ private
63
+
64
+ def parse_opts(opts)
65
+ options = DEFAULTS.dup.merge(opts).merge(opts[:redis_opts])
66
+ options[:id] = SecureRandom.hex if !options[:id]
67
+
68
+ REQUIRED.each do |opt|
69
+ raise "#{opt.inspect} option not provided" if !options[opt]
70
+ end
71
+
72
+ options
73
+ end
74
+ end
75
+ end
@@ -0,0 +1,14 @@
1
+ module Rafka
2
+ class Error < StandardError
3
+ end
4
+
5
+ class MalformedMessage < Error
6
+ def initialize(msg)
7
+ @msg = msg
8
+ end
9
+
10
+ def to_s
11
+ "The message #{@msg.inspect} could not be parsed"
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ module Rafka
2
+ # Commands available on both {Producer} and {Consumer}.
3
+ module GenericCommands
4
+ # @see https://redis.io/commands/ping
5
+ def ping
6
+ @redis.ping
7
+ end
8
+
9
+ # @see https://redis.io/commands/quit
10
+ def quit
11
+ @redis.quit
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,17 @@
1
+ module Rafka
2
+ # Message represents a message consumed from a topic.
3
+ class Message
4
+ attr :topic, :partition, :offset, :value
5
+
6
+ def initialize(msg)
7
+ if !msg.is_a?(Array) || msg.size != 8
8
+ raise MalformedMessage.new(msg)
9
+ end
10
+
11
+ @topic = msg[1]
12
+ @partition = msg[3]
13
+ @offset = msg[5]
14
+ @value = msg[7]
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,49 @@
1
+ module Rafka
2
+ class Producer
3
+ # Access the underlying Redis client object
4
+ attr_reader :redis
5
+
6
+ # Create a new client instance.
7
+ #
8
+ # @param [Hash] opts
9
+ # @option opts [String] :host ("localhost") server hostname
10
+ # @option opts [Fixnum] :port (6380) server port
11
+ # @options opts [Hash] :redis_opts Configuration options for the underlying
12
+ # Redis client
13
+ #
14
+ # @return [Producer]
15
+ def initialize(opts = {})
16
+ opts[:redis_opts] = {} if !opts[:redis_opts]
17
+ opts = parse_opts(opts)
18
+ @redis = Redis.new(host: opts[:host], port: opts[:port])
19
+ end
20
+
21
+ # Produce a message. This is a non-blocking operation.
22
+ #
23
+ # @param topic [String]
24
+ # @param message [#to_s]
25
+ #
26
+ # @example
27
+ # produce("greetings", "Hello there!")
28
+ def produce(topic, message)
29
+ @redis.rpushx("topics:#{topic}", message.to_s)
30
+ end
31
+
32
+ # Flush any buffered messages. Blocks until all messages are flushed or
33
+ # timeout exceeds.
34
+ #
35
+ # @param timeout_ms [Fixnum] (5000) The timeout in milliseconds
36
+ #
37
+ # @return [Fixnum] The number of unflushed messages
38
+ def flush(timeout_ms=5000)
39
+ @redis.dump(timeout_ms.to_s)
40
+ end
41
+
42
+ private
43
+
44
+ def parse_opts(opts)
45
+ options = DEFAULTS.dup.merge(opts).merge(opts[:redis_opts])
46
+ options
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,3 @@
1
+ module Rafka
2
+ VERSION = "0.0.1".freeze
3
+ end
metadata ADDED
@@ -0,0 +1,81 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: rafka
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Agis Anastasopoulos
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2017-06-26 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: redis
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '3.3'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '3.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: pry-byebug
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ description: A Ruby client library for Rafka, with consumer and producer implementations.
42
+ email:
43
+ - agis.anast@gmail.com
44
+ executables: []
45
+ extensions: []
46
+ extra_rdoc_files: []
47
+ files:
48
+ - CHANGELOG.md
49
+ - README.md
50
+ - lib/rafka.rb
51
+ - lib/rafka/consumer.rb
52
+ - lib/rafka/errors.rb
53
+ - lib/rafka/generic_commands.rb
54
+ - lib/rafka/message.rb
55
+ - lib/rafka/producer.rb
56
+ - lib/rafka/version.rb
57
+ homepage: https://github.com/skroutz/rafka-rb
58
+ licenses:
59
+ - MIT
60
+ metadata: {}
61
+ post_install_message:
62
+ rdoc_options: []
63
+ require_paths:
64
+ - lib
65
+ required_ruby_version: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ required_rubygems_version: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ requirements: []
76
+ rubyforge_project:
77
+ rubygems_version: 2.5.2
78
+ signing_key:
79
+ specification_version: 4
80
+ summary: Ruby driver for Rafka
81
+ test_files: []