kafkr 0.18.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/kafkr +4 -4
- data/exe/kafkr-consumer +5 -6
- data/exe/kafkr-keys +1 -1
- data/exe/kafkr-producer +6 -5
- data/handlers/web_handler.rb +3 -2
- data/lib/kafkr/consumer.rb +5 -57
- data/lib/kafkr/encryptor.rb +2 -2
- data/lib/kafkr/log.rb +5 -3
- data/lib/kafkr/message_broker.rb +1 -1
- data/lib/kafkr/producer.rb +4 -38
- data/lib/kafkr/version.rb +1 -1
- data/lib/kafkr.rb +4 -46
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 27db175bfdb0bab5dec371a1abf78a0f54293ee6c883612f1fed4f7c48d55e01
|
4
|
+
data.tar.gz: d50374c0847f19ac9141bb6eff5d9dd968594c9c4380aecbdc6417dc54322b14
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 050231b1632c5ead63874eee350807cb821584892e87d4dbfdad10d7f10281872ab32fa60f2089d39bf0cf9b26ab41514ad6451430445deca35b4013a85e55e7
|
7
|
+
data.tar.gz: 5cd9a1e0e8b5404120fd8d0aea97a421422b219b5470eda2c0e700db7386583283a3b794bc0a4a2bae96f117c2c39b9c581442113c269f567109e577b53ad973
|
data/exe/kafkr
CHANGED
@@ -6,19 +6,19 @@ PORT = ENV["KAFKR_PORT"] || 4000
|
|
6
6
|
begin
|
7
7
|
require "kafkr"
|
8
8
|
rescue LoadError => e
|
9
|
-
|
9
|
+
Kafkr.log "Failed to load Kafkr: #{e.message}"
|
10
10
|
exit(1)
|
11
11
|
end
|
12
12
|
|
13
13
|
begin
|
14
14
|
server = Kafkr::Log.new(PORT.to_i)
|
15
|
-
|
15
|
+
Kafkr.log "Kafkr Log version #{Kafkr::VERSION} started on port #{PORT}!"
|
16
16
|
server.start
|
17
17
|
rescue => e
|
18
|
-
|
18
|
+
Kafkr.log "An error occurred: #{e.message}"
|
19
19
|
exit(1)
|
20
20
|
rescue Interrupt
|
21
|
-
|
21
|
+
Kafkr.log "\nLog server shutting down gracefully..."
|
22
22
|
server.stop if server.respond_to?(:stop)
|
23
23
|
exit(0)
|
24
24
|
end
|
data/exe/kafkr-consumer
CHANGED
@@ -9,7 +9,7 @@ host = ARGV[0] || "localhost"
|
|
9
9
|
port = ARGV[1] || 4000
|
10
10
|
timeout = ARGV[2] || 300
|
11
11
|
|
12
|
-
|
12
|
+
Kafkr.log "Running on host: #{host} and port: #{port}"
|
13
13
|
|
14
14
|
$current_consumer = nil
|
15
15
|
$restart_required = false
|
@@ -26,16 +26,15 @@ def stop_consumer
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def list_registered_handlers
|
29
|
-
puts "Registered Handlers:"
|
30
29
|
Kafkr::Consumer.handlers.each do |handler|
|
31
30
|
$loaded_handlers = {}
|
32
31
|
handler_name = handler.class.name.split("::").last.gsub(/Handler$/, "")
|
33
|
-
|
32
|
+
Kafkr.log "#{handler_name} handler registered."
|
34
33
|
end
|
35
34
|
end
|
36
35
|
|
37
36
|
def start_consumer(port, host, timeout)
|
38
|
-
|
37
|
+
Kafkr.log "Starting consumer on port #{port}! timeout: #{timeout}"
|
39
38
|
$handlers_changed = false
|
40
39
|
|
41
40
|
Kafkr::Consumer.configure do |config|
|
@@ -52,7 +51,7 @@ def start_consumer(port, host, timeout)
|
|
52
51
|
|
53
52
|
$current_consumer = Kafkr::Consumer.new
|
54
53
|
$current_consumer.listen do |message|
|
55
|
-
# Processing of the message
|
54
|
+
Kafkr.log "Message consumed: #{message}"# Processing of the message
|
56
55
|
end
|
57
56
|
end
|
58
57
|
|
@@ -69,7 +68,7 @@ def reload_handlers(file_checksums)
|
|
69
68
|
if new_handlers.any?
|
70
69
|
new_handlers.each do |handler|
|
71
70
|
handler_name = handler.class.name.split("::").last.gsub(/Handler$/, "").capitalize
|
72
|
-
|
71
|
+
Kafkr.log "#{handler_name} handler updated - ok!"
|
73
72
|
end
|
74
73
|
end
|
75
74
|
end
|
data/exe/kafkr-keys
CHANGED
data/exe/kafkr-producer
CHANGED
@@ -1,16 +1,17 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require "kafkr"
|
2
3
|
require "readline"
|
3
4
|
|
4
5
|
# Parsing command line arguments for host and port
|
5
6
|
host = ARGV[0] || "localhost"
|
6
7
|
port = ARGV[1] || 4000
|
7
8
|
|
8
|
-
|
9
|
+
Kafkr.log "Running on host: #{host} and port: #{port}"
|
9
10
|
|
10
11
|
begin
|
11
12
|
require "kafkr"
|
12
13
|
rescue LoadError => e
|
13
|
-
|
14
|
+
Kafkr.log "Failed to load Kafkr: #{e.message}"
|
14
15
|
exit(1)
|
15
16
|
end
|
16
17
|
|
@@ -24,15 +25,15 @@ begin
|
|
24
25
|
break if line == "exit"
|
25
26
|
|
26
27
|
if line.include? "<=>"
|
27
|
-
|
28
|
+
Kafkr.log Kafkr::Producer.send_message_and_wait(line)
|
28
29
|
else
|
29
30
|
Kafkr::Producer.send_message(line)
|
30
31
|
end
|
31
32
|
end
|
32
33
|
rescue => e
|
33
|
-
|
34
|
+
Kafkr.log "An error occurred: #{e.message}"
|
34
35
|
exit(1)
|
35
36
|
rescue Interrupt
|
36
|
-
|
37
|
+
Kafkr.log "\nProducer server shutting down gracefully..."
|
37
38
|
exit(0)
|
38
39
|
end
|
data/handlers/web_handler.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
|
+
require "pry"
|
2
|
+
|
1
3
|
class WebHandler < Kafkr::Consumer::Handler
|
2
4
|
def handle?(message)
|
3
|
-
|
5
|
+
message.include?("web")
|
4
6
|
end
|
5
7
|
|
6
8
|
def handle(message)
|
7
|
-
puts message
|
8
9
|
if message["sync"]
|
9
10
|
reply to: message, payload: {test: "set"}
|
10
11
|
end
|
data/lib/kafkr/consumer.rb
CHANGED
@@ -37,8 +37,8 @@ module Kafkr
|
|
37
37
|
$handlers_changed = true
|
38
38
|
|
39
39
|
def list_registered_handlers
|
40
|
-
|
41
|
-
$loaded_handlers.keys.each { |handler|
|
40
|
+
Kafkr.log "Registered handlers:"
|
41
|
+
$loaded_handlers.keys.each { |handler| Kafkr.log "- #{handler}" }
|
42
42
|
end
|
43
43
|
|
44
44
|
def load_handlers(directory = "./handlers")
|
@@ -77,23 +77,6 @@ module Kafkr
|
|
77
77
|
Kafkr::Producer.send_message({reply: {payload: payload, uuid: to["sync_uid"]}})
|
78
78
|
end
|
79
79
|
|
80
|
-
private
|
81
|
-
|
82
|
-
def can_handle?(message, name, ignore: :any)
|
83
|
-
if message.is_a?(Numeric)
|
84
|
-
return true if message == name.to_i
|
85
|
-
elsif ignore == :hash
|
86
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body] == name
|
87
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body].start_with?(name)
|
88
|
-
elsif ignore == :string
|
89
|
-
return true if message.key? name
|
90
|
-
else
|
91
|
-
return true if message.key? name
|
92
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body] == name
|
93
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body].start_with?(name)
|
94
|
-
end
|
95
|
-
false
|
96
|
-
end
|
97
80
|
end
|
98
81
|
|
99
82
|
def initialize(host = Consumer.configuration.host, port = Consumer.configuration.port)
|
@@ -113,40 +96,6 @@ module Kafkr
|
|
113
96
|
/^[A-Z]\w*$/.match?(name)
|
114
97
|
end
|
115
98
|
|
116
|
-
def print_handler_class(name)
|
117
|
-
return if name.is_a?(Numeric)
|
118
|
-
name = name.keys.first if name.is_a?(Hash)
|
119
|
-
handler_name = "#{name.downcase}_handler"
|
120
|
-
|
121
|
-
if $loaded_handlers.key?(handler_name)
|
122
|
-
return
|
123
|
-
end
|
124
|
-
|
125
|
-
if Kafkr::Consumer.configuration.suggest_handlers
|
126
|
-
if valid_class_name?(name.capitalize)
|
127
|
-
puts "No handler for this message, you could use this one.\n\n"
|
128
|
-
|
129
|
-
handler_class_string = <<~HANDLER_CLASS
|
130
|
-
class #{name.capitalize}Handler < Kafkr::Consumer::Handler
|
131
|
-
def handle?(message)
|
132
|
-
can_handle? message, '#{name}'
|
133
|
-
end
|
134
|
-
|
135
|
-
def handle(message)
|
136
|
-
puts message
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
# Save the file to ./handlers/#{name}_handler.rb
|
141
|
-
HANDLER_CLASS
|
142
|
-
|
143
|
-
puts handler_class_string
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
147
|
-
|
148
|
-
require "timeout"
|
149
|
-
|
150
99
|
def listen_for(message, send_message)
|
151
100
|
attempt = 0
|
152
101
|
begin
|
@@ -170,7 +119,7 @@ module Kafkr
|
|
170
119
|
sleep(wait_time)
|
171
120
|
retry
|
172
121
|
rescue Interrupt
|
173
|
-
|
122
|
+
Kafkr.log "Received interrupt signal. Shutting down consumer gracefully..."
|
174
123
|
socket&.close
|
175
124
|
exit(0)
|
176
125
|
end
|
@@ -202,7 +151,8 @@ module Kafkr
|
|
202
151
|
private
|
203
152
|
|
204
153
|
def dispatch_to_handlers(message)
|
205
|
-
|
154
|
+
|
155
|
+
message_hash = JSON.parse(message)
|
206
156
|
|
207
157
|
self.class.handlers.each do |handler|
|
208
158
|
if handler.handle?(message_hash)
|
@@ -210,8 +160,6 @@ module Kafkr
|
|
210
160
|
end
|
211
161
|
end
|
212
162
|
|
213
|
-
print_handler_class(message)
|
214
|
-
|
215
163
|
yield message_hash if block_given?
|
216
164
|
end
|
217
165
|
end
|
data/lib/kafkr/encryptor.rb
CHANGED
@@ -19,7 +19,7 @@ module Kafkr
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def decrypt(encrypted_data)
|
22
|
-
#
|
22
|
+
# Kafkr.log "Encrypted data before decoding: #{encrypted_data.inspect}"
|
23
23
|
decipher = OpenSSL::Cipher.new(ALGORITHM)
|
24
24
|
decipher.decrypt
|
25
25
|
decipher.key = @key
|
@@ -27,7 +27,7 @@ module Kafkr
|
|
27
27
|
decipher.iv = raw_data[0, decipher.iv_len]
|
28
28
|
decipher.update(raw_data[decipher.iv_len..-1]) + decipher.final
|
29
29
|
rescue OpenSSL::Cipher::CipherError => e
|
30
|
-
|
30
|
+
Kafkr.log "Decryption failed: #{e.message}"
|
31
31
|
nil
|
32
32
|
end
|
33
33
|
end
|
data/lib/kafkr/log.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require "socket"
|
2
2
|
require "rubygems"
|
3
|
+
require "kafkr"
|
4
|
+
|
3
5
|
|
4
6
|
module Kafkr
|
5
7
|
class Log
|
@@ -27,7 +29,7 @@ module Kafkr
|
|
27
29
|
client_ip = client.peeraddr[3]
|
28
30
|
|
29
31
|
unless whitelisted?(client_ip)
|
30
|
-
|
32
|
+
Kafkr.log "Connection from non-whitelisted IP: #{client_ip}. Ignored."
|
31
33
|
client.close
|
32
34
|
next
|
33
35
|
end
|
@@ -50,13 +52,13 @@ module Kafkr
|
|
50
52
|
decryptor = Kafkr::Encryptor.new
|
51
53
|
message = decryptor.decrypt(encrypted_message.chomp) # Decrypt the message here
|
52
54
|
uuid, message_content = extract_uuid(message)
|
53
|
-
@broker.broadcast(message_content)
|
55
|
+
@broker.broadcast(JSON.dump(message_content))
|
54
56
|
end
|
55
57
|
rescue Errno::ECONNRESET
|
56
58
|
client.close
|
57
59
|
end
|
58
60
|
rescue StandardError => exception
|
59
|
-
|
61
|
+
Kafkr.log "Error: #{exception.message}"
|
60
62
|
end
|
61
63
|
end
|
62
64
|
end
|
data/lib/kafkr/message_broker.rb
CHANGED
data/lib/kafkr/producer.rb
CHANGED
@@ -17,7 +17,7 @@ module Kafkr
|
|
17
17
|
@configuration.queue_file = MESSAGE_QUEUE
|
18
18
|
@configuration.message_queue = []
|
19
19
|
load_queue_from_file
|
20
|
-
@configuration.is_json =
|
20
|
+
@configuration.is_json = true
|
21
21
|
@configuration
|
22
22
|
end
|
23
23
|
|
@@ -27,28 +27,6 @@ module Kafkr
|
|
27
27
|
logger.error("Configuration error: #{e.message}")
|
28
28
|
end
|
29
29
|
|
30
|
-
def self.structured_data_to_hash(input:, sync_uid:)
|
31
|
-
unless /\A\w+\s*(=>|<=>)\s*((\w+:\s*['"]?[^'",]*['"]?,\s*)*(\w+:\s*['"]?[^'",]*['"]?)\s*)\z/.match?(input)
|
32
|
-
return input
|
33
|
-
end
|
34
|
-
|
35
|
-
if input.include?("<=>")
|
36
|
-
type, key_values_str = input.split("<=>").map(&:strip)
|
37
|
-
key_values = key_values_str.scan(/(\w+):\s*['"]?([^'",]*)['"]?/)
|
38
|
-
hash_body = key_values.to_h do |key, value|
|
39
|
-
[key.to_sym, value.strip.gsub(/\A['"]|['"]\z/, "")]
|
40
|
-
end
|
41
|
-
{type.to_sym => hash_body, :sync => true, :sync_uid => sync_uid}
|
42
|
-
else
|
43
|
-
type, key_values_str = input.split("=>").map(&:strip)
|
44
|
-
key_values = key_values_str.scan(/(\w+):\s*['"]?([^'",]*)['"]?/)
|
45
|
-
hash_body = key_values.to_h do |key, value|
|
46
|
-
[key.to_sym, value.strip.gsub(/\A['"]|['"]\z/, "")]
|
47
|
-
end
|
48
|
-
{type.to_sym => hash_body}
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
30
|
def self.send_message(message)
|
53
31
|
return if message.nil? || message.empty?
|
54
32
|
|
@@ -59,15 +37,6 @@ module Kafkr
|
|
59
37
|
json_message = JSON.parse(message)
|
60
38
|
json_message["uuid"] = uuid
|
61
39
|
message_with_uuid = JSON.dump(json_message)
|
62
|
-
else
|
63
|
-
if message.is_a? String
|
64
|
-
message = structured_data_to_hash(input: message, sync_uid: uuid)
|
65
|
-
message_with_uuid = "#{uuid}: #{message}"
|
66
|
-
end
|
67
|
-
|
68
|
-
if message.is_a?(Hash)
|
69
|
-
message_with_uuid = "#{uuid}: #{JSON.generate(message)}"
|
70
|
-
end
|
71
40
|
end
|
72
41
|
|
73
42
|
encrypted_message_with_uuid = Kafkr::Encryptor.new.encrypt(message_with_uuid)
|
@@ -75,13 +44,13 @@ module Kafkr
|
|
75
44
|
begin
|
76
45
|
socket = TCPSocket.new(@configuration.host, @configuration.port)
|
77
46
|
send_queued_messages(socket)
|
78
|
-
socket.puts(encrypted_message_with_uuid)
|
47
|
+
socket.puts (encrypted_message_with_uuid)
|
79
48
|
rescue Errno::ECONNREFUSED
|
80
|
-
|
49
|
+
Kafkr.log "Connection refused. Queuing message: #{encrypted_message_with_uuid}"
|
81
50
|
@configuration.message_queue.push(encrypted_message_with_uuid)
|
82
51
|
save_queue_to_file
|
83
52
|
rescue Errno::EPIPE
|
84
|
-
|
53
|
+
Kafkr.log "Broken pipe error. Retrying connection..."
|
85
54
|
retry_connection(encrypted_message_with_uuid)
|
86
55
|
end
|
87
56
|
|
@@ -126,8 +95,5 @@ module Kafkr
|
|
126
95
|
end
|
127
96
|
end
|
128
97
|
|
129
|
-
def self.logger
|
130
|
-
@logger ||= Logger.new(STDOUT)
|
131
|
-
end
|
132
98
|
end
|
133
99
|
end
|
data/lib/kafkr/version.rb
CHANGED
data/lib/kafkr.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
require "pry"
|
3
3
|
require "logger"
|
4
4
|
require "openssl"
|
5
5
|
require "securerandom"
|
@@ -15,64 +15,21 @@ require_relative "kafkr/version"
|
|
15
15
|
|
16
16
|
module Kafkr
|
17
17
|
class << self
|
18
|
-
attr_accessor :current_environment
|
19
18
|
def logger
|
20
19
|
@logger ||= configure_logger
|
21
20
|
end
|
22
21
|
|
23
|
-
def configure_logger(output =
|
22
|
+
def configure_logger(output = STDOUT)
|
24
23
|
begin
|
25
24
|
@logger = ::Logger.new(output)
|
26
25
|
rescue Errno::EACCES, Errno::ENOENT => e
|
27
26
|
@logger = ::Logger.new(STDOUT)
|
28
27
|
@logger.error("Could not open log file: #{e.message}")
|
29
28
|
end
|
30
|
-
|
29
|
+
@logger.level = ::Logger::DEBUG
|
31
30
|
@logger
|
32
31
|
end
|
33
32
|
|
34
|
-
def default_output
|
35
|
-
case current_environment
|
36
|
-
when "production"
|
37
|
-
"/var/log/kafkr.log"
|
38
|
-
else
|
39
|
-
STDOUT
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def set_logger_level
|
44
|
-
@logger.level = case current_environment
|
45
|
-
when "development"
|
46
|
-
::Logger::DEBUG
|
47
|
-
when "staging"
|
48
|
-
::Logger::INFO
|
49
|
-
when "production"
|
50
|
-
::Logger::WARN
|
51
|
-
else
|
52
|
-
::Logger::DEBUG
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def current_environment
|
57
|
-
@current_environment ||= ENV["KAFKR_ENV"] || "development"
|
58
|
-
end
|
59
|
-
|
60
|
-
def development?
|
61
|
-
current_environment == "development"
|
62
|
-
end
|
63
|
-
|
64
|
-
def test?
|
65
|
-
current_environment == "test"
|
66
|
-
end
|
67
|
-
|
68
|
-
def staging?
|
69
|
-
current_environment == "staging"
|
70
|
-
end
|
71
|
-
|
72
|
-
def production?
|
73
|
-
current_environment == "production"
|
74
|
-
end
|
75
|
-
|
76
33
|
def write(message, unique_id = nil)
|
77
34
|
begin
|
78
35
|
logger.info(message)
|
@@ -100,3 +57,4 @@ module Kafkr
|
|
100
57
|
logger.error("Configuration error: #{e.message}")
|
101
58
|
end
|
102
59
|
end
|
60
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.21.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Delaney Kuldvee Burke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-04-
|
11
|
+
date: 2024-04-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gibberish
|