kafkr 0.18.0 → 0.21.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/kafkr +4 -4
- data/exe/kafkr-consumer +5 -6
- data/exe/kafkr-keys +1 -1
- data/exe/kafkr-producer +6 -5
- data/handlers/web_handler.rb +3 -2
- data/lib/kafkr/consumer.rb +5 -57
- data/lib/kafkr/encryptor.rb +2 -2
- data/lib/kafkr/log.rb +5 -3
- data/lib/kafkr/message_broker.rb +1 -1
- data/lib/kafkr/producer.rb +4 -38
- data/lib/kafkr/version.rb +1 -1
- data/lib/kafkr.rb +4 -46
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 27db175bfdb0bab5dec371a1abf78a0f54293ee6c883612f1fed4f7c48d55e01
|
4
|
+
data.tar.gz: d50374c0847f19ac9141bb6eff5d9dd968594c9c4380aecbdc6417dc54322b14
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 050231b1632c5ead63874eee350807cb821584892e87d4dbfdad10d7f10281872ab32fa60f2089d39bf0cf9b26ab41514ad6451430445deca35b4013a85e55e7
|
7
|
+
data.tar.gz: 5cd9a1e0e8b5404120fd8d0aea97a421422b219b5470eda2c0e700db7386583283a3b794bc0a4a2bae96f117c2c39b9c581442113c269f567109e577b53ad973
|
data/exe/kafkr
CHANGED
@@ -6,19 +6,19 @@ PORT = ENV["KAFKR_PORT"] || 4000
|
|
6
6
|
begin
|
7
7
|
require "kafkr"
|
8
8
|
rescue LoadError => e
|
9
|
-
|
9
|
+
Kafkr.log "Failed to load Kafkr: #{e.message}"
|
10
10
|
exit(1)
|
11
11
|
end
|
12
12
|
|
13
13
|
begin
|
14
14
|
server = Kafkr::Log.new(PORT.to_i)
|
15
|
-
|
15
|
+
Kafkr.log "Kafkr Log version #{Kafkr::VERSION} started on port #{PORT}!"
|
16
16
|
server.start
|
17
17
|
rescue => e
|
18
|
-
|
18
|
+
Kafkr.log "An error occurred: #{e.message}"
|
19
19
|
exit(1)
|
20
20
|
rescue Interrupt
|
21
|
-
|
21
|
+
Kafkr.log "\nLog server shutting down gracefully..."
|
22
22
|
server.stop if server.respond_to?(:stop)
|
23
23
|
exit(0)
|
24
24
|
end
|
data/exe/kafkr-consumer
CHANGED
@@ -9,7 +9,7 @@ host = ARGV[0] || "localhost"
|
|
9
9
|
port = ARGV[1] || 4000
|
10
10
|
timeout = ARGV[2] || 300
|
11
11
|
|
12
|
-
|
12
|
+
Kafkr.log "Running on host: #{host} and port: #{port}"
|
13
13
|
|
14
14
|
$current_consumer = nil
|
15
15
|
$restart_required = false
|
@@ -26,16 +26,15 @@ def stop_consumer
|
|
26
26
|
end
|
27
27
|
|
28
28
|
def list_registered_handlers
|
29
|
-
puts "Registered Handlers:"
|
30
29
|
Kafkr::Consumer.handlers.each do |handler|
|
31
30
|
$loaded_handlers = {}
|
32
31
|
handler_name = handler.class.name.split("::").last.gsub(/Handler$/, "")
|
33
|
-
|
32
|
+
Kafkr.log "#{handler_name} handler registered."
|
34
33
|
end
|
35
34
|
end
|
36
35
|
|
37
36
|
def start_consumer(port, host, timeout)
|
38
|
-
|
37
|
+
Kafkr.log "Starting consumer on port #{port}! timeout: #{timeout}"
|
39
38
|
$handlers_changed = false
|
40
39
|
|
41
40
|
Kafkr::Consumer.configure do |config|
|
@@ -52,7 +51,7 @@ def start_consumer(port, host, timeout)
|
|
52
51
|
|
53
52
|
$current_consumer = Kafkr::Consumer.new
|
54
53
|
$current_consumer.listen do |message|
|
55
|
-
# Processing of the message
|
54
|
+
Kafkr.log "Message consumed: #{message}"# Processing of the message
|
56
55
|
end
|
57
56
|
end
|
58
57
|
|
@@ -69,7 +68,7 @@ def reload_handlers(file_checksums)
|
|
69
68
|
if new_handlers.any?
|
70
69
|
new_handlers.each do |handler|
|
71
70
|
handler_name = handler.class.name.split("::").last.gsub(/Handler$/, "").capitalize
|
72
|
-
|
71
|
+
Kafkr.log "#{handler_name} handler updated - ok!"
|
73
72
|
end
|
74
73
|
end
|
75
74
|
end
|
data/exe/kafkr-keys
CHANGED
data/exe/kafkr-producer
CHANGED
@@ -1,16 +1,17 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
+
require "kafkr"
|
2
3
|
require "readline"
|
3
4
|
|
4
5
|
# Parsing command line arguments for host and port
|
5
6
|
host = ARGV[0] || "localhost"
|
6
7
|
port = ARGV[1] || 4000
|
7
8
|
|
8
|
-
|
9
|
+
Kafkr.log "Running on host: #{host} and port: #{port}"
|
9
10
|
|
10
11
|
begin
|
11
12
|
require "kafkr"
|
12
13
|
rescue LoadError => e
|
13
|
-
|
14
|
+
Kafkr.log "Failed to load Kafkr: #{e.message}"
|
14
15
|
exit(1)
|
15
16
|
end
|
16
17
|
|
@@ -24,15 +25,15 @@ begin
|
|
24
25
|
break if line == "exit"
|
25
26
|
|
26
27
|
if line.include? "<=>"
|
27
|
-
|
28
|
+
Kafkr.log Kafkr::Producer.send_message_and_wait(line)
|
28
29
|
else
|
29
30
|
Kafkr::Producer.send_message(line)
|
30
31
|
end
|
31
32
|
end
|
32
33
|
rescue => e
|
33
|
-
|
34
|
+
Kafkr.log "An error occurred: #{e.message}"
|
34
35
|
exit(1)
|
35
36
|
rescue Interrupt
|
36
|
-
|
37
|
+
Kafkr.log "\nProducer server shutting down gracefully..."
|
37
38
|
exit(0)
|
38
39
|
end
|
data/handlers/web_handler.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
|
+
require "pry"
|
2
|
+
|
1
3
|
class WebHandler < Kafkr::Consumer::Handler
|
2
4
|
def handle?(message)
|
3
|
-
|
5
|
+
message.include?("web")
|
4
6
|
end
|
5
7
|
|
6
8
|
def handle(message)
|
7
|
-
puts message
|
8
9
|
if message["sync"]
|
9
10
|
reply to: message, payload: {test: "set"}
|
10
11
|
end
|
data/lib/kafkr/consumer.rb
CHANGED
@@ -37,8 +37,8 @@ module Kafkr
|
|
37
37
|
$handlers_changed = true
|
38
38
|
|
39
39
|
def list_registered_handlers
|
40
|
-
|
41
|
-
$loaded_handlers.keys.each { |handler|
|
40
|
+
Kafkr.log "Registered handlers:"
|
41
|
+
$loaded_handlers.keys.each { |handler| Kafkr.log "- #{handler}" }
|
42
42
|
end
|
43
43
|
|
44
44
|
def load_handlers(directory = "./handlers")
|
@@ -77,23 +77,6 @@ module Kafkr
|
|
77
77
|
Kafkr::Producer.send_message({reply: {payload: payload, uuid: to["sync_uid"]}})
|
78
78
|
end
|
79
79
|
|
80
|
-
private
|
81
|
-
|
82
|
-
def can_handle?(message, name, ignore: :any)
|
83
|
-
if message.is_a?(Numeric)
|
84
|
-
return true if message == name.to_i
|
85
|
-
elsif ignore == :hash
|
86
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body] == name
|
87
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body].start_with?(name)
|
88
|
-
elsif ignore == :string
|
89
|
-
return true if message.key? name
|
90
|
-
else
|
91
|
-
return true if message.key? name
|
92
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body] == name
|
93
|
-
return true if message[:message] && message[:message][:body] && message[:message][:body].start_with?(name)
|
94
|
-
end
|
95
|
-
false
|
96
|
-
end
|
97
80
|
end
|
98
81
|
|
99
82
|
def initialize(host = Consumer.configuration.host, port = Consumer.configuration.port)
|
@@ -113,40 +96,6 @@ module Kafkr
|
|
113
96
|
/^[A-Z]\w*$/.match?(name)
|
114
97
|
end
|
115
98
|
|
116
|
-
def print_handler_class(name)
|
117
|
-
return if name.is_a?(Numeric)
|
118
|
-
name = name.keys.first if name.is_a?(Hash)
|
119
|
-
handler_name = "#{name.downcase}_handler"
|
120
|
-
|
121
|
-
if $loaded_handlers.key?(handler_name)
|
122
|
-
return
|
123
|
-
end
|
124
|
-
|
125
|
-
if Kafkr::Consumer.configuration.suggest_handlers
|
126
|
-
if valid_class_name?(name.capitalize)
|
127
|
-
puts "No handler for this message, you could use this one.\n\n"
|
128
|
-
|
129
|
-
handler_class_string = <<~HANDLER_CLASS
|
130
|
-
class #{name.capitalize}Handler < Kafkr::Consumer::Handler
|
131
|
-
def handle?(message)
|
132
|
-
can_handle? message, '#{name}'
|
133
|
-
end
|
134
|
-
|
135
|
-
def handle(message)
|
136
|
-
puts message
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
# Save the file to ./handlers/#{name}_handler.rb
|
141
|
-
HANDLER_CLASS
|
142
|
-
|
143
|
-
puts handler_class_string
|
144
|
-
end
|
145
|
-
end
|
146
|
-
end
|
147
|
-
|
148
|
-
require "timeout"
|
149
|
-
|
150
99
|
def listen_for(message, send_message)
|
151
100
|
attempt = 0
|
152
101
|
begin
|
@@ -170,7 +119,7 @@ module Kafkr
|
|
170
119
|
sleep(wait_time)
|
171
120
|
retry
|
172
121
|
rescue Interrupt
|
173
|
-
|
122
|
+
Kafkr.log "Received interrupt signal. Shutting down consumer gracefully..."
|
174
123
|
socket&.close
|
175
124
|
exit(0)
|
176
125
|
end
|
@@ -202,7 +151,8 @@ module Kafkr
|
|
202
151
|
private
|
203
152
|
|
204
153
|
def dispatch_to_handlers(message)
|
205
|
-
|
154
|
+
|
155
|
+
message_hash = JSON.parse(message)
|
206
156
|
|
207
157
|
self.class.handlers.each do |handler|
|
208
158
|
if handler.handle?(message_hash)
|
@@ -210,8 +160,6 @@ module Kafkr
|
|
210
160
|
end
|
211
161
|
end
|
212
162
|
|
213
|
-
print_handler_class(message)
|
214
|
-
|
215
163
|
yield message_hash if block_given?
|
216
164
|
end
|
217
165
|
end
|
data/lib/kafkr/encryptor.rb
CHANGED
@@ -19,7 +19,7 @@ module Kafkr
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def decrypt(encrypted_data)
|
22
|
-
#
|
22
|
+
# Kafkr.log "Encrypted data before decoding: #{encrypted_data.inspect}"
|
23
23
|
decipher = OpenSSL::Cipher.new(ALGORITHM)
|
24
24
|
decipher.decrypt
|
25
25
|
decipher.key = @key
|
@@ -27,7 +27,7 @@ module Kafkr
|
|
27
27
|
decipher.iv = raw_data[0, decipher.iv_len]
|
28
28
|
decipher.update(raw_data[decipher.iv_len..-1]) + decipher.final
|
29
29
|
rescue OpenSSL::Cipher::CipherError => e
|
30
|
-
|
30
|
+
Kafkr.log "Decryption failed: #{e.message}"
|
31
31
|
nil
|
32
32
|
end
|
33
33
|
end
|
data/lib/kafkr/log.rb
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
require "socket"
|
2
2
|
require "rubygems"
|
3
|
+
require "kafkr"
|
4
|
+
|
3
5
|
|
4
6
|
module Kafkr
|
5
7
|
class Log
|
@@ -27,7 +29,7 @@ module Kafkr
|
|
27
29
|
client_ip = client.peeraddr[3]
|
28
30
|
|
29
31
|
unless whitelisted?(client_ip)
|
30
|
-
|
32
|
+
Kafkr.log "Connection from non-whitelisted IP: #{client_ip}. Ignored."
|
31
33
|
client.close
|
32
34
|
next
|
33
35
|
end
|
@@ -50,13 +52,13 @@ module Kafkr
|
|
50
52
|
decryptor = Kafkr::Encryptor.new
|
51
53
|
message = decryptor.decrypt(encrypted_message.chomp) # Decrypt the message here
|
52
54
|
uuid, message_content = extract_uuid(message)
|
53
|
-
@broker.broadcast(message_content)
|
55
|
+
@broker.broadcast(JSON.dump(message_content))
|
54
56
|
end
|
55
57
|
rescue Errno::ECONNRESET
|
56
58
|
client.close
|
57
59
|
end
|
58
60
|
rescue StandardError => exception
|
59
|
-
|
61
|
+
Kafkr.log "Error: #{exception.message}"
|
60
62
|
end
|
61
63
|
end
|
62
64
|
end
|
data/lib/kafkr/message_broker.rb
CHANGED
data/lib/kafkr/producer.rb
CHANGED
@@ -17,7 +17,7 @@ module Kafkr
|
|
17
17
|
@configuration.queue_file = MESSAGE_QUEUE
|
18
18
|
@configuration.message_queue = []
|
19
19
|
load_queue_from_file
|
20
|
-
@configuration.is_json =
|
20
|
+
@configuration.is_json = true
|
21
21
|
@configuration
|
22
22
|
end
|
23
23
|
|
@@ -27,28 +27,6 @@ module Kafkr
|
|
27
27
|
logger.error("Configuration error: #{e.message}")
|
28
28
|
end
|
29
29
|
|
30
|
-
def self.structured_data_to_hash(input:, sync_uid:)
|
31
|
-
unless /\A\w+\s*(=>|<=>)\s*((\w+:\s*['"]?[^'",]*['"]?,\s*)*(\w+:\s*['"]?[^'",]*['"]?)\s*)\z/.match?(input)
|
32
|
-
return input
|
33
|
-
end
|
34
|
-
|
35
|
-
if input.include?("<=>")
|
36
|
-
type, key_values_str = input.split("<=>").map(&:strip)
|
37
|
-
key_values = key_values_str.scan(/(\w+):\s*['"]?([^'",]*)['"]?/)
|
38
|
-
hash_body = key_values.to_h do |key, value|
|
39
|
-
[key.to_sym, value.strip.gsub(/\A['"]|['"]\z/, "")]
|
40
|
-
end
|
41
|
-
{type.to_sym => hash_body, :sync => true, :sync_uid => sync_uid}
|
42
|
-
else
|
43
|
-
type, key_values_str = input.split("=>").map(&:strip)
|
44
|
-
key_values = key_values_str.scan(/(\w+):\s*['"]?([^'",]*)['"]?/)
|
45
|
-
hash_body = key_values.to_h do |key, value|
|
46
|
-
[key.to_sym, value.strip.gsub(/\A['"]|['"]\z/, "")]
|
47
|
-
end
|
48
|
-
{type.to_sym => hash_body}
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
30
|
def self.send_message(message)
|
53
31
|
return if message.nil? || message.empty?
|
54
32
|
|
@@ -59,15 +37,6 @@ module Kafkr
|
|
59
37
|
json_message = JSON.parse(message)
|
60
38
|
json_message["uuid"] = uuid
|
61
39
|
message_with_uuid = JSON.dump(json_message)
|
62
|
-
else
|
63
|
-
if message.is_a? String
|
64
|
-
message = structured_data_to_hash(input: message, sync_uid: uuid)
|
65
|
-
message_with_uuid = "#{uuid}: #{message}"
|
66
|
-
end
|
67
|
-
|
68
|
-
if message.is_a?(Hash)
|
69
|
-
message_with_uuid = "#{uuid}: #{JSON.generate(message)}"
|
70
|
-
end
|
71
40
|
end
|
72
41
|
|
73
42
|
encrypted_message_with_uuid = Kafkr::Encryptor.new.encrypt(message_with_uuid)
|
@@ -75,13 +44,13 @@ module Kafkr
|
|
75
44
|
begin
|
76
45
|
socket = TCPSocket.new(@configuration.host, @configuration.port)
|
77
46
|
send_queued_messages(socket)
|
78
|
-
socket.puts(encrypted_message_with_uuid)
|
47
|
+
socket.puts (encrypted_message_with_uuid)
|
79
48
|
rescue Errno::ECONNREFUSED
|
80
|
-
|
49
|
+
Kafkr.log "Connection refused. Queuing message: #{encrypted_message_with_uuid}"
|
81
50
|
@configuration.message_queue.push(encrypted_message_with_uuid)
|
82
51
|
save_queue_to_file
|
83
52
|
rescue Errno::EPIPE
|
84
|
-
|
53
|
+
Kafkr.log "Broken pipe error. Retrying connection..."
|
85
54
|
retry_connection(encrypted_message_with_uuid)
|
86
55
|
end
|
87
56
|
|
@@ -126,8 +95,5 @@ module Kafkr
|
|
126
95
|
end
|
127
96
|
end
|
128
97
|
|
129
|
-
def self.logger
|
130
|
-
@logger ||= Logger.new(STDOUT)
|
131
|
-
end
|
132
98
|
end
|
133
99
|
end
|
data/lib/kafkr/version.rb
CHANGED
data/lib/kafkr.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
2
|
+
require "pry"
|
3
3
|
require "logger"
|
4
4
|
require "openssl"
|
5
5
|
require "securerandom"
|
@@ -15,64 +15,21 @@ require_relative "kafkr/version"
|
|
15
15
|
|
16
16
|
module Kafkr
|
17
17
|
class << self
|
18
|
-
attr_accessor :current_environment
|
19
18
|
def logger
|
20
19
|
@logger ||= configure_logger
|
21
20
|
end
|
22
21
|
|
23
|
-
def configure_logger(output =
|
22
|
+
def configure_logger(output = STDOUT)
|
24
23
|
begin
|
25
24
|
@logger = ::Logger.new(output)
|
26
25
|
rescue Errno::EACCES, Errno::ENOENT => e
|
27
26
|
@logger = ::Logger.new(STDOUT)
|
28
27
|
@logger.error("Could not open log file: #{e.message}")
|
29
28
|
end
|
30
|
-
|
29
|
+
@logger.level = ::Logger::DEBUG
|
31
30
|
@logger
|
32
31
|
end
|
33
32
|
|
34
|
-
def default_output
|
35
|
-
case current_environment
|
36
|
-
when "production"
|
37
|
-
"/var/log/kafkr.log"
|
38
|
-
else
|
39
|
-
STDOUT
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def set_logger_level
|
44
|
-
@logger.level = case current_environment
|
45
|
-
when "development"
|
46
|
-
::Logger::DEBUG
|
47
|
-
when "staging"
|
48
|
-
::Logger::INFO
|
49
|
-
when "production"
|
50
|
-
::Logger::WARN
|
51
|
-
else
|
52
|
-
::Logger::DEBUG
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def current_environment
|
57
|
-
@current_environment ||= ENV["KAFKR_ENV"] || "development"
|
58
|
-
end
|
59
|
-
|
60
|
-
def development?
|
61
|
-
current_environment == "development"
|
62
|
-
end
|
63
|
-
|
64
|
-
def test?
|
65
|
-
current_environment == "test"
|
66
|
-
end
|
67
|
-
|
68
|
-
def staging?
|
69
|
-
current_environment == "staging"
|
70
|
-
end
|
71
|
-
|
72
|
-
def production?
|
73
|
-
current_environment == "production"
|
74
|
-
end
|
75
|
-
|
76
33
|
def write(message, unique_id = nil)
|
77
34
|
begin
|
78
35
|
logger.info(message)
|
@@ -100,3 +57,4 @@ module Kafkr
|
|
100
57
|
logger.error("Configuration error: #{e.message}")
|
101
58
|
end
|
102
59
|
end
|
60
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kafkr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.21.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Delaney Kuldvee Burke
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-04-
|
11
|
+
date: 2024-04-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: gibberish
|