quake-log-parser 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7bdff44e4bb78e09438ca2c9a2b7112cfcc35362d2e902cf559d52e79ca097b8
-  data.tar.gz: 1bbfd5525b1f8ba829388541564ffe1865a3929fb0fc4c77a7704bacee45effe
+  metadata.gz: a3432357346962f3de62da5245ba27d2ff6034ed78c7692d43999ad4fe13ecb7
+  data.tar.gz: 0d899c7d18cc5d7b45f54367bb5a301d05509ce3acc31272abf8ea886f2087dc
 SHA512:
-  metadata.gz: 501640c4b5278aee65618854cdf4de8ff1c5425a18f73d37ca85c0819ab0f9879add1fa4b02c3d54dee42b7c7d4e31d2c68fc5ca1b225c814da963c3efc343f3
-  data.tar.gz: 6cfc084db65c8d1c2b07157a2cc04bf7189db4cd08c15642a9f771a5a0d9423e57524535ea09fe6c355e20b024e9f0dee8b8fe0905e3aa5ad164241353ca7c8e
+  metadata.gz: 193f317ca950907029fe6b476f45cfad9a264e7390f3e298441b71f4908e799c3d4cb9e503b90b507bad5d72053e4e92335e9c43ae68cf3f3e1d1741d978bbda
+  data.tar.gz: b4b499bea6a19fb2948cf3467183fcd870825ef61ed7638464484c7a2ad54f2e9822be0b3205cfd773fc9b34e24db4548c914a2ea9436e46d1d61cd6296234c0

data/.gitignore

@@ -2,3 +2,5 @@
 /Gemfile.lock
 *.gem
 coverage/
+log/
+.irb-history

data/.rubocop.yml

@@ -7,6 +7,12 @@ AllCops:
 Style/FrozenStringLiteralComment:
   Enabled: false
 
+Style/Next:
+  Enabled: false
+
+Style/SafeNavigation:
+  Enabled: false
+
 Layout/EndOfLine:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [1.1.0]
+
+- Kafka integration
+
 ## [1.0.0]
 
 - Final release to publish the gem

data/Gemfile

@@ -1,3 +1,5 @@
 source 'https://rubygems.org'
 gemspec
 gem 'codecov', require: false, group: :test
+gem 'karafka'
+gem 'karafka-web'

data/README.md

@@ -3,7 +3,6 @@
 [![Maintainability](https://api.codeclimate.com/v1/badges/8bbcb90abf1f392d7e68/maintainability)](https://codeclimate.com/github/pedrofurtado/quake-log-parser/maintainability)
 ![CI](https://github.com/pedrofurtado/quake-log-parser/actions/workflows/ci.yml/badge.svg)
 [![codecov](https://codecov.io/gh/pedrofurtado/quake-log-parser/graph/badge.svg?token=DUC0CORI0N)](https://codecov.io/gh/pedrofurtado/quake-log-parser)
-[![Gem Version](https://badge.fury.io/rb/quake-log-parser.svg)](https://badge.fury.io/rb/quake-log-parser)
 
 Ruby gem for quake log parsing.
 
@@ -23,7 +22,7 @@ Or install it yourself as:
 
     gem install quake-log-parser
 
-## Usage
+## Usage - Log reader
 
 ```ruby
 require 'quake-log-parser'
@@ -40,6 +39,87 @@ parser.read
 puts JSON.pretty_generate(parser.results)
 ```
 
+## Usage - Emulated Kafka topic reader
+
+```ruby
+=begin
+All kafka topic messages must have this structure:
+
+{
+  "lines": [
+    "content of line 01 with slashs escaped",
+    "content of line 02 with slashs escaped",
+    "content of line 03 with slashs escaped",
+    "..."
+  ]
+}
+
+Example of message (with slashs escaped):
+
+{
+  "lines": [
+    "0:00 InitGame: \\sv_floodProtect\\1\\sv_maxPing\\0\\sv_minPing\\0\\sv_maxRate\\10000\\sv_minRate\\0\\sv_hostname\\Code Miner Server\\g_gametype\\0\\sv_privateClients\\2\\sv_maxclients\\16\\sv_allowDownload\\0\\dmflags\\0\fraglimit\\20\\timelimit\\15\\g_maxGameClients\\0\\capturelimit\\8\\version\\ioq3 1.36 linux-x86_64 Apr 12 2009\\protocol\\68\\mapname\\q3dm17\\gamename\baseq3\\g_needpass\\0",
+    "20:34 ClientUserinfoChanged: 2 n\\Isgalamido\\t\\0\\model\\xian/default\\hmodel\\xian/default\\g_redteam\\g_blueteam\\c1\\4\\c2\\5\\hc\\100\\w\\0\\l\\0\\tt\\0\\tl\\0",
+    "20:54 Kill: 1022 2 22: <world> killed Isgalamido by MOD_TRIGGER_HURT",
+    "21:51 ClientUserinfoChanged: 3 n\\Dono da Bola\\t\\0\\model\\sarge/krusade\\hmodel\\sarge/krusade\\g_redteam\\g_blueteam\\c1\\5\\c2\\5\\hc\\95\\w\\0\\l\\0\\tt\\0\\tl\\0",
+    "22:06 Kill: 2 3 7: Isgalamido killed Dono da Bola by MOD_ROCKET_SPLASH"
+  ]
+}
+=end
+
+require 'quake-log-parser'
+require 'ostruct'
+
+if defined?(Rails)
+  QuakeLogParser::Logger.logger = Rails.logger
+else
+  QuakeLogParser::Logger.logger = Logger.new($stdout)
+end
+
+quake_log_lines = [
+  'content of line 01 with slashs escaped',
+  'content of line 02 with slashs escaped',
+  'content of line 03 with slashs escaped',
+  '...'
+]
+
+emulated_messages_from_kafka = [
+  OpenStruct.new(payload: { 'lines' => quake_log_lines })
+]
+
+parser = QuakeLogParser::KafkaTopicReader.new(emulated_messages_from_kafka)
+parser.read
+
+puts JSON.pretty_generate(parser.results)
+```
+
+## Usage - Kafka producer inside Karafka console
+
+```ruby
+# Enter the Karafka console with the command below:
+# docker-compose up --build -d (Wait Kafka components and Karafka consumer to be ready)
+# docker container exec -it quake-log-parser_kafka_consumer_1 bundle exec karafka console
+
+# Then, open a second terminal to see logs of Karafka consumer with the command below:
+# docker container logs -f quake-log-parser_kafka_consumer_1 | grep -i 'quake'
+
+# Then, run the commands below to produce a message to a Kafka topic:
+
+topic_name = 'quake_log_parser_topic'
+key = SecureRandom.uuid
+payload = {
+  "lines" => [
+    "0:00 InitGame: \\sv_floodProtect\\1\\sv_maxPing\\0\\sv_minPing\\0\\sv_maxRate\\10000\\sv_minRate\\0\\sv_hostname\\Code Miner Server\\g_gametype\\0\\sv_privateClients\\2\\sv_maxclients\\16\\sv_allowDownload\\0\\dmflags\\0\fraglimit\\20\\timelimit\\15\\g_maxGameClients\\0\\capturelimit\\8\\version\\ioq3 1.36 linux-x86_64 Apr 12 2009\\protocol\\68\\mapname\\q3dm17\\gamename\baseq3\\g_needpass\\0",
+    "20:34 ClientUserinfoChanged: 2 n\\Isgalamido\\t\\0\\model\\xian/default\\hmodel\\xian/default\\g_redteam\\g_blueteam\\c1\\4\\c2\\5\\hc\\100\\w\\0\\l\\0\\tt\\0\\tl\\0",
+    "20:54 Kill: 1022 2 22: <world> killed Isgalamido by MOD_TRIGGER_HURT",
+    "21:51 ClientUserinfoChanged: 3 n\\Dono da Bola\\t\\0\\model\\sarge/krusade\\hmodel\\sarge/krusade\\g_redteam\\g_blueteam\\c1\\5\\c2\\5\\hc\\95\\w\\0\\l\\0\\tt\\0\\tl\\0",
+    "22:06 Kill: 2 3 7: Isgalamido killed Dono da Bola by MOD_ROCKET_SPLASH"
+  ]
+}
+producer = Karafka.producer.produce_sync(topic: topic_name, payload: payload.to_json, key: key)
+Karafka.logger.info("[QuakeLogParser::KafkaTopicProducer] send_sync topic=#{topic_name} key=#{key} offset=#{producer.offset} partition=#{producer.partition} payload=#{payload}")
+```
+
 ## Execute tests/specs
 
 To execute gem tests locally, use Docker with the commands below:
@@ -56,7 +136,7 @@ docker run --rm -v $(pwd):/app/ -it quake-log-parser_specs
 
 # Or, if you want to run a example of usage of gem,
 # you can run the command below.
-docker run --rm -v $(pwd):/app/ -it quake-log-parser_specs ruby real_example_to_run.rb
+docker run --rm -v $(pwd):/app/ -it quake-log-parser_specs bundle exec ruby real_example_to_run.rb
 ```
 
 ## Contributing

data/docker-compose.yaml

@@ -0,0 +1,49 @@
+version: '3'
+
+services:
+  zookeeper:
+    restart: always
+    image: confluentinc/cp-zookeeper:7.6.0
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+
+  kafka:
+    restart: always
+    image: confluentinc/cp-kafka:7.6.0
+    depends_on:
+      - zookeeper
+    ports:
+      - "4000:9092"
+      - "3004:9094"
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
+      KAFKA_LISTENERS: INTERNAL://:9092,OUTSIDE://:9094
+      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:9092,OUTSIDE://host.docker.internal:9094
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
+
+  control-center:
+    restart: always
+    image: confluentinc/cp-enterprise-control-center:7.6.0
+    hostname: control-center
+    depends_on:
+      - kafka
+    ports:
+      - "9998:9021"
+    environment:
+      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka:9092'
+      CONTROL_CENTER_REPLICATION_FACTOR: 1
+      CONTROL_CENTER_CONNECT_CLUSTER: http://kafka-connect:8083
+      PORT: 9021
+
+  kafka_consumer:
+    depends_on:
+      - kafka
+    restart: always
+    image: ruby:3.1.6
+    working_dir: /app
+    volumes:
+      - ./:/app
+    command: /bin/bash -c "bundle install && bundle exec karafka-web migrate && bundle exec karafka topics migrate && bundle exec karafka server"

data/karafka.rb

@@ -0,0 +1,46 @@
+$stdout.sync = true
+$stderr.sync = true
+
+ENV['KARAFKA_ENV'] ||= 'development'
+Bundler.require(:default, ENV['KARAFKA_ENV'])
+
+require_relative 'lib/quake-log-parser'
+
+class KarafkaApp < Karafka::App
+  setup do |config|
+    config.kafka = {
+      'bootstrap.servers': 'kafka:9092',
+      'allow.auto.create.topics': false
+    }
+    config.client_id = 'quake_log_parser_client'
+    config.initial_offset = 'earliest'
+    config.consumer_persistence = ENV['KARAFKA_ENV'] != 'development'
+  end
+
+  Karafka.monitor.subscribe(Karafka::Instrumentation::LoggerListener.new)
+  Karafka.producer.monitor.subscribe(
+    WaterDrop::Instrumentation::LoggerListener.new(
+      Karafka.logger,
+      log_messages: false
+    )
+  )
+
+  routes.draw do
+    topic :quake_log_parser_topic do
+      config(partitions: 3)
+      consumer QuakeLogParser::KafkaTopicConsumer
+      dead_letter_queue(
+        topic: 'quake_log_parser_dead_letter_topic',
+        max_retries: 0,
+        independent: false
+      )
+    end
+  end
+end
+
+Karafka::Web.setup do |config|
+  config.ui.sessions.secret = 'dda09abf2cbd7d2e171dc44ed11877560a8bbc1ddbf54d2a58b0171634863513'
+  config.tracking.interval = 5_000
+end
+
+Karafka::Web.enable!

data/lib/quake-log-parser/kafka_topic_consumer.rb

@@ -0,0 +1,12 @@
+require 'karafka'
+
+module QuakeLogParser
+  class KafkaTopicConsumer < ::Karafka::BaseConsumer
+    def consume
+      ::Karafka.logger.info "[QuakeLogParser::KafkaTopicConsumer] Consuming #{messages.size} messages from TOPIC #{topic.name}"
+      kafka_topic_reader = ::QuakeLogParser::KafkaTopicReader.new(messages)
+      kafka_topic_reader.read
+      ::Karafka.logger.info "[QuakeLogParser::KafkaTopicConsumer] Results: #{kafka_topic_reader.results}"
+    end
+  end
+end

data/lib/quake-log-parser/kafka_topic_reader.rb

@@ -0,0 +1,29 @@
+module QuakeLogParser
+  class KafkaTopicReader
+    def initialize(messages)
+      @messages = messages
+      @line_handler = QuakeLogParser::LineHandler.new
+    end
+
+    def read
+      @messages.each do |message|
+        if message.payload['lines']
+          message.payload['lines'].each do |line|
+            case line
+            when QuakeLogParser::Patterns.new_game
+              @line_handler.handle_new_game(line)
+            when QuakeLogParser::Patterns.new_player
+              @line_handler.handle_new_player(line)
+            when QuakeLogParser::Patterns.new_kill
+              @line_handler.handle_new_kill(line)
+            end
+          end
+        end
+      end
+    end
+
+    def results
+      @line_handler.results
+    end
+  end
+end

data/lib/quake-log-parser/version.rb

@@ -1,3 +1,3 @@
 module QuakeLogParser
-  VERSION = '1.0.0'.freeze
+  VERSION = '1.1.0'.freeze
 end

data/lib/quake-log-parser.rb

@@ -1,4 +1,6 @@
 require_relative 'quake-log-parser/version'
+require_relative 'quake-log-parser/kafka_topic_reader'
+require_relative 'quake-log-parser/kafka_topic_consumer'
 require_relative 'quake-log-parser/logger'
 require_relative 'quake-log-parser/patterns'
 require_relative 'quake-log-parser/kill'

data/quake-log-parser.gemspec

@@ -21,6 +21,9 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'karafka'
+  spec.add_dependency 'karafka-web'
+  spec.add_development_dependency 'karafka-testing'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: quake-log-parser
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Pedro Furtado
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-09-20 00:00:00.000000000 Z
+date: 2024-09-27 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: karafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: karafka-web
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: karafka-testing
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -75,8 +117,12 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- docker-compose.yaml
+- karafka.rb
 - lib/quake-log-parser.rb
 - lib/quake-log-parser/game.rb
+- lib/quake-log-parser/kafka_topic_consumer.rb
+- lib/quake-log-parser/kafka_topic_reader.rb
 - lib/quake-log-parser/kill.rb
 - lib/quake-log-parser/line_handler.rb
 - lib/quake-log-parser/log_reader.rb