karafka-web 0.10.4 → 0.11.0

data/lib/karafka/web/pro/ui/views/topics/topics/_new_form.erb

@@ -0,0 +1,80 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="card bg-base-100 mt-4">
+  <div class="card-body border-error">
+    <form class="space-y-4 w-full" method="post" action="<%= topics_path %>" data-turbo="false">
+      <%== csrf_tag(topics_path) %>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Topic Name:</label>
+        <div class="w-3/4">
+          <input
+            type="text"
+            name="topic_name"
+            class="block input input-bordered w-full"
+            placeholder="Enter topic name"
+            pattern="[A-Za-z0-9\-_.]+"
+            minlength="1"
+            maxlength="249"
+            value="<%= params.fetch(:topic_name, '') %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Only alphanumeric characters, dots, underscores, and hyphens are allowed
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full mb-8">
+        <label class="w-1/4 text-gray-700">Number of Partitions:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="10000"
+            name="partitions_count"
+            class="block input input-bordered w-full"
+            placeholder="Enter number of partitions"
+            value="<%= params.fetch(:partitions_count, 5) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Minimum 1 partition, cannot be decreased after creation
+          </div>
+        </div>
+      </div>
+
+      <div class="flex items-center w-full">
+        <label class="w-1/4 text-gray-700">Replication Factor:</label>
+        <div class="w-3/4">
+          <input
+            type="number"
+            min="1"
+            max="100"
+            name="replication_factor"
+            class="block input input-bordered w-full"
+            placeholder="Enter replication factor"
+            value="<%= params.fetch(:replication_factor, 1) %>"
+            required
+          />
+          <div class="text-sm text-gray-500 mt-1">
+            Number of replicas for each partition (minimum 1, recommended 3 for production)
+          </div>
+        </div>
+      </div>
+
+      <div class="fieldset text-center mt-6">
+        <div class="flex gap-4 justify-end">
+          <a href="<%= topics_path %>" class="btn btn-ghost">
+            Cancel
+          </a>
+          <button type="submit" class="btn btn-primary gap-2 btn-lockable">
+            <%== icon(:plus) %>
+            Create Topic
+          </button>
+        </div>
+      </div>
+    </form>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/{_tabs.erb → topics/_tabs.erb}

@@ -1,28 +1,31 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
 <div class="tab-container-wrapper">
   <div class="tab-container">
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'config') %>"
+      href="<%= topics_path(@topic.topic_name, 'config') %>"
       class="custom-tab <%= nav_class(end_with: 'config') %>"
     >
       Configuration
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'replication') %>"
+      href="<%= topics_path(@topic.topic_name, 'replication') %>"
       class="custom-tab <%= nav_class(end_with: 'replication') %>"
     >
       Replication
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'distribution') %>"
+      href="<%= topics_path(@topic.topic_name, 'distribution') %>"
       class="custom-tab <%= nav_class(end_with: 'distribution') %>"
     >
       Distribution
     </a>
 
     <a
-      href="<%= root_path('topics', @topic.topic_name, 'offsets') %>"
+      href="<%= topics_path(@topic.topic_name, 'offsets') %>"
       class="custom-tab <%= nav_class(end_with: 'offsets') %>"
     >
       Offsets

data/lib/karafka/web/pro/ui/views/topics/topics/_topic.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<div class="topic-tile">
+  <div class="topic-tile-body">
+    <p class="topic-tile-text">
+      <a href="<%= topics_path(topic.topic_name, 'config') %>" class="topic-tile-link">
+        <%= topic.topic_name %> / <%= topic.partition_count %>
+      </a>
+    </p>
+  </div>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/edit.erb

@@ -0,0 +1,10 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Topic #{@topic.topic_name} Removal Confirmation" %>
+
+<div class="space-y-4">
+  <%== partial 'topics/topics/delete_warning' %>
+  <%== partial 'topics/topics/delete_hints' %>
+  <%== partial 'topics/topics/delete_form' %>
+</div>

data/lib/karafka/web/pro/ui/views/topics/topics/index.erb

@@ -0,0 +1,19 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title 'Topics' %>
+
+<%== partial 'topics/topics/create_button' %>
+
+<% if @topics.empty? %>
+  <%== partial 'explorer/explorer/no_topics' %>
+<% else %>
+  <div class="topic-tiles">
+    <%==
+      each_partial(
+        @topics,
+        'topics/topics/topic'
+      )
+    %>
+  </div>
+<% end %>

data/lib/karafka/web/pro/ui/views/topics/topics/new.erb

@@ -0,0 +1,12 @@
+<%# This code is part of Karafka Pro, a commercial component not licensed under LGPL. %>
+<%# See LICENSE for details. %>
+
+<% view_title "Creating New Topic" %>
+
+<% if @form_error %>
+  <%== partial 'shared/rdkafka_form_error_alert_box' %>
+<% else %>
+  <%== partial 'topics/topics/create_hints' %>
+<% end %>
+
+<%== partial 'topics/topics/new_form' %>

data/lib/karafka/web/processing/consumer.rb

@@ -29,14 +29,14 @@ module Karafka
 
               raise ::Karafka::Web::Errors::Processing::IncompatibleSchemaError
             # Older reports mean someone is in the middle of upgrade. Schema change related
-            # upgrades always should happen without a rolling-upgrade, hence we can reject those
-            # requests without significant or any impact on data quality but without having to
-            # worry about backwards compatibility. Errors are tracked independently, so it should
-            # not be a problem.
-            #
-            # In case user wants to do a rolling upgrade, the user docs state that this can happen
-            # and it is something user should be aware
+            # upgrades always should happen without a rolling-upgrade. For such, since we cannot
+            # reason about their statistics structure, we only track state, so we can provide
+            # basic upgrade reporting details in the status page. All other data is rejected and
+            # since in most cases this is intermediate due to rolling upgrades, this should not
+            # significantly impact the state tracking and processing.
             when :older
+              @state_aggregator.add_state(message.payload, message.offset)
+
               next
             else
               raise ::Karafka::Errors::UnsupportedCaseError

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -36,7 +36,7 @@ module Karafka
             def add(report, offset)
               super(report)
               increment_total_counters(report)
-              update_process_state(report, offset)
+              add_state(report, offset)
               # We always evict after counters updates because we want to use expired (stopped)
               # data for counters as it was valid previously. This can happen only when web consumer
               # had a lag and is catching up.
@@ -46,6 +46,19 @@ module Karafka
               refresh_current_stats
             end
 
+            # Registers or updates the given process state based on the report
+            #
+            # @param report [Hash]
+            # @param offset [Integer]
+            def add_state(report, offset)
+              process_id = report[:process][:id]
+
+              state[:processes][process_id] = {
+                dispatched_at: report[:dispatched_at],
+                offset: offset
+              }
+            end
+
             # @return [Array<Hash, Float>] aggregated current stats value and time from which this
             #   aggregation comes from
             #
@@ -84,19 +97,6 @@ module Karafka
               end
             end
 
-            # Registers or updates the given process state based on the report
-            #
-            # @param report [Hash]
-            # @param offset [Integer]
-            def update_process_state(report, offset)
-              process_id = report[:process][:id]
-
-              state[:processes][process_id] = {
-                dispatched_at: report[:dispatched_at],
-                offset: offset
-              }
-            end
-
             # Evicts expired processes from the current state
             # We consider processes dead if they do not report often enough
             # @note We do not evict based on states (stopped), because we want to report the

data/lib/karafka/web/processing/consumers/metrics.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] latest (current) aggregated metrics state
             def current!
               metrics_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.metrics,
+                Karafka::Web.config.topics.consumers.metrics.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/consumers/state.rb

@@ -12,7 +12,7 @@ module Karafka
             # @return [Hash] last (current) aggregated processes state
             def current!
               state_message = ::Karafka::Admin.read_topic(
-                Karafka::Web.config.topics.consumers.states,
+                Karafka::Web.config.topics.consumers.states.name,
                 0,
                 # We need to take more in case there would be transactions running.
                 # In theory we could take two but this compensates for any involuntary

data/lib/karafka/web/processing/publisher.rb

@@ -36,17 +36,17 @@ module Karafka
           def prepare_data(consumers_state, consumers_metrics)
             [
               {
-                topic: Karafka::Web.config.topics.consumers.states,
+                topic: Karafka::Web.config.topics.consumers.states.name,
                 payload: Zlib::Deflate.deflate(consumers_state.to_json),
                 # This will ensure that the consumer states are compacted
-                key: Karafka::Web.config.topics.consumers.states,
+                key: Karafka::Web.config.topics.consumers.states.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               },
               {
-                topic: Karafka::Web.config.topics.consumers.metrics,
+                topic: Karafka::Web.config.topics.consumers.metrics.name,
                 payload: Zlib::Deflate.deflate(consumers_metrics.to_json),
-                key: Karafka::Web.config.topics.consumers.metrics,
+                key: Karafka::Web.config.topics.consumers.metrics.name,
                 partition: 0,
                 headers: { 'zlib' => 'true' }
               }

data/lib/karafka/web/tracking/consumers/contracts/partition.rb

@@ -28,6 +28,7 @@ module Karafka
             required(:ls_offset) { |val| val.is_a?(Integer) }
             required(:ls_offset_d) { |val| val.is_a?(Integer) }
             required(:ls_offset_fd) { |val| val.is_a?(Integer) && val >= 0 }
+            required(:transactional) { |val| [true, false].include?(val) }
           end
         end
       end

data/lib/karafka/web/tracking/consumers/listeners/errors.rb

@@ -85,6 +85,7 @@ module Karafka
                 # with -1001, which is "N/A" offset position for all the offsets here
                 committed_offset: (consumer.coordinator.seek_offset || -1_000) - 1,
                 consumer: consumer.class.to_s,
+                trace_id: Karafka.pro? ? consumer.errors_tracker.trace_id : nil,
                 tags: consumer.tags
               }
             end

data/lib/karafka/web/tracking/consumers/listeners/pausing.rb

@@ -8,10 +8,10 @@ module Karafka
           # Tracks pausing and un-pausing of topics partitions for both user requested and
           # automatic events.
           class Pausing < Base
-            # Indicate pause
+            # Tracks the pause start
             #
             # @param event [Karafka::Core::Monitoring::Event]
-            def on_consumer_consuming_pause(event)
+            def on_client_pause(event)
               track do |sampler|
                 sampler.pauses[pause_id(event)] = {
                   timeout: event[:timeout],

data/lib/karafka/web/tracking/consumers/listeners/transactions.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      module Consumers
+        module Listeners
+          # Tracks data related to transactions
+          # seek offsets are needed because when consumer offsets are committed in transactions,
+          # librdkafka does not publish the lags in a regular way (they are set to -1) and we need
+          # to compute them via enrichment of information.
+          class Transactions < Base
+            # Tracking of things needed to support transactional consumers post successful
+            # transaction.
+            #
+            # @param event [Karafka::Core::Monitoring::Event]
+            def on_consumer_consuming_transaction(event)
+              consumer = event[:caller]
+              sg_id = consumer.topic.subscription_group.id
+              topic_name = consumer.topic.name
+              # We store it as a string because librdkafka also does that and its easier to align
+              # without casting it later
+              partition_id = consumer.partition
+
+              track do |sampler|
+                break unless sampler.subscription_groups.key?(sg_id)
+
+                seek_offset = consumer.coordinator.seek_offset
+
+                break if seek_offset.nil?
+
+                topics_scope = sampler.subscription_groups[sg_id][:topics]
+                p_scope = topics_scope[topic_name][partition_id]
+
+                p_scope[:transactional] = true
+                p_scope[:seek_offset] = seek_offset
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/consumers/reporter.rb

@@ -63,7 +63,7 @@ module Karafka
               # Report consumers statuses
               messages = [
                 {
-                  topic: ::Karafka::Web.config.topics.consumers.reports,
+                  topic: ::Karafka::Web.config.topics.consumers.reports.name,
                   payload: Zlib::Deflate.deflate(report.to_json),
                   key: process_id,
                   partition: 0,
@@ -76,7 +76,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: Zlib::Deflate.deflate(error.to_json),
                   # Always dispatch errors from the same process to the same partition
                   key: process_id,

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -15,7 +15,7 @@ module Karafka
           # Current schema version
           # This is used for detecting incompatible changes and not using outdated data during
           # upgrades
-          SCHEMA_VERSION = '1.4.0'
+          SCHEMA_VERSION = '1.4.1'
 
           # Counters that count events occurrences during the given window
           COUNTERS_BASE = {
@@ -51,7 +51,17 @@ module Karafka
             @subscription_groups = Hash.new do |h, sg_id|
               h[sg_id] = {
                 id: sg_id,
-                polled_at: monotonic_now
+                polled_at: monotonic_now,
+                topics: Hash.new do |h1, topic|
+                  h1[topic] = Hash.new do |h2, partition|
+                    # We track those details in case we need to fill statistical gaps for
+                    # transactional consumers
+                    h2[partition] = {
+                      seek_offset: -1,
+                      transactional: false
+                    }
+                  end
+                end
               }
             end
 
@@ -223,11 +233,9 @@ module Karafka
           def memory_size
             @memory_size ||= case RUBY_PLATFORM
                              when /linux/
-                               @shell
-                             .call('grep MemTotal /proc/meminfo')
-                             .match(/(\d+)/)
-                             .to_s
-                             .to_i
+                               mem_info = File.read('/proc/meminfo')
+                               mem_total_line = mem_info.match(/MemTotal:\s*(?<total>\d+)/)
+                               mem_total_line['total'].to_i
                              when /darwin|bsd/
                                @shell
                              .call('sysctl -a')
@@ -245,7 +253,13 @@ module Karafka
           # @return [Array<Float>] load averages for last 1, 5 and 15 minutes
           def cpu_usage
             case RUBY_PLATFORM
-            when /darwin|bsd|linux/
+            when /linux/
+              File
+                .read('/proc/loadavg')
+                .split(' ')
+                .first(3)
+                .map(&:to_f)
+            when /darwin|bsd/
               @shell
                 .call('w | head -1')
                 .strip
@@ -280,10 +294,21 @@ module Karafka
           def memory_threads_ps
             @memory_threads_ps = case RUBY_PLATFORM
                                  when /linux/
-                                   @shell
-                                 .call('ps -A -o rss=,thcount=,pid=')
-                                 .split("\n")
-                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                   page_size = Karafka::Web::Tracking::Helpers::Sysconf.page_size
+                                   status_file = "/proc/#{::Process.pid}/status"
+
+                                   pid = status_file.match(%r{/proc/(\d+)/status})[1]
+
+                                   # Extract thread count from /proc/<pid>/status
+                                   thcount = File.read(status_file)[/^Threads:\s+(\d+)/, 1].to_i
+
+                                   # Extract RSS from /proc/<pid>/statm (second field)
+                                   statm_file = "/proc/#{pid}/statm"
+                                   rss_pages = File.read(statm_file).split[1].to_i rescue 0
+                                   # page size is retrieved from Sysconf
+                                   rss_kb = (rss_pages * page_size) / 1024
+
+                                   [[rss_kb, thcount, pid.to_i]]
                                  # thcount is not available on macos ps
                                  # because of that we inject 0 as threads count similar to how
                                  # we do on windows
@@ -308,9 +333,51 @@ module Karafka
                   # This should be always available, since the subscription group polled at time
                   # is first initialized before we start polling, there should be no case where
                   # we have statistics about a given subscription group but we do not have the
-                  # last polling time
-                  polled_at = subscription_groups.fetch(sg_id).fetch(:polled_at)
+                  # sg reference
+                  sg_tracking = subscription_groups.fetch(sg_id)
+
+                  polled_at = sg_tracking.fetch(:polled_at)
                   sg_details[:state][:poll_age] = (monotonic_now - polled_at).round(2)
+
+                  sg_details[:topics].each do |topic_name, topic_details|
+                    topic_details[:partitions].each do |partition_id, partition_details|
+                      # Always assume non-transactional as default. Will be overwritten by the
+                      # consumer level details if collected
+                      partition_details[:transactional] ||= false
+
+                      # If we have stored offset or stored lag, it means it's not a transactional
+                      # consumer at all so we can skip enrichment
+                      next if partition_details[:lag_stored].positive?
+                      next if partition_details[:stored_offset].positive?
+                      next unless sg_tracking[:topics].key?(topic_name)
+                      next unless sg_tracking[:topics][topic_name].key?(partition_id)
+
+                      k_partition_details = sg_tracking[:topics][topic_name][partition_id]
+
+                      # If seek offset was not yey set, nothing to enrich
+                      next unless k_partition_details[:seek_offset].positive?
+
+                      partition_details[:transactional] = k_partition_details[:transactional]
+
+                      # Seek offset is always +1 from the last stored in Karafka
+                      seek_offset = k_partition_details[:seek_offset]
+                      stored_offset = seek_offset - 1
+
+                      # In case of transactions we have to compute the lag ourselves
+                      # -1 because ls offset (or high watermark) is last + 1
+                      lag = partition_details[:ls_offset] - seek_offset
+                      # This can happen if ls_offset is refreshed slower than our stored offset
+                      # fetching from Karafka transactional layer
+                      lag = 0 if lag.negative?
+
+                      partition_details[:lag] = lag
+                      partition_details[:lag_d] = 0
+                      partition_details[:lag_stored] = lag
+                      partition_details[:lag_stored_d] = 0
+                      partition_details[:stored_offset] = stored_offset
+                      partition_details[:committed_offset] = stored_offset
+                    end
+                  end
                 end
               end
             end

data/lib/karafka/web/tracking/helpers/sysconf.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Web
+    module Tracking
+      # Namespace for tracking related helpers
+      module Helpers
+        # Namespace for unix-based helper methods used to fetch OS details
+        module Sysconf
+          extend FFI::Library
+
+          case RUBY_PLATFORM
+          when /linux/
+            ffi_lib 'libc.so.6' # Standard C library on Linux
+            SC_PAGESIZE = 30 # _SC_PAGESIZE constant
+          when /darwin/
+            ffi_lib 'libSystem.B.dylib' # Standard C library on macOS
+            SC_PAGESIZE = 29 # _SC_PAGESIZE constant
+          end
+
+          attach_function :sysconf, [:int], :long
+
+          class << self
+            # @return [Integer]
+            def page_size
+              sysconf(SC_PAGESIZE)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/web/tracking/producers/reporter.rb

@@ -33,7 +33,7 @@ module Karafka
                 @error_contract.validate!(error)
 
                 {
-                  topic: Karafka::Web.config.topics.errors,
+                  topic: Karafka::Web.config.topics.errors.name,
                   payload: error.to_json,
                   # Always dispatch errors from the same process to the same partition
                   key: error[:process][:id]