karafka-web 0.7.2 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3de31b98509126d85a621683b04f085f0f0f1e7682c28c1c1558920b87021d3d
-  data.tar.gz: 290dcfa64377e48d2ae03dc0495d2b535c32362d3dc82380ddb36b4b284b3679
+  metadata.gz: 7a31ebcde158ca18cc353d3fefad976b7dfb8d9fe3b3114ea8f43b533229434a
+  data.tar.gz: f9870e5268ebebb38c838dc8a220030beaebfe30d4d53cd63537c5c2f442e2d4
 SHA512:
-  metadata.gz: 30ede1f0bed82fd9af59796a7f7e64e2eab29a0965b3e573f6d40c86fd8546c7b0e7dc21f073ccaf0cafbd6cd9f69760488fa70dd758496eea8a8aec43cb638d
-  data.tar.gz: eebb4f6cc73e8259ab2198ca1ccb9c7dd401586a5929b8e4efde8fafffebe50e8d2a0dbae1b5ba894bc3c07beb6cd01b39a343ece56989051c643cae262f8e87
+  metadata.gz: '02152094838c0be1606f49b4963e7d0791f1800bc7b7fbf71240866ed84e73bc264322415b48272faf46c715d70568fc054fc7183406d182d3c95eeb664019db'
+  data.tar.gz: 787a16d7f91f7ccb907a3cbea0599337bd7525b1f2982f023b639227df1b52511afcc763422c71d0e2bf34216d3f3a4fe6794ae8824abeea9302b84071de8ca5

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Karafka Web changelog
 
+## 0.7.4 (2023-09-19)
+- [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
+- [Fix] Fix not working `ps` for macOS.
+
+## 0.7.3 (2023-09-18)
+- [Improvement] Mitigate a case where a race-condition during upgrade would crash data.
+
 ## 0.7.2 (2023-09-18) 
 - [Improvement] Display hidden by accident errors for OSS metrics.
 - [Improvement] Use a five second cache for non-production environments to improve dev experience.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.2)
+    karafka-web (0.7.4)
       erubi (~> 1.4)
       karafka (>= 2.2.3, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)

data/lib/karafka/web/processing/consumer.rb

@@ -26,6 +26,7 @@ module Karafka
 
           # We set this that way so we report with first batch and so we report as fast as possible
           @flushed_at = monotonic_now - @flush_interval
+          @established = false
         end
 
         # Aggregates consumers state into a single current state representation
@@ -34,10 +35,24 @@ module Karafka
 
           # If there is even one incompatible message, we need to stop
           consumers_messages.each do |message|
-            unless @schema_manager.compatible?(message)
+            case @schema_manager.call(message)
+            when :current
+              true
+            when :newer
+              @schema_manager.invalidate!
+
               dispatch
 
               raise ::Karafka::Web::Errors::Processing::IncompatibleSchemaError
+            # Older reports mean someone is in the middle of upgrade. Schema change related
+            # upgrades always should happen without a rolling-upgrade, hence we can reject those
+            # requests without significant or any impact on data quality but without having to
+            # worry about backwards compatibility. Errors are tracked independently, so it should
+            # not be a problem.
+            when :older
+              next
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
             end
 
             # We need to run the aggregations on each message in order to compensate for
@@ -45,6 +60,10 @@ module Karafka
             @state_aggregator.add(message.payload, message.offset)
             @metrics_aggregator.add_report(message.payload)
             @metrics_aggregator.add_stats(@state_aggregator.stats)
+            # Indicates that we had at least one report we used to enrich data
+            # If there were no state changes, there is no reason to flush data. This can occur
+            # when we had some messages but we skipped them for any reason on a first run
+            @established = true
 
             # Optimize memory usage in pro
             message.clean! if Karafka.pro?
@@ -59,17 +78,15 @@ module Karafka
 
         # Flush final state on shutdown
         def shutdown
-          return unless @state_aggregator
-
-          materialize
-          validate!
-          flush
+          dispatch
         end
 
         private
 
         # Flushes the state of the Web-UI to the DB
         def dispatch
+          return unless @established
+
           materialize
           validate!
           flush

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -102,21 +102,21 @@ module Karafka
                       partitions_data = topic_details.fetch(:partitions).values
 
                       lags = partitions_data
-                             .map { |p_details| p_details[:lag] || 0 }
+                             .map { |p_details| p_details.fetch(:lag, -1) }
                              .reject(&:negative?)
 
                       lags_stored = partitions_data
-                                    .map { |p_details| p_details.fetch(:lag_stored) }
+                                    .map { |p_details| p_details.fetch(:lag_stored, -1) }
                                     .reject(&:negative?)
 
                       offsets_hi = partitions_data
-                                   .map { |p_details| p_details.fetch(:hi_offset) }
+                                   .map { |p_details| p_details.fetch(:hi_offset, -1) }
                                    .reject(&:negative?)
 
                       # Last stable offsets freeze durations - we pick the max freeze to indicate
                       # the longest open transaction that potentially may be hanging
                       ls_offsets_fd = partitions_data
-                                      .map { |p_details| p_details.fetch(:ls_offset_fd) }
+                                      .map { |p_details| p_details.fetch(:ls_offset_fd, 0) }
                                       .reject(&:negative?)
 
                       # If there is no lag that would not be negative, it means we did not mark

data/lib/karafka/web/processing/consumers/aggregators/state.rb

@@ -152,8 +152,8 @@ module Karafka
                   stats[:listeners] += report_process[:listeners] || 0
                   stats[:processes] += 1
                   stats[:rss] += report_process[:memory_usage]
-                  stats[:lag] += lags.reject(&:negative?).sum
-                  stats[:lag_stored] += lags_stored.reject(&:negative?).sum
+                  stats[:lag] += lags.compact.reject(&:negative?).sum
+                  stats[:lag_stored] += lags_stored.compact.reject(&:negative?).sum
                   utilization += report_stats[:utilization]
                 end
 

data/lib/karafka/web/processing/consumers/schema_manager.rb

@@ -27,12 +27,8 @@ module Karafka
           end
 
           # @param message [Karafka::Messages::Message] consumer report
-          # @return [Boolean] true if all good or false if incompatible
-          #
-          # @note The state switch is one-direction only. If we encounter an incompatible message
-          #   we need to stop processing so further checks even with valid should not switch it
-          #   back to valid
-          def compatible?(message)
+          # @return [Symbol] is the given message using older, newer or current schema
+          def call(message)
             schema_version = message.payload[:schema_version]
 
             # Save on memory allocation by reusing
@@ -40,8 +36,19 @@ module Karafka
             # an object with each message
             message_version = @cache[schema_version] ||= ::Gem::Version.new(schema_version)
 
-            return true if message_version <= CURRENT_VERSION
+            return :older if message_version < CURRENT_VERSION
+            return :newer if message_version > CURRENT_VERSION
+
+            :current
+          end
 
+          # Moves the schema manager state to incompatible to indicate in the Web-UI that we
+          # cannot move forward because schema is incompatible.
+          #
+          # @note The state switch is one-direction only. If we encounter an incompatible message
+          #   we need to stop processing so further checks even with valid should not switch it
+          #   back to valid
+          def invalidate!
             @valid = false
           end
 

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -247,11 +247,20 @@ module Karafka
           # Loads our ps results into memory so we can extract from them whatever we need
           def memory_threads_ps
             @memory_threads_ps = case RUBY_PLATFORM
-                                 when /darwin|bsd|linux/
+                                 when /linux/
                                    @shell
-                                 .call('ps -A -o rss=,thcount,pid')
+                                 .call('ps -A -o rss=,thcount=,pid=')
                                  .split("\n")
                                  .map { |row| row.strip.split(' ').map(&:to_i) }
+                                 # thcount is not available on macos ps
+                                 # because of that we inject 0 as threads count similar to how
+                                 # we do on windows
+                                 when /darwin|bsd/
+                                   @shell
+                                 .call('ps -A -o rss=,pid=')
+                                 .split("\n")
+                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                 .map { |row| [row.first, 0, row.last] }
                                  else
                                    @memory_threads_ps = false
                                  end

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.2'
+    VERSION = '0.7.4'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-18 00:00:00.000000000 Z
+date: 2023-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi