karafka-web 0.7.3 → 0.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dc9572e3dfbb4565361fdbc9521fb7a9a7833664f42bd8e9f0e63e940a397034
-  data.tar.gz: 07fb369bcd0d1aa6c80ef20ff96289989c38ff934a1eb795ab85f39b7adf1228
+  metadata.gz: 7a31ebcde158ca18cc353d3fefad976b7dfb8d9fe3b3114ea8f43b533229434a
+  data.tar.gz: f9870e5268ebebb38c838dc8a220030beaebfe30d4d53cd63537c5c2f442e2d4
 SHA512:
-  metadata.gz: 51bc23c17be963c3b19143aa8034046c0bd85df4251dc9e2e911a8ab28ae194d9ae62e2ab1d3aa0b3e2f8dba33649ad51881d9e96fe669fffbdc971f406ce3eb
-  data.tar.gz: 332247644ed57d25687af816ed850f7847b6eb8e63646bf492460424f0d5855c8dace2f1127d0d84631bffa8f6a9fb71e42a64ece7141a6acb96d5b5f987896c
+  metadata.gz: '02152094838c0be1606f49b4963e7d0791f1800bc7b7fbf71240866ed84e73bc264322415b48272faf46c715d70568fc054fc7183406d182d3c95eeb664019db'
+  data.tar.gz: 787a16d7f91f7ccb907a3cbea0599337bd7525b1f2982f023b639227df1b52511afcc763422c71d0e2bf34216d3f3a4fe6794ae8824abeea9302b84071de8ca5

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Karafka Web changelog
 
+## 0.7.4 (2023-09-19)
+- [Improvement] Skip aggregations on older schemas during upgrades. This only skips process-reports (that are going to be rolled) on the 5s window in case of an upgrade that should not be a rolling one anyhow. This simplifies the operations and minimizes the risk on breaking upgrades.
+- [Fix] Fix not working `ps` for macOS.
+
 ## 0.7.3 (2023-09-18)
 - [Improvement] Mitigate a case where a race-condition during upgrade would crash data.
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka-web (0.7.3)
+    karafka-web (0.7.4)
       erubi (~> 1.4)
       karafka (>= 2.2.3, < 3.0.0)
       karafka-core (>= 2.2.2, < 3.0.0)

data/lib/karafka/web/processing/consumer.rb

@@ -26,6 +26,7 @@ module Karafka
 
           # We set this that way so we report with first batch and so we report as fast as possible
           @flushed_at = monotonic_now - @flush_interval
+          @established = false
         end
 
         # Aggregates consumers state into a single current state representation
@@ -34,10 +35,24 @@ module Karafka
 
           # If there is even one incompatible message, we need to stop
           consumers_messages.each do |message|
-            unless @schema_manager.compatible?(message)
+            case @schema_manager.call(message)
+            when :current
+              true
+            when :newer
+              @schema_manager.invalidate!
+
               dispatch
 
               raise ::Karafka::Web::Errors::Processing::IncompatibleSchemaError
+            # Older reports mean someone is in the middle of upgrade. Schema change related
+            # upgrades always should happen without a rolling-upgrade, hence we can reject those
+            # requests without significant or any impact on data quality but without having to
+            # worry about backwards compatibility. Errors are tracked independently, so it should
+            # not be a problem.
+            when :older
+              next
+            else
+              raise ::Karafka::Errors::UnsupportedCaseError
             end
 
             # We need to run the aggregations on each message in order to compensate for
@@ -45,6 +60,10 @@ module Karafka
             @state_aggregator.add(message.payload, message.offset)
             @metrics_aggregator.add_report(message.payload)
             @metrics_aggregator.add_stats(@state_aggregator.stats)
+            # Indicates that we had at least one report we used to enrich data
+            # If there were no state changes, there is no reason to flush data. This can occur
+            # when we had some messages but we skipped them for any reason on a first run
+            @established = true
 
             # Optimize memory usage in pro
             message.clean! if Karafka.pro?
@@ -59,17 +78,15 @@ module Karafka
 
         # Flush final state on shutdown
         def shutdown
-          return unless @state_aggregator
-
-          materialize
-          validate!
-          flush
+          dispatch
         end
 
         private
 
         # Flushes the state of the Web-UI to the DB
         def dispatch
+          return unless @established
+
           materialize
           validate!
           flush

data/lib/karafka/web/processing/consumers/aggregators/metrics.rb

@@ -102,7 +102,7 @@ module Karafka
                       partitions_data = topic_details.fetch(:partitions).values
 
                       lags = partitions_data
-                             .map { |p_details| p_details[:lag] || 0 }
+                             .map { |p_details| p_details.fetch(:lag, -1) }
                              .reject(&:negative?)
 
                       lags_stored = partitions_data

data/lib/karafka/web/processing/consumers/schema_manager.rb

@@ -27,12 +27,8 @@ module Karafka
           end
 
           # @param message [Karafka::Messages::Message] consumer report
-          # @return [Boolean] true if all good or false if incompatible
-          #
-          # @note The state switch is one-direction only. If we encounter an incompatible message
-          #   we need to stop processing so further checks even with valid should not switch it
-          #   back to valid
-          def compatible?(message)
+          # @return [Symbol] is the given message using older, newer or current schema
+          def call(message)
             schema_version = message.payload[:schema_version]
 
             # Save on memory allocation by reusing
@@ -40,8 +36,19 @@ module Karafka
             # an object with each message
             message_version = @cache[schema_version] ||= ::Gem::Version.new(schema_version)
 
-            return true if message_version <= CURRENT_VERSION
+            return :older if message_version < CURRENT_VERSION
+            return :newer if message_version > CURRENT_VERSION
+
+            :current
+          end
 
+          # Moves the schema manager state to incompatible to indicate in the Web-UI that we
+          # cannot move forward because schema is incompatible.
+          #
+          # @note The state switch is one-direction only. If we encounter an incompatible message
+          #   we need to stop processing so further checks even with valid should not switch it
+          #   back to valid
+          def invalidate!
             @valid = false
           end
 

data/lib/karafka/web/tracking/consumers/sampler.rb

@@ -247,11 +247,20 @@ module Karafka
           # Loads our ps results into memory so we can extract from them whatever we need
           def memory_threads_ps
             @memory_threads_ps = case RUBY_PLATFORM
-                                 when /darwin|bsd|linux/
+                                 when /linux/
                                    @shell
-                                 .call('ps -A -o rss=,thcount,pid')
+                                 .call('ps -A -o rss=,thcount=,pid=')
                                  .split("\n")
                                  .map { |row| row.strip.split(' ').map(&:to_i) }
+                                 # thcount is not available on macos ps
+                                 # because of that we inject 0 as threads count similar to how
+                                 # we do on windows
+                                 when /darwin|bsd/
+                                   @shell
+                                 .call('ps -A -o rss=,pid=')
+                                 .split("\n")
+                                 .map { |row| row.strip.split(' ').map(&:to_i) }
+                                 .map { |row| [row.first, 0, row.last] }
                                  else
                                    @memory_threads_ps = false
                                  end

data/lib/karafka/web/version.rb

@@ -3,6 +3,6 @@
 module Karafka
   module Web
     # Current gem version
-    VERSION = '0.7.3'
+    VERSION = '0.7.4'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-web
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.7.4
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-09-18 00:00:00.000000000 Z
+date: 2023-09-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: erubi