ci-queue 0.78.0 → 0.80.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68c5b1264396c9d74c0de954e922c38bed02035599ac2a5739f6ff4f5d304ce2
-  data.tar.gz: 9879b14f07961c581cc6ee98b33e97e9e5ea7ce97ad6be8c114ddb946ab9c26e
+  metadata.gz: b8f6b51411e322411c49898ac1a082148ca8b584769d8d3e52ddfd87b11cbb96
+  data.tar.gz: ee4e4beba8f6d2f97400631f7ac7b47ed8561c2af223e369ad9fcdbe88f4c6e3
 SHA512:
-  metadata.gz: f9791de60dc9e636ce6dca98ff370afad264f95437af6d232df651a7c66b607fc4cb2dbfb9955f3e3f558f7cd16e330548f0a9ef2c1fec67ce56e3157ff2accd
-  data.tar.gz: 21a6d847e04e2d8cb9b5e21154d29d41040c9000fd809c65d8946478d8dd6a70358d077411dbaa798d17fc495c77a488486930137342354cb9c63860a78c28d9
+  metadata.gz: d8f03c40ee904cc069300219ffd71262237de5d291eae1ab682289faae8108f90ec127cbd327b5ea1c0afe9a674e493670e3097922f8848bc3757ebbcb8c38f4
+  data.tar.gz: 8c9904ae453b65bd1be76f5cf3c49098f33a2cc1270699ec7569477465ff3c6369931da7e9a2eae789fc1949b07341148d2e8e73276d0362568bd65801fc6723

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.78.0)
+    ci-queue (0.80.0)
       logger
 
 GEM

data/lib/ci/queue/redis/base.rb

@@ -214,7 +214,7 @@ module CI
         end
 
         def key(*args)
-          ['build', build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
 
         def build_id

data/lib/ci/queue/redis/build_record.rb

@@ -139,7 +139,7 @@ module CI
         end
 
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
       end
     end

data/lib/ci/queue/redis/grind_record.rb

@@ -52,7 +52,7 @@ module CI
         attr_reader :redis, :config
 
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
 
         def record_stats(stats, pipeline: redis)

data/lib/ci/queue/redis/key_shortener.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+require 'digest/md5'
+
+module CI
+  module Queue
+    module Redis
+      module KeyShortener
+        # Suffix mapping for common key patterns
+        SUFFIX_ALIASES = {
+          'running' => 'r',
+          'processed' => 'p',
+          'queue' => 'q',
+          'owners' => 'o',
+          'error-reports' => 'e',
+          'requeues-count' => 'rc',
+          'assertions' => 'a',
+          'errors' => 'er',
+          'failures' => 'f',
+          'skips' => 's',
+          'requeues' => 'rq',
+          'total_time' => 't',
+          'test_failed_count' => 'fc',
+          'completed' => 'c',
+          'master-status' => 'm',
+          'created-at' => 'ca',
+          'workers' => 'w',
+          'worker' => 'w',
+          'warnings' => 'wn',
+          'worker-errors' => 'we',
+          'flaky-reports' => 'fl',
+        }.freeze
+
+        # We're transforming the key to a shorter format to minimize network traffic.
+        #
+        # Strategy:
+        # - Shorten prefix: 'b' instead of 'build'
+        # - Hash UUID: 8-char MD5 instead of 36-char UUID
+        # - Alias suffixes: single letters instead of full words
+        #
+        # Example:
+        #   build:unit:019aef0e-c010-433e-b706-c658d3c16372:running (55 bytes)
+        #   -> b:f03d3bef:r (13 bytes, 76% reduction)
+
+        def self.key(build_id, *args)
+          digest = Digest::MD5.hexdigest(build_id)[0..7]
+          shortened_args = args.map { |arg| SUFFIX_ALIASES[arg] || arg }
+
+          ['b', digest, *shortened_args].join(':')
+        end
+      end
+    end
+  end
+end

data/lib/ci/queue/redis.rb

@@ -11,6 +11,7 @@ require 'ci/queue/redis/retry'
 require 'ci/queue/redis/supervisor'
 require 'ci/queue/redis/grind_supervisor'
 require 'ci/queue/redis/test_time_record'
+require 'ci/queue/redis/key_shortener'
 
 module CI
   module Queue

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.78.0'
+    VERSION = '0.80.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -153,7 +153,10 @@ module Minitest
         puts
 
         errors = error_reports
-        puts errors
+        if errors.any?
+          pretty_print_summary(errors)
+          pretty_print_failures(errors)
+        end
 
         build.worker_errors.to_a.sort.each do |worker_id, error|
           puts red("Worker #{worker_id } crashed")
@@ -224,6 +227,37 @@ module Minitest
 
       attr_reader :build, :supervisor
 
+      def pretty_print_summary(errors)
+        test_paths = errors.map(&:test_file).compact
+        return unless test_paths.any?
+
+        file_counts = test_paths.each_with_object(Hash.new(0)) { |path, counts| counts[path] += 1 }
+
+        puts "\n" + "=" * 80
+        puts "FAILED TESTS SUMMARY:"
+        puts "=" * 80
+        file_counts.sort_by { |path, _| path }.each do |path, count|
+          relative_path = Minitest::Queue.relative_path(path)
+          if count == 1
+            puts "  #{relative_path}"
+          else
+            puts "  #{relative_path} (#{count} failures)"
+          end
+        end
+        puts "=" * 80
+      end
+
+      def pretty_print_failures(errors)
+        errors.each_with_index do |error, index|
+          puts "\n" + "-" * 80
+          puts "Error #{index + 1} of #{errors.size}"
+          puts "-" * 80
+          puts error
+        end
+
+        puts "=" * 80
+      end
+
       def timed_out?
         supervisor.time_left.to_i <= 0
       end

data/lib/rspec/queue/build_status_recorder.rb

@@ -1,4 +1,7 @@
 # frozen_string_literal: true
+require 'rspec/queue/failure_formatter'
+require 'rspec/queue/error_report'
+
 module RSpec
   module Queue
     class BuildStatusRecorder
@@ -6,7 +9,9 @@ module RSpec
 
       class << self
         attr_accessor :build
+        attr_accessor :failure_formatter
       end
+      self.failure_formatter = FailureFormatter
 
       def initialize(*)
       end
@@ -18,17 +23,13 @@ module RSpec
 
       def example_failed(notification)
         example = notification.example
-        build.record_error(example.id, [
-          notification.fully_formatted(nil),
-          colorized_rerun_command(example),
-        ].join("\n"))
+        build.record_error(example.id, dump(notification))
       end
 
       private
 
-      def colorized_rerun_command(example, colorizer=::RSpec::Core::Formatters::ConsoleCodes)
-        colorizer.wrap("rspec #{example.location_rerun_argument}", RSpec.configuration.failure_color) + " " +
-        colorizer.wrap("# #{example.full_description}",   RSpec.configuration.detail_color)
+      def dump(notification)
+        ErrorReport.new(self.class.failure_formatter.new(notification).to_h).dump
       end
 
       def build

data/lib/rspec/queue/error_report.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+module RSpec
+  module Queue
+    class ErrorReport
+      class << self
+        attr_accessor :coder
+
+        def load(payload)
+          new(coder.load(payload))
+        end
+      end
+
+      # Default to Marshal
+      self.coder = Marshal
+
+      # Try to use SnappyPack if available from consumer's bundle
+      begin
+        require 'snappy'
+        require 'msgpack'
+        require 'stringio'
+
+        module SnappyPack
+          extend self
+
+          MSGPACK = MessagePack::Factory.new
+          MSGPACK.register_type(0x00, Symbol)
+
+          def load(payload)
+            io = StringIO.new(Snappy.inflate(payload))
+            MSGPACK.unpacker(io).unpack
+          end
+
+          def dump(object)
+            io = StringIO.new
+            packer = MSGPACK.packer(io)
+            packer.pack(object)
+            packer.flush
+            io.rewind
+            Snappy.deflate(io.string).force_encoding(Encoding::UTF_8)
+          end
+        end
+
+        self.coder = SnappyPack
+      rescue LoadError
+      end
+
+      def initialize(data)
+        @data = data
+      end
+
+      def dump
+        self.class.coder.dump(@data)
+      end
+
+      def test_name
+        @data[:test_name]
+      end
+
+      def error_class
+        @data[:error_class]
+      end
+
+      def test_and_module_name
+        @data[:test_and_module_name]
+      end
+
+      def test_suite
+        @data[:test_suite]
+      end
+
+      def test_file
+        @data[:test_file]
+      end
+
+      def test_line
+        @data[:test_line]
+      end
+
+      def to_h
+        @data
+      end
+
+      def to_s
+        output
+      end
+
+      def output
+        @data[:output]
+      end
+    end
+  end
+end

data/lib/rspec/queue/failure_formatter.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+require 'delegate'
+require 'ci/queue/output_helpers'
+
+module RSpec
+  module Queue
+    class FailureFormatter < SimpleDelegator
+      include ::CI::Queue::OutputHelpers
+
+      def initialize(notification)
+        @notification = notification
+        super
+      end
+
+      def to_s
+        [
+          @notification.fully_formatted(nil),
+          colorized_rerun_command(@notification.example)
+        ].join("\n")
+      end
+
+      def to_h
+        example = @notification.example
+        {
+          test_file: example.file_path,
+          test_line: example.metadata[:line_number],
+          test_and_module_name: example.id,
+          test_name: example.description,
+          test_suite: example.example_group.description,
+          error_class: @notification.exception.class.name,
+          output: to_s,
+        }
+      end
+
+      private
+
+      attr_reader :notification
+
+      def colorized_rerun_command(example, colorizer=::RSpec::Core::Formatters::ConsoleCodes)
+        colorizer.wrap("rspec #{example.location_rerun_argument}", RSpec.configuration.failure_color) + " " +
+        colorizer.wrap("# #{example.full_description}", RSpec.configuration.detail_color)
+      end
+    end
+  end
+end

data/lib/rspec/queue.rb

@@ -5,6 +5,7 @@ require 'rspec/core'
 require 'ci/queue'
 require 'rspec/queue/build_status_recorder'
 require 'rspec/queue/order_recorder'
+require 'rspec/queue/error_report'
 
 module RSpec
   module Queue
@@ -291,16 +292,56 @@ module RSpec
           end
         end
 
-        # TODO: better reporting
-        errors = supervisor.build.error_reports.sort_by(&:first).map(&:last)
+        errors = supervisor.build.error_reports.sort_by(&:first).map do |_, error_data|
+            RSpec::Queue::ErrorReport.load(error_data)
+        end
         if errors.empty?
           step(green('No errors found'))
           0
         else
           message = errors.size == 1 ? "1 error found" : "#{errors.size} errors found"
           step(red(message), collapsed: false)
-          puts errors
+
+          pretty_print_summary(errors)
+          pretty_print_failures(errors)
           1
+          # Example output
+          #
+          # FAILED TESTS SUMMARY:
+          # =================================================================================
+          #   ./spec/dummy_spec.rb
+          #   ./spec/dummy_spec_2.rb (2 failures)
+          #   ./spec/dummy_spec_3.rb (3 failures)
+          # =================================================================================
+          #
+          # --------------------------------------------------------------------------------
+          # Error 1 of 3
+          # --------------------------------------------------------------------------------
+          #
+          #   Object doesn't work on first try
+          #   Failure/Error: expect(1 + 1).to be == 42
+          #
+          #     expected: == 42
+          #      got:    2
+          #
+          # --- stacktrace will be here ---
+          # --- rerun command will be here ---
+          #
+          # --------------------------------------------------------------------------------
+          # Error 2 of 3
+          # --------------------------------------------------------------------------------
+          #
+          #   Object doesn't work on first try
+          #   Failure/Error: expect(1 + 1).to be == 42
+          #
+          #     expected: == 42
+          #      got:    2
+          #
+          # --- stacktrace will be here ---
+          # --- rerun command will be here ---
+          #
+          #  ... etc
+          # =================================================================================
         end
       end
 
@@ -320,6 +361,38 @@ module RSpec
         invalid_usage!('Missing --queue parameter') unless queue_url
         invalid_usage!('Missing --build parameter') unless RSpec::Queue.config.build_id
       end
+
+      private
+
+      def pretty_print_summary(errors)
+        test_paths = errors.map(&:test_file).compact
+        return unless test_paths.any?
+
+        file_counts = test_paths.each_with_object(Hash.new(0)) { |path, counts| counts[path] += 1 }
+
+        puts "\n" + "=" * 80
+        puts "FAILED TESTS SUMMARY:"
+        puts "=" * 80
+        file_counts.sort_by { |path, _| path }.each do |path, count|
+          if count == 1
+            puts "  #{path}"
+          else
+            puts "  #{path} (#{count} failures)"
+          end
+        end
+        puts "=" * 80
+      end
+
+      def pretty_print_failures(errors)
+        errors.each_with_index do |error, index|
+          puts "\n" + "-" * 80
+          puts "Error #{index + 1} of #{errors.size}"
+          puts "-" * 80
+          puts error.to_s
+        end
+
+        puts "=" * 80
+      end
     end
 
     class QueueReporter < SimpleDelegator
@@ -362,7 +435,18 @@ module RSpec
         invalid_usage!('Missing --queue parameter') unless queue_url
         invalid_usage!('Missing --build parameter') unless RSpec::Queue.config.build_id
         invalid_usage!('Missing --worker parameter') unless RSpec::Queue.config.worker_id
-        RSpec.configuration.backtrace_formatter.filter_gem('ci-queue')
+        RSpec.configure do |config|
+          config.backtrace_exclusion_patterns = [
+            # Filter bundler paths
+            %r{/tmp/bundle/},
+            # RSpec internals
+            %r{/gems/rspec-},
+            # ci-queue and rspec-queue internals
+            %r{exe/rspec-queue},
+            %r{lib/ci/queue/},
+            %r{rspec/queue}
+          ]
+        end
       end
 
       def run_specs(example_groups)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.78.0
+  version: 0.80.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -203,6 +203,7 @@ files:
 - lib/ci/queue/redis/grind_record.rb
 - lib/ci/queue/redis/grind_supervisor.rb
 - lib/ci/queue/redis/heartbeat.lua
+- lib/ci/queue/redis/key_shortener.rb
 - lib/ci/queue/redis/monitor.rb
 - lib/ci/queue/redis/release.lua
 - lib/ci/queue/redis/requeue.lua
@@ -234,6 +235,8 @@ files:
 - lib/minitest/reporters/statsd_reporter.rb
 - lib/rspec/queue.rb
 - lib/rspec/queue/build_status_recorder.rb
+- lib/rspec/queue/error_report.rb
+- lib/rspec/queue/failure_formatter.rb
 - lib/rspec/queue/order_recorder.rb
 homepage: https://github.com/Shopify/ci-queue
 licenses: