karafka 2.4.18 → 2.5.0.beta1

data/bin/verify_topics_naming

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+# This script verifies that we do not create (except few needed exceptions) test topics that do
+# not start with the "it-" prefix which is our standard.
+#
+# This ensures that we can clearly identify all test topics for removal in case of doing dev work
+# on a long-lived Kafka cluster without option to fully reset it.
+#
+# It also ensures we have one convention that we can follow.
+
+require_relative '../spec/integrations_helper.rb'
+
+setup_karafka
+
+# Please note that "__" starting topics are not here by default. It is expected.
+invalid = Karafka::Admin
+          .cluster_info
+          .topics
+          .map { |topic| topic[:topic_name] }
+          .select { |topic| !topic.start_with?('it-') }
+          .select { |topic| topic.length <= 6 }
+
+invalid.each do |invalid_name|
+  puts "#{invalid_name} does not start with the \"it-\" prefix"
+end
+
+exit invalid.empty? ? 0 : 1

data/config/locales/errors.yml

@@ -24,6 +24,7 @@ en:
       key_must_be_a_symbol: All keys under the kafka settings scope need to be symbols
       max_timeout_vs_pause_max_timeout: pause_timeout must be less or equal to pause_max_timeout
       shutdown_timeout_vs_max_wait_time: shutdown_timeout must be more than max_wait_time
+      worker_thread_priority_format: must be between -3 and 3
 
       oauth.token_provider_listener_format: 'must be false or respond to #on_oauthbearer_token_refresh'
 
@@ -36,6 +37,7 @@ en:
       internal.processing.expansions_selector_format: cannot be nil
       internal.processing.executor_class_format: cannot be nil
       internal.processing.worker_job_call_wrapper_format: 'needs to be false or respond to #wrap'
+      internal.processing.errors_tracker_class_format: 'needs to be nil or a class'
 
       internal.active_job.dispatcher_format: cannot be nil
       internal.active_job.job_options_contract_format: cannot be nil
@@ -67,6 +69,7 @@ en:
       internal.connection.proxy.metadata.timeout_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.max_attempts_format: needs to be an integer bigger than 0
       internal.connection.proxy.metadata.wait_time_format: needs to be an integer bigger than 0
+      internal.connection.listener_thread_priority_format: must be between -3 and 3
 
       internal.swarm.manager_format: cannot be nil
       internal.swarm.orphaned_exit_code_format: needs to be an integer bigger or equal to 0

data/config/locales/pro_errors.yml

@@ -5,6 +5,7 @@ en:
       virtual_partitions.max_partitions_format: needs to be equal or more than 1
       virtual_partitions.offset_metadata_strategy_format: needs to be either :exact or :current
       virtual_partitions.reducer_format: "needs to respond to `#call`"
+      virtual_partitions.distribution_format: "needs to be either :consistent or :balanced"
 
       long_running_job.active_format: needs to be either true or false
 
@@ -58,6 +59,8 @@ en:
       subscription_group_details_multiplexing_boot_mismatch: 'boot needs to be between min and max'
       subscription_group_details.multiplexing_boot_format: 'needs to be an integer equal or more than 1'
       subscription_group_details.multiplexing_boot_not_dynamic: 'needs to be equal to max when not in dynamic mode'
+      subscription_group_details_multiplexing_one_not_enough: 'min and max cannot equal 1'
+      subscription_group_details.multiplexing_scale_delay_format: 'needs to be an integer equal or more than 1000'
 
       swarm.active_format: needs to be true
       swarm.nodes_format: needs to be a range, array of nodes ids or a hash with direct assignments
@@ -88,8 +91,16 @@ en:
       patterns_format: must be an array with hashes
       patterns_missing: needs to be present
       patterns_regexps_not_unique: 'must be unique within consumer group'
+
       direct_assignments_homogenous: 'single consumer group cannot mix regular and direct assignments'
 
+      parallel_segments.partitioner_format: needs to be defined and needs to respond to `#call`
+      parallel_segments.partitioner_respond_to_call: needs to be defined and needs to respond to `#call`
+      parallel_segments.count_format: needs to be equal or more than 1
+      parallel_segments.active_format: needs to be boolean
+      parallel_segments.reducer_format: "needs to respond to `#call`"
+      parallel_segments.merge_key_format: "needs to be a non-empty string"
+
     pattern:
       regexp_format: must be a regular expression
       name_format: 'needs to be a string with a Kafka accepted format'
@@ -113,8 +124,8 @@ en:
 
       recurring_tasks.consumer_class_format: 'needs to inherit from Karafka::BaseConsumer'
       recurring_tasks.group_id_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.schedules_format: 'needs to be a string with a Kafka accepted format'
-      recurring_tasks.topics.logs_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.schedules.name_format: 'needs to be a string with a Kafka accepted format'
+      recurring_tasks.topics.logs.name_format: 'needs to be a string with a Kafka accepted format'
       recurring_tasks.interval_format: 'needs to be equal or more than 1000 and an integer'
       recurring_tasks.deserializer_format: 'needs to be configured'
       recurring_tasks.logging_format: needs to be a boolean

data/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   kafka:
     container_name: kafka
-    image: confluentinc/cp-kafka:7.8.0
+    image: confluentinc/cp-kafka:7.9.1
 
     ports:
       - 9092:9092

data/examples/payloads/json/enrollment_event.json

@@ -0,0 +1,579 @@
+{
+  "event_type": "enrollment",
+  "data": {
+    "tenant_id": "tenant_5",
+    "tenant_dashed_id": "tenant_5",
+    "tenant_name": "Tenant 5",
+    "rc_base_url": "https://tenant-5.example.com",
+    "domain": "tenant-5.example.com",
+    "admin_domain": "tenant-5.admin.example.com",
+    "admin_email": "email@example.com",
+    "country_codes": [
+      "US"
+    ],
+    "cash_currency": "USD",
+    "locales": [
+      "en-US"
+    ],
+    "tenant_locales": [
+      {
+        "locale": "en-US",
+        "enabled": true,
+        "default_for_tenant": true
+      }
+    ],
+    "time_zone": "America/New_York",
+    "provider_config": {
+      "request_phase_url": "https://random.tenant-5.com/example"
+    },
+    "partner_name_for_te": "tenant_InternalId partner",
+    "tenant_code_for_tc": "CODE_FOR_TC",
+    "webhook_resource_app_name_for_tc": "service_4",
+    "tenant_status_for_tc": "live",
+    "nn_internal_domain": "service-4.internal.com",
+    "mc_internal_domain": "service-5.internal.com",
+    "nn_internal_webhook_url": "https://service-4.internal.com/int/webhooks",
+    "loyalty_programs_for_nn": [
+      {
+        "loyalty_program": {
+          "transfer_connect_identifier": "tenant_InternalId",
+          "active": true,
+          "tenant": true,
+          "manual_verification_required": false,
+          "tenant_id": "tenant_InternalId",
+          "name_required": false,
+          "category": "airline",
+          "name": "tenant_InternalId Rewards",
+          "short_name": "tenant_InternalId Rewards",
+          "description": "tenant_InternalId",
+          "translations": {
+            "en-US": {
+              "name": "tenant_InternalId Rewards",
+              "short_name": "tenant_InternalId Rewards"
+            }
+          }
+        },
+        "capability": {},
+        "loyalty_currency": {
+          "name": "Rewards Points",
+          "short_name": "Rewards Points",
+          "translations": {
+            "en-US": {
+              "name": "Rewards Points",
+              "short_name": "Rewards Points"
+            }
+          }
+        }
+      }
+    ],
+    "payment_tier_id": "paid_at_redemption",
+    "demo": true,
+    "feature_settings": {
+      "accrual_campaign": {
+        "enabled": true
+      },
+      "gift_card_campaign": {
+        "enabled": true
+      },
+      "comms_campaign": {
+        "enabled": true
+      }
+    },
+    "ui_settings": {
+      "application": {
+        "components": {
+          "buttons": {
+            "primary": {
+              "borderRadius": "4px",
+              "borderWidth": "1px",
+              "textColor": "#FFFFFF"
+            },
+            "secondary": {
+              "borderRadius": "4px",
+              "borderWidth": "1px",
+              "textColor": "#22285A"
+            }
+          },
+          "breadcrumbs": {
+            "enabled": true
+          },
+          "footer": {
+            "enabled": true,
+            "backgroundColor": "#22285A",
+            "color": "#E3E4EB",
+            "logo": "inverted",
+            "footerLogo": "https://some.random.url.com/logo.png",
+            "hideBranding": false,
+            "textAlign": "text-left",
+            "textDecoration": "no-underline",
+            "textTransform": "none",
+            "variant": "footer-2"
+          },
+          "header": {
+            "enabled": true,
+            "avatar": "primary",
+            "backgroundColor": "#22285A",
+            "color": "#FFFFFF",
+            "logo": "primary",
+            "showLogo": true,
+            "twoLayer": true
+          },
+          "heroBannerAndQuickLinks": {
+            "backgroundColor": "#22285A",
+            "textColor": "#FFFFFF"
+          },
+          "navigation": {
+            "descriptionColor": "#818181",
+            "hoverBackgroundColor": "#00000000"
+          },
+          "spinner": {}
+        },
+        "features": {
+          "cashback": {
+            "description": {
+              "color": "#FFFFFF",
+              "text": " Maximize Your Rewards: Cash in on Incredible Cashback Offers",
+              "transform": "none"
+            },
+            "enabled": false,
+            "listingImageUrl": "https://some-random-url.com/assets/image.jpg",
+            "productImageUrl": "https://some-random-url.com/assets/image.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "Cashback",
+              "transform": "capitalize"
+            }
+          },
+          "crypto": {
+            "description": {
+              "color": "#FFFFFF",
+              "text": "Unlock the Future: Convert Your Events into Stream",
+              "transform": "none"
+            },
+            "enabled": false,
+            "listingImageUrl": "https://some-random-url.com/assets/crypto.jpg",
+            "productImageUrl": "https://some-random-url.com/assets/crypto.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "Crypto",
+              "transform": "capitalize"
+            }
+          },
+          "giftCard": {
+            "description": {
+              "color": "#FFFFFF",
+              "text": "Unlock Endless Possibilities: Redeem Gift Cards for Your Favorite Events",
+              "transform": "none"
+            },
+            "enabled": false,
+            "listingImageUrl": "https://some-random-url.com/assets/gift_card.jpg",
+            "productImageUrl": "https://some-random-url.com/assets/gift_card.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "Gift Card",
+              "transform": "capitalize"
+            }
+          },
+          "offer": {
+            "title": {
+              "text": "Offers",
+              "color": "#FFFFFF",
+              "transform": "capitalize"
+            },
+            "enabled": false,
+            "description": {
+              "text": "Save with free hand-picked coupons, promo codes, discounts & deals.",
+              "color": "#FFFFFF",
+              "transform": "uppercase"
+            },
+            "listingImageUrl": "https://some-random-url.com/assets/offers_banner.jpeg",
+            "productImageUrl": "https://some-random-url.com/assets/offers_banner.jpeg"
+          },
+          "dining": {
+            "enabled": false,
+            "title": {
+              "text": "Premium dining",
+              "color": "#FFFFFF",
+              "transform": "capitalize"
+            },
+            "description": {
+              "text": "Earn points on your purchases when you make payment using an eligible card.",
+              "color": "#FFFFFF",
+              "transform": "uppercase"
+            },
+            "listingImageUrl": "https://some-random-url.com/assets/dining-background.jpeg",
+            "productImageUrl": "https://some-random-url.com/assets/dining-background.jpeg"
+          },
+          "extraCredit": {
+            "title": {
+              "text": "Karafka Credit",
+              "color": "#FFFFFF",
+              "transform": "capitalize"
+            },
+            "enabled": false,
+            "description": {
+              "text": "Turn points into Karafka credit for convenient rides and food delivery",
+              "color": "#FFFFFF",
+              "transform": "uppercase"
+            },
+            "listingImageUrl": "https://some-random-url.com/assets/extra.png",
+            "productImageUrl": "https://some-random-url.com/assets/extra.png"
+          },
+          "pointsTransfer": {
+            "description": {
+              "color": "#FFFFFF",
+              "text": "Extend your Reach: Transfer Points to Unlock New Rewards",
+              "transform": "none"
+            },
+            "enabled": false,
+            "listingImageUrl": "https://some-random-url.com/assets/frequent_traveller_programs.jpg",
+            "productImageUrl": "https://some-random-url.com/assets/frequent_traveller_programs.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "Transfer",
+              "transform": "capitalize"
+            }
+          },
+          "rewardsSummary": {
+            "listingImageUrl": "https://some-random-url.com/assets/reward_summary.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "My Rewards Summary",
+              "transform": "capitalize"
+            },
+            "includeHeadlessRedemptions": true
+          },
+          "sustainability": {
+            "description": {
+              "color": "#FFFFFF",
+              "text": "Rewarding Sustainability: Make Every Point Count for a Better Planet",
+              "transform": "none"
+            },
+            "enabled": false,
+            "listingImageUrl": "https://some-random-url.com/assets/sustainability.jpg",
+            "productImageUrl": "https://some-random-url.com/assets/sustainability.jpg",
+            "title": {
+              "color": "#FFFFFF",
+              "text": "Carbon offset",
+              "transform": "capitalize"
+            }
+          },
+          "termsAndConditions": {
+            "enabled": true
+          },
+          "travel": {
+            "enabled": false,
+            "features": {
+              "cars": {
+                "title": {
+                  "text": "Travel",
+                  "color": "#FFFFFF",
+                  "transform": "none"
+                },
+                "enabled": false,
+                "listingImageUrl": "https://some-random-url.com/assets/banner.jpeg"
+              },
+              "experiences": {
+                "title": {
+                  "text": "Travel",
+                  "color": "#FFFFFF",
+                  "transform": "none"
+                },
+                "enabled": false,
+                "listingImageUrl": "https://some-random-url.com/assets/banner.jpeg"
+              },
+              "hotels": {
+                "title": {
+                  "text": "Travel",
+                  "color": "#FFFFFF",
+                  "transform": "none"
+                },
+                "enabled": false,
+                "listingImageUrl": "https://some-random-url.com/assets/banner.jpeg"
+              },
+              "flights": {
+                "title": {
+                  "text": "Travel",
+                  "color": "#FFFFFF",
+                  "transform": "none"
+                },
+                "enabled": false,
+                "listingImageUrl": "https://some-random-url.com/assets/banner.jpeg"
+              }
+            },
+            "description": {
+              "color": "#FFFFFF",
+              "text": "Book with points and choose from 200 events & 900,000 streams globally",
+              "transform": "none"
+            }
+          },
+          "travelEarn": {
+            "enabled": false
+          },
+          "pointsEarn": {
+            "enabled": false
+          },
+          "helpCenter": {
+            "contactUs": {
+              "externalContactUsEnabled": false,
+              "imageUrl": "https://some-random-url.com/assets/help_center.jpg",
+              "title": {
+                "color": "#FFFFFF",
+                "text": "Contact Us",
+                "transform": "none"
+              }
+            },
+            "customerService": {
+              "imageUrl": "https://some-random-url.com/assets/customer_service.jpg"
+            },
+            "email": {
+              "emailAddress": "cs@example.com",
+              "enabled": true
+            },
+            "listing": {
+              "imageUrl": "https://some-random-url.com/assets/help_center.jpg",
+              "title": {
+                "color": "#FFFFFF",
+                "text": "Help Centre",
+                "transform": "none"
+              }
+            },
+            "phone": {
+              "contactNumber": "0000 999 9999",
+              "enabled": true,
+              "hours": "Mon - Fri (9am-6pm)"
+            },
+            "privacyPolicy": {
+              "imageUrl": "https://some-random-url.com/assets/privacy_policy.jpg"
+            },
+            "termsAndConditions": {
+              "imageUrl": "https://some-random-url.com/assets/terms_of_use.jpg"
+            }
+          },
+          "personalization": {
+            "similarItems": {
+              "giftCardDetails": {
+                "enabled": false
+              },
+              "giftCardCheckout": {
+                "enabled": false
+              },
+              "pointsTransferDetails": {
+                "enabled": false
+              },
+              "pointsTransferCheckout": {
+                "enabled": false
+              },
+              "cashbackDetails": {
+                "enabled": false
+              },
+              "cashbackCheckout": {
+                "enabled": false
+              },
+              "sustainabilityDetails": {
+                "enabled": false
+              },
+              "sustainabilityCheckout": {
+                "enabled": false
+              },
+              "cryptoDetails": {
+                "enabled": false
+              },
+              "cryptoCheckout": {
+                "enabled": false
+              },
+              "voucherDetails": {
+                "enabled": false
+              },
+              "voucherCheckout": {
+                "enabled": false
+              }
+            },
+            "giftCardListing": {
+              "enabled": false
+            },
+            "pointsTransferListing": {
+              "enabled": false
+            }
+          },
+          "appleMerchandise": {
+            "enabled": false
+          },
+          "freeformPointsInput": {
+            "travel": {
+              "enabled": false,
+              "defaultMaxPoints": false
+            }
+          }
+        },
+        "homepage": [
+          {
+            "data": [
+              {
+                "body": {
+                  "content": "Earning loyalty points has never been more rewarding. Redeem them for exclusive events packages and stream the world in style. Start planning your dream cluster now.",
+                  "textTransform": "none"
+                },
+                "image": {
+                  "alt": "third carou image",
+                  "url": "https://some-random-url.com/assets/hero_banner.png"
+                },
+                "title": {
+                  "content": "A world of rewards awaits",
+                  "textTransform": "none"
+                },
+                "config": {
+                  "variant": "marketing-banner-1",
+                  "textAlign": "text-left"
+                }
+              }
+            ],
+            "type": "marketing-banner"
+          },
+          {
+            "type": "redeem-again"
+          },
+          {
+            "type": "spend-x-points"
+          },
+          {
+            "type": "save-up-to-your-points"
+          },
+          {
+            "data": [
+              {
+                "body": {
+                  "content": "Earning loyalty points has never been more rewarding. Redeem them for exclusive events packages and stream the world in style. Start streaming your dreams now.",
+                  "textTransform": "none"
+                },
+                "image": {
+                  "alt": "Earning loyalty points has never been more rewarding.",
+                  "url": "https://some-random-url.com/assets/carousel.png"
+                },
+                "title": {
+                  "content": "Escape to your next adventure with a click of a button.",
+                  "textTransform": "none"
+                },
+                "variant": "carousel-1"
+              }
+            ],
+            "type": "carousel"
+          },
+          {
+            "fallbackData": [],
+            "config": {
+              "title": "Recommended for you"
+            },
+            "type": "personalization"
+          },
+          {
+            "config": {
+              "title": "You'll love these ways to earn points faster",
+              "backgroundColor": "#f6f6f6",
+              "tileBackgroundColor": "#FFFFFF"
+            },
+            "data": [
+              {
+                "icon": null,
+                "id": "pt_0",
+                "image": {
+                  "url": "https://some-random-url.com/assets/frequent_traveller_programs_icon.jpg",
+                  "alt": "Frequent Streamer Program icon"
+                },
+                "productType": "points-transfer",
+                "title": "Frequent Streamer Program",
+                "type": "listing"
+              },
+              {
+                "icon": null,
+                "id": "gc_0",
+                "image": {
+                  "url": "https://some-random-url.com/assets/gift_card_icon.png",
+                  "alt": "Gift Card Icon"
+                },
+                "productType": "gift-card",
+                "title": "Gift Card",
+                "type": "listing"
+              },
+              {
+                "icon": null,
+                "id": "crypto_0",
+                "image": {
+                  "url": "https://some-random-url.com/assets/crypto_icon.jpg",
+                  "alt": "Crypto Icon"
+                },
+                "productType": "crypto",
+                "title": "Crypto",
+                "type": "listing"
+              },
+              {
+                "icon": null,
+                "id": "substain_0",
+                "image": {
+                  "url": "https://some-random-url.com/assets/sustainability_icon.jpg",
+                  "alt": "Sustainability Icon"
+                },
+                "productType": "carbon",
+                "title": "Sustainability",
+                "type": "listing"
+              },
+              {
+                "icon": null,
+                "id": "cashback_0",
+                "image": {
+                  "url": "https://some-random-url.com/assets/cashback_icon.png",
+                  "alt": "Cashback icon"
+                },
+                "productType": "cashback",
+                "title": "Cashback",
+                "type": "listing"
+              }
+            ],
+            "type": "quick-links"
+          },
+          {
+            "config": {
+              "title": "Campaigns for you"
+            },
+            "type": "campaign-banner"
+          }
+        ],
+        "theme": {
+          "bodyFont": "Poppins",
+          "brandLogo": "https://some-random-url.com/assets/alternative_logo.png",
+          "colors": {
+            "accent": "#05A0D1",
+            "primary": "#22285A",
+            "secondary": "#05A0D1"
+          },
+          "favicon": "https://some-random-url.com/assets/favicon.png",
+          "headingFont": "Poppins",
+          "iconStyle": "solid",
+          "invertedLogo": "https://some-random-url.com/assets/alternative_logo.png",
+          "overlayStyle": "#22285A"
+        },
+        "tenantLocales": [
+          {
+            "locale": "en-US",
+            "enabled": true,
+            "defaultForTenant": true
+          }
+        ]
+      },
+      "platform": {
+        "auth": {
+          "appId": "bf9f91ab-ccbd-4a45-a7c8-63893298b71e",
+          "connection": "magic_link",
+          "idleCheckConfig": {
+            "promptTimeout": 120,
+            "timeout": 36000
+          },
+          "scope": "openid,address,email,phone,profile,custom",
+          "sessionCheckPollInterval": 60
+        }
+      }
+    },
+    "is_corporate": false,
+    "configuration_change_id": "0727c699-212a-4c1f-b0d4-e975d44aecab"
+  }
+}