@platformatic/rdkafka 4.0.1 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/bench/seed.sh +0 -0
- package/binding.gyp +8 -1
- package/ci/prepublish.js +0 -1
- package/config.d.ts +126 -10
- package/configure +0 -0
- package/deps/librdkafka/.semaphore/run-all-tests.yml +77 -0
- package/deps/librdkafka/.semaphore/semaphore-integration.yml +250 -0
- package/deps/librdkafka/.semaphore/semaphore.yml +67 -44
- package/deps/librdkafka/.semaphore/verify-linux-packages.yml +41 -0
- package/deps/librdkafka/CHANGELOG.md +471 -1
- package/deps/librdkafka/CONFIGURATION.md +29 -9
- package/deps/librdkafka/INTRODUCTION.md +909 -601
- package/deps/librdkafka/LICENSES.txt +592 -1
- package/deps/librdkafka/Makefile +2 -1
- package/deps/librdkafka/_mkltmpuuWEwa.c +13 -0
- package/deps/librdkafka/configure +0 -0
- package/deps/librdkafka/debian/rules +0 -0
- package/deps/librdkafka/dev-conf.sh +0 -0
- package/deps/librdkafka/examples/consumer.c +18 -8
- package/deps/librdkafka/examples/describe_consumer_groups.c +25 -6
- package/deps/librdkafka/examples/incremental_alter_configs.c +4 -5
- package/deps/librdkafka/examples/kafkatest_verifiable_client.cpp +26 -42
- package/deps/librdkafka/examples/openssl_engine_example.cpp +3 -4
- package/deps/librdkafka/examples/producer.cpp +2 -2
- package/deps/librdkafka/examples/user_scram.c +4 -5
- package/deps/librdkafka/examples/win_ssl_cert_store.cpp +4 -3
- package/deps/librdkafka/lds-gen.py +0 -0
- package/deps/librdkafka/mklove/Makefile.base +0 -0
- package/deps/librdkafka/packaging/RELEASE.md +8 -0
- package/deps/librdkafka/packaging/alpine/build-alpine.sh +0 -0
- package/deps/librdkafka/packaging/cp/README.md +4 -1
- package/deps/librdkafka/packaging/cp/check_features.c +12 -4
- package/deps/librdkafka/packaging/cp/verify-deb.sh +6 -6
- package/deps/librdkafka/packaging/cp/verify-packages.sh +52 -31
- package/deps/librdkafka/packaging/cp/verify-rpm.sh +6 -13
- package/deps/librdkafka/packaging/debian/rules +0 -0
- package/deps/librdkafka/packaging/get_version.py +0 -0
- package/deps/librdkafka/packaging/homebrew/brew-update-pr.sh +0 -0
- package/deps/librdkafka/packaging/mingw-w64/{travis-before-install.sh → export-variables.sh} +0 -7
- package/deps/librdkafka/packaging/mingw-w64/semaphoreci-build.sh +1 -1
- package/deps/librdkafka/packaging/nuget/artifact.py +0 -0
- package/deps/librdkafka/packaging/nuget/cleanup-s3.py +0 -0
- package/deps/librdkafka/packaging/nuget/nuget.sh +0 -0
- package/deps/librdkafka/packaging/nuget/packaging.py +0 -0
- package/deps/librdkafka/packaging/nuget/push-to-nuget.sh +0 -0
- package/deps/librdkafka/packaging/nuget/release.py +0 -0
- package/deps/librdkafka/packaging/nuget/requirements.txt +3 -3
- package/deps/librdkafka/packaging/rpm/mock-on-docker.sh +0 -0
- package/deps/librdkafka/packaging/rpm/tests/run-test.sh +0 -0
- package/deps/librdkafka/packaging/rpm/tests/test-on-docker.sh +0 -0
- package/deps/librdkafka/packaging/tools/build-configurations-checks.sh +3 -1
- package/deps/librdkafka/packaging/tools/build-deb-package.sh +0 -0
- package/deps/librdkafka/packaging/tools/build-debian.sh +0 -0
- package/deps/librdkafka/packaging/tools/build-manylinux.sh +0 -0
- package/deps/librdkafka/packaging/tools/build-release-artifacts.sh +0 -0
- package/deps/librdkafka/packaging/tools/distro-build.sh +0 -0
- package/deps/librdkafka/packaging/tools/gh-release-checksums.py +0 -0
- package/deps/librdkafka/packaging/tools/rdutcoverage.sh +0 -0
- package/deps/librdkafka/packaging/tools/run-in-docker.sh +28 -0
- package/deps/librdkafka/packaging/tools/run-integration-tests.sh +31 -0
- package/deps/librdkafka/packaging/tools/run-style-check.sh +4 -0
- package/deps/librdkafka/packaging/tools/style-format.sh +6 -5
- package/deps/librdkafka/packaging/tools/update_rpcs_max_versions.py +100 -0
- package/deps/librdkafka/service.yml +154 -0
- package/deps/librdkafka/src/generate_proto.sh +0 -0
- package/deps/librdkafka/src/rd.h +145 -2
- package/deps/librdkafka/src/rdaddr.h +6 -7
- package/deps/librdkafka/src/rdatomic.h +15 -11
- package/deps/librdkafka/src/rdavg.h +3 -2
- package/deps/librdkafka/src/rdbase64.c +33 -2
- package/deps/librdkafka/src/rdbase64.h +2 -0
- package/deps/librdkafka/src/rdendian.h +10 -9
- package/deps/librdkafka/src/rdhttp.c +418 -100
- package/deps/librdkafka/src/rdhttp.h +23 -5
- package/deps/librdkafka/src/rdkafka.c +299 -145
- package/deps/librdkafka/src/rdkafka.h +89 -6
- package/deps/librdkafka/src/rdkafka_admin.c +482 -105
- package/deps/librdkafka/src/rdkafka_admin.h +42 -0
- package/deps/librdkafka/src/rdkafka_broker.c +463 -312
- package/deps/librdkafka/src/rdkafka_broker.h +68 -10
- package/deps/librdkafka/src/rdkafka_buf.c +8 -5
- package/deps/librdkafka/src/rdkafka_buf.h +4 -3
- package/deps/librdkafka/src/rdkafka_cert.c +25 -1
- package/deps/librdkafka/src/rdkafka_cert.h +4 -3
- package/deps/librdkafka/src/rdkafka_cgrp.c +468 -143
- package/deps/librdkafka/src/rdkafka_cgrp.h +36 -2
- package/deps/librdkafka/src/rdkafka_conf.c +526 -102
- package/deps/librdkafka/src/rdkafka_conf.h +68 -4
- package/deps/librdkafka/src/rdkafka_feature.c +472 -35
- package/deps/librdkafka/src/rdkafka_feature.h +2 -0
- package/deps/librdkafka/src/rdkafka_fetcher.c +22 -2
- package/deps/librdkafka/src/rdkafka_fetcher.h +2 -2
- package/deps/librdkafka/src/rdkafka_int.h +84 -46
- package/deps/librdkafka/src/rdkafka_metadata.c +221 -136
- package/deps/librdkafka/src/rdkafka_metadata.h +15 -11
- package/deps/librdkafka/src/rdkafka_metadata_cache.c +279 -73
- package/deps/librdkafka/src/rdkafka_mock.c +795 -19
- package/deps/librdkafka/src/rdkafka_mock.h +119 -0
- package/deps/librdkafka/src/rdkafka_mock_cgrp.c +1298 -132
- package/deps/librdkafka/src/rdkafka_mock_handlers.c +345 -49
- package/deps/librdkafka/src/rdkafka_mock_int.h +180 -61
- package/deps/librdkafka/src/rdkafka_msg.c +35 -19
- package/deps/librdkafka/src/rdkafka_msg.h +3 -3
- package/deps/librdkafka/src/rdkafka_msgset_writer.c +14 -10
- package/deps/librdkafka/src/rdkafka_offset.c +33 -5
- package/deps/librdkafka/src/rdkafka_op.c +1 -1
- package/deps/librdkafka/src/rdkafka_op.h +45 -21
- package/deps/librdkafka/src/rdkafka_partition.c +118 -23
- package/deps/librdkafka/src/rdkafka_partition.h +16 -7
- package/deps/librdkafka/src/rdkafka_proto.h +4 -1
- package/deps/librdkafka/src/rdkafka_protocol.h +1 -0
- package/deps/librdkafka/src/rdkafka_queue.c +125 -48
- package/deps/librdkafka/src/rdkafka_queue.h +19 -0
- package/deps/librdkafka/src/rdkafka_range_assignor.c +1 -1
- package/deps/librdkafka/src/rdkafka_request.c +336 -64
- package/deps/librdkafka/src/rdkafka_request.h +24 -1
- package/deps/librdkafka/src/rdkafka_sasl.c +5 -3
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer.c +14 -10
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.c +1166 -89
- package/deps/librdkafka/src/rdkafka_sasl_oauthbearer_oidc.h +13 -3
- package/deps/librdkafka/src/rdkafka_sasl_plain.c +3 -3
- package/deps/librdkafka/src/rdkafka_sasl_scram.c +4 -3
- package/deps/librdkafka/src/rdkafka_ssl.c +306 -81
- package/deps/librdkafka/src/rdkafka_ssl.h +22 -0
- package/deps/librdkafka/src/rdkafka_sticky_assignor.c +4 -0
- package/deps/librdkafka/src/rdkafka_telemetry.c +80 -23
- package/deps/librdkafka/src/rdkafka_telemetry_encode.c +5 -1
- package/deps/librdkafka/src/rdkafka_telemetry_encode.h +20 -20
- package/deps/librdkafka/src/rdkafka_topic.c +220 -137
- package/deps/librdkafka/src/rdkafka_topic.h +14 -8
- package/deps/librdkafka/src/rdkafka_transport.c +59 -47
- package/deps/librdkafka/src/rdkafka_transport.h +5 -0
- package/deps/librdkafka/src/rdkafka_txnmgr.c +6 -1
- package/deps/librdkafka/src/rdmap.c +6 -1
- package/deps/librdkafka/src/rdrand.c +11 -1
- package/deps/librdkafka/src/rdtime.h +48 -17
- package/deps/librdkafka/src/rdunittest.c +42 -25
- package/deps/librdkafka/src/rdunittest.h +2 -0
- package/deps/librdkafka/src/rdwin32.h +1 -1
- package/deps/librdkafka/src/regexp.c +47 -42
- package/deps/librdkafka/src/tinycthread_extra.c +45 -21
- package/deps/librdkafka/src/tinycthread_extra.h +7 -3
- package/deps/librdkafka/src-cpp/rdkafkacpp.h +28 -2
- package/deps/librdkafka/tests/0001-multiobj.c +4 -0
- package/deps/librdkafka/tests/0004-conf.c +103 -36
- package/deps/librdkafka/tests/0009-mock_cluster.c +1 -0
- package/deps/librdkafka/tests/0011-produce_batch.c +2 -1
- package/deps/librdkafka/tests/0016-client_swname.c +18 -3
- package/deps/librdkafka/tests/0017-compression.c +6 -8
- package/deps/librdkafka/tests/0018-cgrp_term.c +8 -3
- package/deps/librdkafka/tests/0021-rkt_destroy.c +1 -0
- package/deps/librdkafka/tests/0022-consume_batch.c +4 -1
- package/deps/librdkafka/tests/0026-consume_pause.c +12 -14
- package/deps/librdkafka/tests/0028-long_topicnames.c +1 -1
- package/deps/librdkafka/tests/0029-assign_offset.c +6 -8
- package/deps/librdkafka/tests/0030-offset_commit.c +2 -2
- package/deps/librdkafka/tests/0033-regex_subscribe.c +33 -17
- package/deps/librdkafka/tests/0034-offset_reset.c +25 -4
- package/deps/librdkafka/tests/0036-partial_fetch.c +1 -0
- package/deps/librdkafka/tests/0038-performance.c +1 -0
- package/deps/librdkafka/tests/0040-io_event.c +11 -5
- package/deps/librdkafka/tests/0041-fetch_max_bytes.c +1 -0
- package/deps/librdkafka/tests/0044-partition_cnt.c +3 -2
- package/deps/librdkafka/tests/0045-subscribe_update.c +284 -20
- package/deps/librdkafka/tests/0047-partial_buf_tmout.c +1 -0
- package/deps/librdkafka/tests/0048-partitioner.c +1 -1
- package/deps/librdkafka/tests/0050-subscribe_adds.c +1 -0
- package/deps/librdkafka/tests/0051-assign_adds.c +1 -0
- package/deps/librdkafka/tests/0052-msg_timestamps.c +22 -4
- package/deps/librdkafka/tests/0053-stats_cb.cpp +12 -20
- package/deps/librdkafka/tests/0055-producer_latency.c +176 -3
- package/deps/librdkafka/tests/0056-balanced_group_mt.c +6 -2
- package/deps/librdkafka/tests/0058-log.cpp +3 -3
- package/deps/librdkafka/tests/0059-bsearch.cpp +8 -4
- package/deps/librdkafka/tests/0061-consumer_lag.cpp +25 -5
- package/deps/librdkafka/tests/0067-empty_topic.cpp +3 -0
- package/deps/librdkafka/tests/0068-produce_timeout.c +21 -23
- package/deps/librdkafka/tests/0069-consumer_add_parts.c +3 -7
- package/deps/librdkafka/tests/0075-retry.c +38 -1
- package/deps/librdkafka/tests/0076-produce_retry.c +2 -0
- package/deps/librdkafka/tests/0077-compaction.c +8 -2
- package/deps/librdkafka/tests/0080-admin_ut.c +34 -4
- package/deps/librdkafka/tests/0081-admin.c +455 -155
- package/deps/librdkafka/tests/0082-fetch_max_bytes.cpp +4 -0
- package/deps/librdkafka/tests/0083-cb_event.c +8 -3
- package/deps/librdkafka/tests/0084-destroy_flags.c +14 -18
- package/deps/librdkafka/tests/0085-headers.cpp +4 -0
- package/deps/librdkafka/tests/0086-purge.c +40 -7
- package/deps/librdkafka/tests/0088-produce_metadata_timeout.c +1 -1
- package/deps/librdkafka/tests/0089-max_poll_interval.c +12 -7
- package/deps/librdkafka/tests/0090-idempotence.c +1 -2
- package/deps/librdkafka/tests/0091-max_poll_interval_timeout.c +5 -7
- package/deps/librdkafka/tests/0092-mixed_msgver.c +6 -0
- package/deps/librdkafka/tests/0093-holb.c +8 -5
- package/deps/librdkafka/tests/0094-idempotence_msg_timeout.c +1 -0
- package/deps/librdkafka/tests/0097-ssl_verify.cpp +262 -70
- package/deps/librdkafka/tests/0098-consumer-txn.cpp +16 -16
- package/deps/librdkafka/tests/0099-commit_metadata.c +1 -1
- package/deps/librdkafka/tests/0101-fetch-from-follower.cpp +1 -1
- package/deps/librdkafka/tests/0102-static_group_rebalance.c +317 -16
- package/deps/librdkafka/tests/0103-transactions.c +7 -7
- package/deps/librdkafka/tests/0104-fetch_from_follower_mock.c +1 -0
- package/deps/librdkafka/tests/0105-transactions_mock.c +18 -11
- package/deps/librdkafka/tests/0106-cgrp_sess_timeout.c +35 -14
- package/deps/librdkafka/tests/0107-topic_recreate.c +2 -2
- package/deps/librdkafka/tests/0109-auto_create_topics.cpp +86 -26
- package/deps/librdkafka/tests/0110-batch_size.cpp +1 -2
- package/deps/librdkafka/tests/0112-assign_unknown_part.c +3 -14
- package/deps/librdkafka/tests/0113-cooperative_rebalance.cpp +248 -104
- package/deps/librdkafka/tests/0114-sticky_partitioning.cpp +3 -3
- package/deps/librdkafka/tests/0115-producer_auth.cpp +4 -1
- package/deps/librdkafka/tests/0116-kafkaconsumer_close.cpp +7 -5
- package/deps/librdkafka/tests/0117-mock_errors.c +15 -5
- package/deps/librdkafka/tests/0118-commit_rebalance.c +41 -8
- package/deps/librdkafka/tests/0119-consumer_auth.cpp +20 -1
- package/deps/librdkafka/tests/0120-asymmetric_subscription.c +8 -3
- package/deps/librdkafka/tests/0125-immediate_flush.c +1 -1
- package/deps/librdkafka/tests/0126-oauthbearer_oidc.c +323 -8
- package/deps/librdkafka/tests/0129-fetch_aborted_msgs.c +1 -0
- package/deps/librdkafka/tests/0132-strategy_ordering.c +12 -4
- package/deps/librdkafka/tests/0133-ssl_keys.c +28 -6
- package/deps/librdkafka/tests/0137-barrier_batch_consume.c +24 -14
- package/deps/librdkafka/tests/0139-offset_validation_mock.c +510 -2
- package/deps/librdkafka/tests/0140-commit_metadata.cpp +1 -1
- package/deps/librdkafka/tests/0142-reauthentication.c +37 -17
- package/deps/librdkafka/tests/0143-exponential_backoff_mock.c +39 -40
- package/deps/librdkafka/tests/0146-metadata_mock.c +76 -15
- package/deps/librdkafka/tests/0147-consumer_group_consumer_mock.c +952 -0
- package/deps/librdkafka/tests/0148-offset_fetch_commit_error_mock.c +563 -0
- package/deps/librdkafka/tests/0149-broker-same-host-port.c +140 -0
- package/deps/librdkafka/tests/0150-telemetry_mock.c +9 -6
- package/deps/librdkafka/tests/0151-purge-brokers.c +566 -0
- package/deps/librdkafka/tests/0152-rebootstrap.c +59 -0
- package/deps/librdkafka/tests/0153-memberid.c +128 -0
- package/deps/librdkafka/tests/CMakeLists.txt +6 -0
- package/deps/librdkafka/tests/LibrdkafkaTestApp.py +34 -9
- package/deps/librdkafka/tests/autotest.sh +0 -0
- package/deps/librdkafka/tests/broker_version_tests.py +0 -0
- package/deps/librdkafka/tests/buildbox.sh +0 -0
- package/deps/librdkafka/tests/cleanup-checker-tests.sh +0 -0
- package/deps/librdkafka/tests/cluster_testing.py +0 -0
- package/deps/librdkafka/tests/delete-test-topics.sh +0 -0
- package/deps/librdkafka/tests/fixtures/oauthbearer/jwt_assertion_template.json +10 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.intermediate.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client.keystore.p12 +0 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.intermediate.pem +72 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.certificate.pem +42 -101
- package/deps/librdkafka/tests/fixtures/ssl/client2.intermediate.key +46 -0
- package/deps/librdkafka/tests/fixtures/ssl/client2.key +41 -29
- package/deps/librdkafka/tests/fixtures/ssl/create_keys.sh +112 -37
- package/deps/librdkafka/tests/gen-ssl-certs.sh +0 -0
- package/deps/librdkafka/tests/interactive_broker_version.py +0 -0
- package/deps/librdkafka/tests/java/Makefile +2 -1
- package/deps/librdkafka/tests/java/run-class.sh +0 -0
- package/deps/librdkafka/tests/lz4_manual_test.sh +0 -0
- package/deps/librdkafka/tests/multi-broker-version-test.sh +0 -0
- package/deps/librdkafka/tests/parse-refcnt.sh +0 -0
- package/deps/librdkafka/tests/performance_plot.py +0 -0
- package/deps/librdkafka/tests/requirements.txt +1 -1
- package/deps/librdkafka/tests/run-all-tests.sh +79 -0
- package/deps/librdkafka/tests/run-consumer-tests.sh +0 -0
- package/deps/librdkafka/tests/run-producer-tests.sh +0 -0
- package/deps/librdkafka/tests/run-test-batches.py +157 -0
- package/deps/librdkafka/tests/run-test.sh +0 -0
- package/deps/librdkafka/tests/sasl_test.py +0 -0
- package/deps/librdkafka/tests/test.c +478 -18
- package/deps/librdkafka/tests/test.h +62 -13
- package/deps/librdkafka/tests/testcpp.cpp +6 -1
- package/deps/librdkafka/tests/testcpp.h +30 -2
- package/deps/librdkafka/tests/testshared.h +16 -2
- package/deps/librdkafka/tests/tools/stats/graph.py +0 -0
- package/deps/librdkafka/tests/tools/stats/to_csv.py +1 -1
- package/deps/librdkafka/tests/trivup/trivup-0.14.0.tar.gz +0 -0
- package/deps/librdkafka/tests/until-fail.sh +0 -0
- package/deps/librdkafka/tests/xxxx-metadata.cpp +1 -1
- package/deps/librdkafka/vcpkg.json +1 -1
- package/deps/librdkafka/win32/setup-msys2.ps1 +24 -8
- package/deps/librdkafka/win32/setup-vcpkg.ps1 +28 -6
- package/deps/librdkafka/win32/tests/tests.vcxproj +6 -0
- package/deps/librdkafka/win32/wingetopt.h +5 -4
- package/errors.d.ts +27 -1
- package/lib/error.js +29 -3
- package/make_docs.sh +0 -0
- package/package.json +11 -11
- package/run_docker.sh +0 -0
- package/src/callbacks.cc +6 -9
- package/src/callbacks.h +1 -5
- package/test/kafka-consumer-worker.js +10 -7
- package/test/kafka-consumer.spec.js +32 -21
- package/ci/update-version.js +0 -123
- package/deps/librdkafka/.semaphore/project.yml +0 -43
- package/deps/librdkafka/.semaphore/project_public.yml +0 -20
- package/deps/librdkafka/win32/install-openssl.ps1 +0 -33
|
@@ -429,7 +429,7 @@ static void do_empty_commit(void) {
|
|
|
429
429
|
TEST_SAY("First commit\n");
|
|
430
430
|
expect = RD_KAFKA_RESP_ERR_NO_ERROR;
|
|
431
431
|
err = rd_kafka_commit_queue(rk, NULL, NULL, empty_offset_commit_cb,
|
|
432
|
-
|
|
432
|
+
&expect);
|
|
433
433
|
if (err != expect)
|
|
434
434
|
TEST_FAIL("commit failed: %s", rd_kafka_err2str(err));
|
|
435
435
|
else
|
|
@@ -438,7 +438,7 @@ static void do_empty_commit(void) {
|
|
|
438
438
|
TEST_SAY("Second commit, should be empty\n");
|
|
439
439
|
expect = RD_KAFKA_RESP_ERR__NO_OFFSET;
|
|
440
440
|
err = rd_kafka_commit_queue(rk, NULL, NULL, empty_offset_commit_cb,
|
|
441
|
-
|
|
441
|
+
&expect);
|
|
442
442
|
if (err != RD_KAFKA_RESP_ERR__NO_OFFSET)
|
|
443
443
|
TEST_FAIL("unexpected commit result, wanted NO_OFFSET, got: %s",
|
|
444
444
|
rd_kafka_err2str(err));
|
|
@@ -47,13 +47,14 @@ struct expect {
|
|
|
47
47
|
int exp_err; /* expected error from subscribe() */
|
|
48
48
|
int stat[4]; /* per exp status */
|
|
49
49
|
int fails;
|
|
50
|
-
enum {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
50
|
+
enum {
|
|
51
|
+
_EXP_NONE,
|
|
52
|
+
_EXP_FAIL,
|
|
53
|
+
_EXP_OK,
|
|
54
|
+
_EXP_ASSIGN,
|
|
55
|
+
_EXP_REVOKE,
|
|
56
|
+
_EXP_ASSIGNED,
|
|
57
|
+
_EXP_REVOKED,
|
|
57
58
|
} result;
|
|
58
59
|
};
|
|
59
60
|
|
|
@@ -135,7 +136,8 @@ static void rebalance_cb(rd_kafka_t *rk,
|
|
|
135
136
|
exp->name, exp->result);
|
|
136
137
|
}
|
|
137
138
|
expect_match(exp, parts);
|
|
138
|
-
|
|
139
|
+
test_consumer_assign_by_rebalance_protocol("rebalance", rk,
|
|
140
|
+
parts);
|
|
139
141
|
exp->result = _EXP_ASSIGNED;
|
|
140
142
|
break;
|
|
141
143
|
|
|
@@ -147,7 +149,8 @@ static void rebalance_cb(rd_kafka_t *rk,
|
|
|
147
149
|
exp->name, exp->result);
|
|
148
150
|
}
|
|
149
151
|
|
|
150
|
-
|
|
152
|
+
test_consumer_unassign_by_rebalance_protocol("rebalance", rk,
|
|
153
|
+
parts);
|
|
151
154
|
exp->result = _EXP_REVOKED;
|
|
152
155
|
break;
|
|
153
156
|
|
|
@@ -374,7 +377,7 @@ static int do_test(const char *assignor) {
|
|
|
374
377
|
.name = rd_strdup(tsprintf("%s: regex 0&1", assignor)),
|
|
375
378
|
.sub = {rd_strdup(tsprintf(
|
|
376
379
|
"^.*[tToOpPiIcC]_0+[12]_[^_]+_%s", groupid)),
|
|
377
|
-
|
|
380
|
+
NULL},
|
|
378
381
|
.exp = {topics[0], topics[1], NULL}};
|
|
379
382
|
|
|
380
383
|
fails += test_subscribe(rk, &expect);
|
|
@@ -387,7 +390,7 @@ static int do_test(const char *assignor) {
|
|
|
387
390
|
.name = rd_strdup(tsprintf("%s: regex 2", assignor)),
|
|
388
391
|
.sub = {rd_strdup(
|
|
389
392
|
tsprintf("^.*TOOTHPIC_000._._%s", groupid)),
|
|
390
|
-
|
|
393
|
+
NULL},
|
|
391
394
|
.exp = {topics[2], NULL}};
|
|
392
395
|
|
|
393
396
|
fails += test_subscribe(rk, &expect);
|
|
@@ -401,7 +404,7 @@ static int do_test(const char *assignor) {
|
|
|
401
404
|
"nonexistent(not seen)",
|
|
402
405
|
assignor)),
|
|
403
406
|
.sub = {rd_strdup(tsprintf("^.*_000[34]_..?_%s", groupid)),
|
|
404
|
-
|
|
407
|
+
NULL},
|
|
405
408
|
.exp = {topics[2], NULL}};
|
|
406
409
|
|
|
407
410
|
fails += test_subscribe(rk, &expect);
|
|
@@ -421,9 +424,23 @@ static int do_test(const char *assignor) {
|
|
|
421
424
|
rd_free(expect.name);
|
|
422
425
|
}
|
|
423
426
|
|
|
427
|
+
{
|
|
428
|
+
struct expect expect = {
|
|
429
|
+
.name = rd_strdup(
|
|
430
|
+
tsprintf("%s: multiple regex 1&2 matches", assignor)),
|
|
431
|
+
.sub = {"^.*regex_subscribe_to.*",
|
|
432
|
+
"^.*regex_subscribe_TOO.*", NULL},
|
|
433
|
+
.exp = {topics[1], topics[2], NULL}};
|
|
434
|
+
|
|
435
|
+
fails += test_subscribe(rk, &expect);
|
|
436
|
+
rd_free(expect.name);
|
|
437
|
+
}
|
|
424
438
|
|
|
425
439
|
test_consumer_close(rk);
|
|
426
440
|
|
|
441
|
+
for (i = 0; i < topic_cnt; i++)
|
|
442
|
+
test_delete_topic(rk, topics[i]);
|
|
443
|
+
|
|
427
444
|
rd_kafka_destroy(rk);
|
|
428
445
|
|
|
429
446
|
if (fails)
|
|
@@ -434,11 +451,10 @@ static int do_test(const char *assignor) {
|
|
|
434
451
|
|
|
435
452
|
|
|
436
453
|
int main_0033_regex_subscribe(int argc, char **argv) {
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
}
|
|
454
|
+
|
|
455
|
+
do_test("range");
|
|
456
|
+
do_test("roundrobin");
|
|
457
|
+
|
|
442
458
|
return 0;
|
|
443
459
|
}
|
|
444
460
|
|
|
@@ -230,6 +230,7 @@ static void offset_reset_errors(void) {
|
|
|
230
230
|
for (i = 0; i < (int)RD_ARRAYSIZE(test); i++) {
|
|
231
231
|
rd_kafka_event_t *ev;
|
|
232
232
|
rd_bool_t broker_down = rd_false;
|
|
233
|
+
rd_ts_t abs_timeout, now;
|
|
233
234
|
|
|
234
235
|
/* Make sure consumer is connected */
|
|
235
236
|
test_wait_topic_exists(c, topic, 5000);
|
|
@@ -259,13 +260,32 @@ static void offset_reset_errors(void) {
|
|
|
259
260
|
test_consumer_assign_partition("ASSIGN", c, topic, partition,
|
|
260
261
|
test[i].start_offset);
|
|
261
262
|
|
|
263
|
+
/* 5 seconds timeout */
|
|
264
|
+
abs_timeout = test_clock() + 5 * 1000000;
|
|
262
265
|
while (1) {
|
|
266
|
+
rd_bool_t not_auto_offset_reset_error;
|
|
263
267
|
/* Poll until we see an AUTO_OFFSET_RESET error,
|
|
264
268
|
* timeout, or a message, depending on what we're
|
|
265
269
|
* looking for. */
|
|
266
|
-
ev
|
|
270
|
+
ev = rd_kafka_queue_poll(queue, 5000);
|
|
271
|
+
now = test_clock();
|
|
272
|
+
not_auto_offset_reset_error =
|
|
273
|
+
ev &&
|
|
274
|
+
rd_kafka_event_type(ev) == RD_KAFKA_EVENT_ERROR &&
|
|
275
|
+
rd_kafka_event_error(ev) !=
|
|
276
|
+
RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET;
|
|
277
|
+
|
|
278
|
+
if (!ev || (not_auto_offset_reset_error &&
|
|
279
|
+
now > abs_timeout)) {
|
|
280
|
+
if (ev) {
|
|
281
|
+
TEST_SAY(
|
|
282
|
+
"#%d: Ignoring %s event: %s\n", i,
|
|
283
|
+
rd_kafka_event_name(ev),
|
|
284
|
+
rd_kafka_event_error_string(ev));
|
|
285
|
+
rd_kafka_event_destroy(ev);
|
|
286
|
+
ev = NULL;
|
|
287
|
+
}
|
|
267
288
|
|
|
268
|
-
if (!ev) {
|
|
269
289
|
TEST_ASSERT(broker_down,
|
|
270
290
|
"#%d: poll timeout, but broker "
|
|
271
291
|
"was not down",
|
|
@@ -278,12 +298,13 @@ static void offset_reset_errors(void) {
|
|
|
278
298
|
broker_id);
|
|
279
299
|
|
|
280
300
|
broker_down = rd_false;
|
|
301
|
+
/* 5 seconds timeout */
|
|
302
|
+
abs_timeout = test_clock() + 5 * 1000000;
|
|
281
303
|
|
|
282
304
|
} else if (rd_kafka_event_type(ev) ==
|
|
283
305
|
RD_KAFKA_EVENT_ERROR) {
|
|
284
306
|
|
|
285
|
-
if (
|
|
286
|
-
RD_KAFKA_RESP_ERR__AUTO_OFFSET_RESET) {
|
|
307
|
+
if (not_auto_offset_reset_error) {
|
|
287
308
|
TEST_SAY(
|
|
288
309
|
"#%d: Ignoring %s event: %s\n", i,
|
|
289
310
|
rd_kafka_event_name(ev),
|
|
@@ -59,6 +59,7 @@ int main_0036_partial_fetch(int argc, char **argv) {
|
|
|
59
59
|
testid = test_id_generate();
|
|
60
60
|
rk = test_create_producer();
|
|
61
61
|
rkt = test_create_producer_topic(rk, topic, NULL);
|
|
62
|
+
test_wait_topic_exists(rk, topic, 5000);
|
|
62
63
|
|
|
63
64
|
test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt, NULL, msgsize);
|
|
64
65
|
|
|
@@ -68,6 +68,7 @@ int main_0038_performance(int argc, char **argv) {
|
|
|
68
68
|
test_conf_set(conf, "linger.ms", "100");
|
|
69
69
|
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
|
|
70
70
|
rkt = test_create_producer_topic(rk, topic, "acks", "1", NULL);
|
|
71
|
+
test_wait_topic_exists(rk, topic, 5000);
|
|
71
72
|
|
|
72
73
|
/* First produce one message to create the topic, etc, this might take
|
|
73
74
|
* a while and we dont want this to affect the throughput timing. */
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* librdkafka - Apache Kafka C library
|
|
3
3
|
*
|
|
4
4
|
* Copyright (c) 2012-2022, Magnus Edenhill
|
|
5
|
+
* 2025, Confluent Inc.
|
|
5
6
|
* All rights reserved.
|
|
6
7
|
*
|
|
7
8
|
* Redistribution and use in source and binary forms, with or without
|
|
@@ -74,7 +75,8 @@ int main_0040_io_event(int argc, char **argv) {
|
|
|
74
75
|
|
|
75
76
|
rk_p = test_create_producer();
|
|
76
77
|
rkt_p = test_create_producer_topic(rk_p, topic, NULL);
|
|
77
|
-
|
|
78
|
+
test_wait_topic_exists(rk_p, topic, 5000);
|
|
79
|
+
err = test_auto_create_topic_rkt(rk_p, rkt_p, tmout_multip(5000));
|
|
78
80
|
TEST_ASSERT(!err, "Topic auto creation failed: %s",
|
|
79
81
|
rd_kafka_err2str(err));
|
|
80
82
|
|
|
@@ -168,13 +170,17 @@ int main_0040_io_event(int argc, char **argv) {
|
|
|
168
170
|
"expecting message\n");
|
|
169
171
|
if (rd_kafka_event_error(rkev) ==
|
|
170
172
|
RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS) {
|
|
171
|
-
|
|
172
|
-
rk_c,
|
|
173
|
+
test_consumer_assign_by_rebalance_protocol(
|
|
174
|
+
"rebalance event", rk_c,
|
|
173
175
|
rd_kafka_event_topic_partition_list(
|
|
174
176
|
rkev));
|
|
175
177
|
expecting_io = _NOPE;
|
|
176
|
-
} else
|
|
177
|
-
|
|
178
|
+
} else {
|
|
179
|
+
test_consumer_unassign_by_rebalance_protocol(
|
|
180
|
+
"rebalance event", rk_c,
|
|
181
|
+
rd_kafka_event_topic_partition_list(
|
|
182
|
+
rkev));
|
|
183
|
+
}
|
|
178
184
|
break;
|
|
179
185
|
|
|
180
186
|
case RD_KAFKA_EVENT_FETCH:
|
|
@@ -61,6 +61,7 @@ int main_0041_fetch_max_bytes(int argc, char **argv) {
|
|
|
61
61
|
testid = test_id_generate();
|
|
62
62
|
rk = test_create_producer();
|
|
63
63
|
rkt = test_create_producer_topic(rk, topic, NULL);
|
|
64
|
+
test_wait_topic_exists(rk, topic, 5000);
|
|
64
65
|
|
|
65
66
|
test_produce_msgs(rk, rkt, testid, partition, 0, msgcnt / 2, NULL,
|
|
66
67
|
MAX_BYTES / 10);
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* librdkafka - Apache Kafka C library
|
|
3
3
|
*
|
|
4
4
|
* Copyright (c) 2012-2022, Magnus Edenhill
|
|
5
|
+
* 2025, Confluent Inc.
|
|
5
6
|
* All rights reserved.
|
|
6
7
|
*
|
|
7
8
|
* Redistribution and use in source and binary forms, with or without
|
|
@@ -60,10 +61,10 @@ static void test_producer_partition_cnt_change(void) {
|
|
|
60
61
|
rd_kafka_conf_set_dr_msg_cb(conf, test_dr_msg_cb);
|
|
61
62
|
rk = test_create_handle(RD_KAFKA_PRODUCER, conf);
|
|
62
63
|
|
|
63
|
-
|
|
64
|
+
test_create_topic_wait_exists(rk, topic, partition_cnt / 2, 1, 5000);
|
|
64
65
|
|
|
65
66
|
rkt =
|
|
66
|
-
test_create_topic_object(rk,
|
|
67
|
+
test_create_topic_object(rk, topic, "message.timeout.ms",
|
|
67
68
|
tsprintf("%d", tmout_multip(10000)), NULL);
|
|
68
69
|
|
|
69
70
|
test_produce_msgs_nowait(rk, rkt, 0, RD_KAFKA_PARTITION_UA, 0,
|
|
@@ -101,7 +101,7 @@ static void await_assignment(const char *pfx,
|
|
|
101
101
|
if (fails > 0)
|
|
102
102
|
TEST_FAIL("%s: assignment mismatch: see above", pfx);
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
test_consumer_assign_by_rebalance_protocol("rebalance event", rk, tps);
|
|
105
105
|
rd_kafka_event_destroy(rkev);
|
|
106
106
|
}
|
|
107
107
|
|
|
@@ -112,6 +112,7 @@ static void await_assignment(const char *pfx,
|
|
|
112
112
|
static void
|
|
113
113
|
await_revoke(const char *pfx, rd_kafka_t *rk, rd_kafka_queue_t *queue) {
|
|
114
114
|
rd_kafka_event_t *rkev;
|
|
115
|
+
rd_kafka_topic_partition_list_t *tps;
|
|
115
116
|
|
|
116
117
|
TEST_SAY("%s: waiting for revoke\n", pfx);
|
|
117
118
|
rkev = test_wait_event(queue, RD_KAFKA_EVENT_REBALANCE, 30000);
|
|
@@ -121,7 +122,10 @@ await_revoke(const char *pfx, rd_kafka_t *rk, rd_kafka_queue_t *queue) {
|
|
|
121
122
|
RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS,
|
|
122
123
|
"expected REVOKE, got %s",
|
|
123
124
|
rd_kafka_err2str(rd_kafka_event_error(rkev)));
|
|
124
|
-
|
|
125
|
+
tps = rd_kafka_event_topic_partition_list(rkev);
|
|
126
|
+
|
|
127
|
+
test_consumer_unassign_by_rebalance_protocol("rebalance event", rk,
|
|
128
|
+
tps);
|
|
125
129
|
rd_kafka_event_destroy(rkev);
|
|
126
130
|
}
|
|
127
131
|
|
|
@@ -231,7 +235,7 @@ static void do_test_non_exist_and_partchange(void) {
|
|
|
231
235
|
await_no_rebalance("#1: empty", rk, queue, 10000);
|
|
232
236
|
|
|
233
237
|
TEST_SAY("#1: creating topic %s\n", topic_a);
|
|
234
|
-
|
|
238
|
+
test_create_topic_wait_exists(NULL, topic_a, 2, 1, 5000);
|
|
235
239
|
|
|
236
240
|
await_assignment("#1: proper", rk, queue, 1, topic_a, 2);
|
|
237
241
|
|
|
@@ -266,6 +270,7 @@ static void do_test_regex(void) {
|
|
|
266
270
|
rd_kafka_t *rk;
|
|
267
271
|
rd_kafka_conf_t *conf;
|
|
268
272
|
rd_kafka_queue_t *queue;
|
|
273
|
+
const char *rebalance_protocol;
|
|
269
274
|
|
|
270
275
|
/**
|
|
271
276
|
* Regex test:
|
|
@@ -290,8 +295,7 @@ static void do_test_regex(void) {
|
|
|
290
295
|
queue = rd_kafka_queue_get_consumer(rk);
|
|
291
296
|
|
|
292
297
|
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_b);
|
|
293
|
-
|
|
294
|
-
rd_sleep(1); // FIXME: do check&wait loop instead
|
|
298
|
+
test_create_topic_wait_exists(NULL, topic_b, 2, 1, 5000);
|
|
295
299
|
|
|
296
300
|
TEST_SAY("Regex: Subscribing to %s & %s & %s\n", topic_b, topic_d,
|
|
297
301
|
topic_e);
|
|
@@ -301,18 +305,26 @@ static void do_test_regex(void) {
|
|
|
301
305
|
2);
|
|
302
306
|
|
|
303
307
|
TEST_SAY("Regex: creating topic %s (not subscribed)\n", topic_c);
|
|
304
|
-
|
|
308
|
+
test_create_topic_wait_exists(NULL, topic_c, 4, 1, 5000);
|
|
305
309
|
|
|
306
310
|
/* Should not see a rebalance since no topics are matched. */
|
|
307
311
|
await_no_rebalance("Regex: empty", rk, queue, 10000);
|
|
308
312
|
|
|
309
313
|
TEST_SAY("Regex: creating topic %s (subscribed)\n", topic_d);
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
314
|
+
test_create_topic_wait_exists(NULL, topic_d, 1, 1, 5000);
|
|
315
|
+
|
|
316
|
+
if (test_consumer_group_protocol_classic())
|
|
317
|
+
await_revoke("Regex: rebalance after topic creation", rk,
|
|
318
|
+
queue);
|
|
319
|
+
|
|
320
|
+
rebalance_protocol = rd_kafka_rebalance_protocol(rk);
|
|
321
|
+
if (!strcmp(rebalance_protocol, "COOPERATIVE")) {
|
|
322
|
+
await_assignment("Regex: two topics exist", rk, queue, 1,
|
|
323
|
+
topic_d, 1);
|
|
324
|
+
} else {
|
|
325
|
+
await_assignment("Regex: two topics exist", rk, queue, 2,
|
|
326
|
+
topic_b, 2, topic_d, 1);
|
|
327
|
+
}
|
|
316
328
|
|
|
317
329
|
test_consumer_close(rk);
|
|
318
330
|
rd_kafka_queue_destroy(queue);
|
|
@@ -364,12 +376,10 @@ static void do_test_topic_remove(void) {
|
|
|
364
376
|
queue = rd_kafka_queue_get_consumer(rk);
|
|
365
377
|
|
|
366
378
|
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_f);
|
|
367
|
-
|
|
379
|
+
test_create_topic_wait_exists(NULL, topic_f, parts_f, 1, 5000);
|
|
368
380
|
|
|
369
381
|
TEST_SAY("Topic removal: creating topic %s (subscribed)\n", topic_g);
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
rd_sleep(1); // FIXME: do check&wait loop instead
|
|
382
|
+
test_create_topic_wait_exists(NULL, topic_g, parts_g, 1, 5000);
|
|
373
383
|
|
|
374
384
|
TEST_SAY("Topic removal: Subscribing to %s & %s\n", topic_f, topic_g);
|
|
375
385
|
topics = rd_kafka_topic_partition_list_new(2);
|
|
@@ -493,7 +503,7 @@ static void do_test_replica_rack_change_mock(const char *assignment_strategy,
|
|
|
493
503
|
const char *topic = "topic";
|
|
494
504
|
const char *test_name = tsprintf(
|
|
495
505
|
"Replica rack changes (%s, subscription = \"%s\", %s client.rack, "
|
|
496
|
-
|
|
506
|
+
"%s replica.rack)",
|
|
497
507
|
assignment_strategy, subscription,
|
|
498
508
|
use_client_rack ? "with" : "without",
|
|
499
509
|
use_replica_rack ? "with" : "without");
|
|
@@ -677,6 +687,242 @@ static void do_test_replica_rack_change_leader_no_rack_mock(
|
|
|
677
687
|
SUB_TEST_PASS();
|
|
678
688
|
}
|
|
679
689
|
|
|
690
|
+
/**
|
|
691
|
+
* Checking assignments and revocation operations for subscribe and
|
|
692
|
+
* unsubcribe with regular topic names and regex.
|
|
693
|
+
*/
|
|
694
|
+
static void do_test_resubscribe_with_regex() {
|
|
695
|
+
char *topic1 = rd_strdup(test_mk_topic_name("topic_regex1", 1));
|
|
696
|
+
char *topic2 = rd_strdup(test_mk_topic_name("topic_regex2", 1));
|
|
697
|
+
char *topic_a = rd_strdup(test_mk_topic_name("topic_a", 1));
|
|
698
|
+
char *group = rd_strdup(
|
|
699
|
+
tsprintf("group_test_sub_regex_%s", test_str_id_generate_tmp()));
|
|
700
|
+
rd_kafka_t *rk;
|
|
701
|
+
rd_kafka_conf_t *conf;
|
|
702
|
+
rd_kafka_queue_t *queue;
|
|
703
|
+
|
|
704
|
+
SUB_TEST("Resubscribe with Regex");
|
|
705
|
+
|
|
706
|
+
/**
|
|
707
|
+
* Topic resubscribe with regex test:
|
|
708
|
+
* - Create topic topic_regex1 & topic_regex2
|
|
709
|
+
* - Subscribe to topic_regex1
|
|
710
|
+
* - Verify topic_regex1 assignment
|
|
711
|
+
* - Unsubscribe
|
|
712
|
+
* - Verify revocation
|
|
713
|
+
* - Subscribe to topic_regex2
|
|
714
|
+
* - Verify topic_regex2 assignment
|
|
715
|
+
* - Unsubscribe
|
|
716
|
+
* - Verify revocation
|
|
717
|
+
* - Subscribe to regex ^.*topic_regex.*
|
|
718
|
+
* - Verify topic_regex1 & topic_regex2 assignment
|
|
719
|
+
* - Unsubscribe
|
|
720
|
+
* - Verify revocation
|
|
721
|
+
* - Subscribe to regex ^.*topic_regex.* and topic_a
|
|
722
|
+
* - Verify topic_regex1, topic_regex2 and topic_a assignment
|
|
723
|
+
* - Unsubscribe
|
|
724
|
+
* - Verify revocation
|
|
725
|
+
*/
|
|
726
|
+
|
|
727
|
+
TEST_SAY("Creating topic %s\n", topic1);
|
|
728
|
+
test_create_topic_wait_exists(NULL, topic1, 4, 1, 5000);
|
|
729
|
+
|
|
730
|
+
TEST_SAY("Creating topic %s\n", topic2);
|
|
731
|
+
test_create_topic_wait_exists(NULL, topic2, 4, 1, 5000);
|
|
732
|
+
|
|
733
|
+
TEST_SAY("Creating topic %s\n", topic_a);
|
|
734
|
+
test_create_topic_wait_exists(NULL, topic_a, 2, 1, 5000);
|
|
735
|
+
|
|
736
|
+
test_conf_init(&conf, NULL, 60);
|
|
737
|
+
|
|
738
|
+
rd_kafka_conf_set_events(conf, RD_KAFKA_EVENT_REBALANCE);
|
|
739
|
+
rk = test_create_consumer(group, NULL, conf, NULL);
|
|
740
|
+
queue = rd_kafka_queue_get_consumer(rk);
|
|
741
|
+
|
|
742
|
+
/* Subscribe to topic1 */
|
|
743
|
+
TEST_SAY("Subscribing to %s\n", topic1);
|
|
744
|
+
test_consumer_subscribe(rk, topic1);
|
|
745
|
+
/* Wait for assignment */
|
|
746
|
+
await_assignment("Assignment for topic1", rk, queue, 1, topic1, 4);
|
|
747
|
+
|
|
748
|
+
/* Unsubscribe from topic1 */
|
|
749
|
+
TEST_SAY("Unsubscribing from %s\n", topic1);
|
|
750
|
+
rd_kafka_unsubscribe(rk);
|
|
751
|
+
/* Wait for revocation */
|
|
752
|
+
await_revoke("Revocation after unsubscribing", rk, queue);
|
|
753
|
+
|
|
754
|
+
/* Subscribe to topic2 */
|
|
755
|
+
TEST_SAY("Subscribing to %s\n", topic2);
|
|
756
|
+
test_consumer_subscribe(rk, topic2);
|
|
757
|
+
/* Wait for assignment */
|
|
758
|
+
await_assignment("Assignment for topic2", rk, queue, 1, topic2, 4);
|
|
759
|
+
|
|
760
|
+
/* Unsubscribe from topic2 */
|
|
761
|
+
TEST_SAY("Unsubscribing from %s\n", topic2);
|
|
762
|
+
rd_kafka_unsubscribe(rk);
|
|
763
|
+
/* Wait for revocation */
|
|
764
|
+
await_revoke("Revocation after unsubscribing", rk, queue);
|
|
765
|
+
|
|
766
|
+
/* Subscribe to regex ^.*topic_regex.* */
|
|
767
|
+
TEST_SAY("Subscribing to regex ^.*topic_regex.*\n");
|
|
768
|
+
test_consumer_subscribe(rk, "^.*topic_regex.*");
|
|
769
|
+
if (!test_consumer_group_protocol_classic()) {
|
|
770
|
+
/** Regex matching is async on the broker side for KIP-848
|
|
771
|
+
* protocol. */
|
|
772
|
+
rd_sleep(5);
|
|
773
|
+
}
|
|
774
|
+
/* Wait for assignment */
|
|
775
|
+
await_assignment("Assignment for topic1 and topic2", rk, queue, 2,
|
|
776
|
+
topic1, 4, topic2, 4);
|
|
777
|
+
|
|
778
|
+
/* Unsubscribe from regex ^.*topic_regex.* */
|
|
779
|
+
TEST_SAY("Unsubscribing from regex ^.*topic_regex.*\n");
|
|
780
|
+
rd_kafka_unsubscribe(rk);
|
|
781
|
+
/* Wait for revocation */
|
|
782
|
+
await_revoke("Revocation after unsubscribing", rk, queue);
|
|
783
|
+
|
|
784
|
+
/* Subscribe to regex ^.*topic_regex.* and topic_a literal */
|
|
785
|
+
TEST_SAY("Subscribing to regex ^.*topic_regex.* and topic_a\n");
|
|
786
|
+
test_consumer_subscribe_multi(rk, 2, "^.*topic_regex.*", topic_a);
|
|
787
|
+
/* Wait for assignment */
|
|
788
|
+
if (test_consumer_group_protocol_classic()) {
|
|
789
|
+
await_assignment("Assignment for topic1, topic2 and topic_a",
|
|
790
|
+
rk, queue, 3, topic1, 4, topic2, 4, topic_a,
|
|
791
|
+
2);
|
|
792
|
+
} else {
|
|
793
|
+
/* KIP-848 broker is assigning literal topics first in 1 HB call
|
|
794
|
+
* and all the matched ones in later HB call*/
|
|
795
|
+
await_assignment("Assignment for topic_a", rk, queue, 1,
|
|
796
|
+
topic_a, 2);
|
|
797
|
+
await_assignment("Assignment for topic1 and topic2", rk, queue,
|
|
798
|
+
2, topic1, 4, topic2, 4);
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
/* Unsubscribe */
|
|
802
|
+
TEST_SAY("Unsubscribing\n");
|
|
803
|
+
rd_kafka_unsubscribe(rk);
|
|
804
|
+
await_revoke("Revocation after unsubscribing", rk, queue);
|
|
805
|
+
|
|
806
|
+
/* Cleanup */
|
|
807
|
+
test_delete_topic(rk, topic1);
|
|
808
|
+
test_delete_topic(rk, topic2);
|
|
809
|
+
test_delete_topic(rk, topic_a);
|
|
810
|
+
|
|
811
|
+
test_consumer_close(rk);
|
|
812
|
+
rd_kafka_queue_destroy(queue);
|
|
813
|
+
|
|
814
|
+
rd_kafka_destroy(rk);
|
|
815
|
+
|
|
816
|
+
rd_free(topic1);
|
|
817
|
+
rd_free(topic2);
|
|
818
|
+
rd_free(topic_a);
|
|
819
|
+
rd_free(group);
|
|
820
|
+
|
|
821
|
+
SUB_TEST_PASS();
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* @brief Create many topics and apply several subscription
|
|
826
|
+
* updates, unsubscribing and re-subscribing too.
|
|
827
|
+
* After changing some subscriptions verifies that the assignment
|
|
828
|
+
* corresponds to last one.
|
|
829
|
+
*
|
|
830
|
+
* @param with_rebalance_cb Use a rebalance callback to perform the assignment.
|
|
831
|
+
* It needs to poll the consumer when awaiting for the
|
|
832
|
+
* assignment in this case.
|
|
833
|
+
*/
|
|
834
|
+
static void do_test_subscribe_many_updates(rd_bool_t with_rebalance_cb) {
|
|
835
|
+
#define TOPIC_CNT 100
|
|
836
|
+
char *topics[TOPIC_CNT] = {0};
|
|
837
|
+
char *topic;
|
|
838
|
+
char *group;
|
|
839
|
+
size_t i;
|
|
840
|
+
rd_kafka_t *rk;
|
|
841
|
+
rd_kafka_conf_t *conf;
|
|
842
|
+
const int partition_cnt = 4;
|
|
843
|
+
|
|
844
|
+
SUB_TEST("%s", with_rebalance_cb ? "with rebalance callback"
|
|
845
|
+
: "without rebalance callback");
|
|
846
|
+
|
|
847
|
+
RD_ARRAY_FOREACH_INDEX(topic, topics, i) {
|
|
848
|
+
char topic_i[17];
|
|
849
|
+
rd_snprintf(topic_i, sizeof(topic_i), "topic%" PRIusz, i);
|
|
850
|
+
topics[i] = rd_strdup(test_mk_topic_name(topic_i, 1));
|
|
851
|
+
};
|
|
852
|
+
group = topics[0];
|
|
853
|
+
|
|
854
|
+
test_conf_init(&conf, NULL, 60);
|
|
855
|
+
if (with_rebalance_cb)
|
|
856
|
+
rd_kafka_conf_set_rebalance_cb(conf, test_rebalance_cb);
|
|
857
|
+
rk = test_create_consumer(group, NULL, conf, NULL);
|
|
858
|
+
|
|
859
|
+
TEST_SAY("Creating %d topics\n", TOPIC_CNT);
|
|
860
|
+
TEST_CALL_ERR__(test_CreateTopics_simple(rk, NULL, topics, TOPIC_CNT,
|
|
861
|
+
partition_cnt, NULL));
|
|
862
|
+
test_wait_topic_exists(rk, topics[TOPIC_CNT - 1], 5000);
|
|
863
|
+
/* Give the cluster some more time to propagate metadata
|
|
864
|
+
* for TOPICS_CNT topics */
|
|
865
|
+
rd_sleep(1);
|
|
866
|
+
|
|
867
|
+
RD_ARRAY_FOREACH_INDEX(topic, topics, i) {
|
|
868
|
+
const int max_subscription_size = 5;
|
|
869
|
+
size_t j;
|
|
870
|
+
int k;
|
|
871
|
+
int subscription_size =
|
|
872
|
+
RD_MIN(max_subscription_size, TOPIC_CNT - i);
|
|
873
|
+
int expected_assignment_cnt = subscription_size * partition_cnt;
|
|
874
|
+
rd_kafka_topic_partition_list_t *expected_assignment = NULL;
|
|
875
|
+
|
|
876
|
+
rd_kafka_topic_partition_list_t *subscription =
|
|
877
|
+
rd_kafka_topic_partition_list_new(subscription_size);
|
|
878
|
+
rd_bool_t check_expected_assignment =
|
|
879
|
+
(i % 5 == 0 || i == TOPIC_CNT - 1);
|
|
880
|
+
rd_bool_t do_unsubscribe = i % 7 == 0;
|
|
881
|
+
|
|
882
|
+
if (check_expected_assignment)
|
|
883
|
+
expected_assignment = rd_kafka_topic_partition_list_new(
|
|
884
|
+
expected_assignment_cnt);
|
|
885
|
+
|
|
886
|
+
for (j = i; j < i + subscription_size; j++) {
|
|
887
|
+
rd_kafka_topic_partition_list_add(
|
|
888
|
+
subscription, topics[j], RD_KAFKA_PARTITION_UA);
|
|
889
|
+
/* We unsubscribe every 7 iteration and
|
|
890
|
+
* we check assignments every 5 iteration so
|
|
891
|
+
* at 7 * 5 we unsubscribe and check that assignment
|
|
892
|
+
* is empty. */
|
|
893
|
+
if (check_expected_assignment && !do_unsubscribe)
|
|
894
|
+
for (k = 0; k < partition_cnt; k++)
|
|
895
|
+
rd_kafka_topic_partition_list_add(
|
|
896
|
+
expected_assignment, topics[j], k);
|
|
897
|
+
}
|
|
898
|
+
TEST_CALL_ERR__(rd_kafka_subscribe(rk, subscription));
|
|
899
|
+
rd_kafka_topic_partition_list_destroy(subscription);
|
|
900
|
+
|
|
901
|
+
if (do_unsubscribe)
|
|
902
|
+
TEST_CALL_ERR__(rd_kafka_unsubscribe(rk));
|
|
903
|
+
if (check_expected_assignment) {
|
|
904
|
+
test_consumer_wait_assignment_topic_partition_list(
|
|
905
|
+
rk,
|
|
906
|
+
/* poll when we have a rebalance callback */
|
|
907
|
+
with_rebalance_cb, expected_assignment, 30000);
|
|
908
|
+
rd_kafka_topic_partition_list_destroy(
|
|
909
|
+
expected_assignment);
|
|
910
|
+
}
|
|
911
|
+
};
|
|
912
|
+
|
|
913
|
+
TEST_CALL_ERR__(
|
|
914
|
+
test_DeleteTopics_simple(rk, NULL, topics, TOPIC_CNT, NULL));
|
|
915
|
+
RD_ARRAY_FOREACH_INDEX(topic, topics, i) {
|
|
916
|
+
rd_free(topic);
|
|
917
|
+
};
|
|
918
|
+
|
|
919
|
+
test_consumer_close(rk);
|
|
920
|
+
rd_kafka_destroy(rk);
|
|
921
|
+
|
|
922
|
+
SUB_TEST_PASS();
|
|
923
|
+
#undef TOPIC_CNT
|
|
924
|
+
}
|
|
925
|
+
|
|
680
926
|
int main_0045_subscribe_update(int argc, char **argv) {
|
|
681
927
|
|
|
682
928
|
if (!test_can_create_topics(1))
|
|
@@ -706,6 +952,8 @@ int main_0045_subscribe_update_topic_remove(int argc, char **argv) {
|
|
|
706
952
|
|
|
707
953
|
|
|
708
954
|
int main_0045_subscribe_update_mock(int argc, char **argv) {
|
|
955
|
+
TEST_SKIP_MOCK_CLUSTER(0);
|
|
956
|
+
|
|
709
957
|
do_test_regex_many_mock("range", rd_false);
|
|
710
958
|
do_test_regex_many_mock("cooperative-sticky", rd_false);
|
|
711
959
|
do_test_regex_many_mock("cooperative-sticky", rd_true);
|
|
@@ -713,6 +961,16 @@ int main_0045_subscribe_update_mock(int argc, char **argv) {
|
|
|
713
961
|
return 0;
|
|
714
962
|
}
|
|
715
963
|
|
|
964
|
+
int main_0045_resubscribe_with_regex(int argc, char **argv) {
|
|
965
|
+
do_test_resubscribe_with_regex();
|
|
966
|
+
return 0;
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
int main_0045_subscribe_many_updates(int argc, char **argv) {
|
|
970
|
+
do_test_subscribe_many_updates(rd_false);
|
|
971
|
+
do_test_subscribe_many_updates(rd_true);
|
|
972
|
+
return 0;
|
|
973
|
+
}
|
|
716
974
|
|
|
717
975
|
int main_0045_subscribe_update_racks_mock(int argc, char **argv) {
|
|
718
976
|
int use_replica_rack = 0;
|
|
@@ -720,6 +978,12 @@ int main_0045_subscribe_update_racks_mock(int argc, char **argv) {
|
|
|
720
978
|
|
|
721
979
|
TEST_SKIP_MOCK_CLUSTER(0);
|
|
722
980
|
|
|
981
|
+
/* KIP 848 Mock broker assignor isn't rack-aware. */
|
|
982
|
+
if (!test_consumer_group_protocol_classic()) {
|
|
983
|
+
TEST_SKIP("Test meaningful only with classic protocol\n");
|
|
984
|
+
return 0;
|
|
985
|
+
}
|
|
986
|
+
|
|
723
987
|
for (use_replica_rack = 0; use_replica_rack < 2; use_replica_rack++) {
|
|
724
988
|
for (use_client_rack = 0; use_client_rack < 2;
|
|
725
989
|
use_client_rack++) {
|
|
@@ -737,9 +1001,9 @@ int main_0045_subscribe_update_racks_mock(int argc, char **argv) {
|
|
|
737
1001
|
use_client_rack, use_replica_rack);
|
|
738
1002
|
}
|
|
739
1003
|
}
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
*
|
|
1004
|
+
/* Do not test with range assignor (yet) since it does not do
|
|
1005
|
+
* rack aware assignment properly with the NULL rack, even for
|
|
1006
|
+
* the Java client. */
|
|
743
1007
|
do_test_replica_rack_change_leader_no_rack_mock("cooperative-sticky");
|
|
744
1008
|
|
|
745
1009
|
return 0;
|