@platformatic/rdkafka 4.0.1 → 4.1.0

package/README.md

@@ -11,7 +11,7 @@ This project was forked from the great [https://github.com/blizzard/node-rdkafka
 
 The `@platformatic/rdkafka` library is a high-performance NodeJS client for [Apache Kafka](http://kafka.apache.org/) that wraps the native  [librdkafka](https://github.com/edenhill/librdkafka) library.  All the complexity of balancing writes across partitions and managing (possibly ever-changing) brokers should be encapsulated in the library.
 
-__This library currently uses `librdkafka` version `2.6.1`.__
+__This library currently uses `librdkafka` version `2.12.0`.__
 
 ## Reference Docs
 
@@ -54,7 +54,7 @@ Using Alpine Linux? Check out the [docs](https://github.com/platformatic/rdkafka
 
 ### Windows
 
-Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.2.6.1.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
+Windows build **is not** compiled from `librdkafka` source but it is rather linked against the appropriate version of [NuGet librdkafka.redist](https://www.nuget.org/packages/librdkafka.redist/) static binary that gets downloaded from `https://globalcdn.nuget.org/packages/librdkafka.redist.2.12.0.nupkg` during installation. This download link can be changed using the environment variable `NODE_RDKAFKA_NUGET_BASE_URL` that defaults to `https://globalcdn.nuget.org/packages/` when it's no set.
 
 Requirements:
  * [node-gyp for Windows](https://github.com/nodejs/node-gyp#on-windows)
@@ -91,7 +91,7 @@ const Kafka = require('@platformatic/rdkafka');
 
 ## Configuration
 
-You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.6.1/CONFIGURATION.md)
+You can pass many configuration options to `librdkafka`.  A full list can be found in `librdkafka`'s [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.12.0/CONFIGURATION.md)
 
 Configuration keys that have the suffix `_cb` are designated as callbacks. Some
 of these keys are informational and you can choose to opt-in (for example, `dr_cb`). Others are callbacks designed to
@@ -126,7 +126,7 @@ You can also get the version of `librdkafka`
 const Kafka = require('@platformatic/rdkafka');
 console.log(Kafka.librdkafkaVersion);
 
-// #=> 2.6.1
+// #=> 2.12.0
 ```
 
 ## Sending Messages
@@ -139,7 +139,7 @@ const producer = new Kafka.Producer({
 });
 ```
 
-A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.6.1/CONFIGURATION.md) file described previously.
+A `Producer` requires only `metadata.broker.list` (the Kafka brokers) to be created.  The values in this list are separated by commas.  For other configuration options, see the [Configuration.md](https://github.com/edenhill/librdkafka/blob/v2.12.0/CONFIGURATION.md) file described previously.
 
 The following example illustrates a list with several `librdkafka` options set.
 

package/binding.gyp

@@ -43,6 +43,9 @@
                 'action': ['python', '<@(_inputs)']
               }
             ],
+            'cflags_cc' : [
+              '-std=c++20'
+            ],
             'msvs_settings': {
               'VCLinkerTool': {
                 'AdditionalDependencies': [
@@ -119,6 +122,9 @@
               [
                 'OS=="linux"',
                 {
+                  'cflags_cc' : [
+                    '-std=c++20'
+                  ],
                   'cflags_cc!': [
                     '-fno-rtti'
                   ]
@@ -134,7 +140,8 @@
                       '-L/usr/local/opt/openssl/lib'
                     ],
                     'OTHER_CPLUSPLUSFLAGS': [
-                      '-I/usr/local/opt/openssl/include'
+                      '-I/usr/local/opt/openssl/include',
+                      '-std=c++20'
                     ],
                   },
                 }

package/ci/prepublish.js

@@ -1,4 +1,3 @@
 require('./checks/librdkafka-exists');
 require('./checks/librdkafka-correct-version');
 require('./librdkafka-defs-generator.js');
-require('./update-version');

package/config.d.ts

@@ -1,4 +1,4 @@
-// ====== Generated from librdkafka 2.6.1 file CONFIGURATION.md ======
+// ====== Generated from librdkafka 2.12.0 file CONFIGURATION.md ======
 // Code that generated this is a derivative work of the code from Nam Nguyen
 // https://gist.github.com/ntgn81/066c2c8ec5b4238f85d1e9168a04e3fb
 
@@ -62,6 +62,20 @@ export interface GlobalConfig {
      */
     "max.in.flight"?: number;
 
+    /**
+     * Controls how the client recovers when none of the brokers known to it is available. If set to `none`, the client doesn't re-bootstrap. If set to `rebootstrap`, the client repeats the bootstrap process using `bootstrap.servers` and brokers added through `rd_kafka_brokers_add()`. Rebootstrapping is useful when a client communicates with brokers so infrequently that the set of brokers may change entirely before the client refreshes metadata. Metadata recovery is triggered when all last-known brokers appear unavailable simultaneously or the client cannot refresh metadata within `metadata.recovery.rebootstrap.trigger.ms` or it's requested in a metadata response.
+     *
+     * @default rebootstrap
+     */
+    "metadata.recovery.strategy"?: 'none' | 'rebootstrap';
+
+    /**
+     * If a client configured to rebootstrap using `metadata.recovery.strategy=rebootstrap` is unable to obtain metadata from any of the brokers for this interval, client repeats the bootstrap process using `bootstrap.servers` configuration and brokers added through `rd_kafka_brokers_add()`.
+     *
+     * @default 300000
+     */
+    "metadata.recovery.rebootstrap.trigger.ms"?: number;
+
     /**
      * Period of time in milliseconds at which topic and broker metadata is refreshed in order to proactively discover any new brokers, topics, partitions or partition leader changes. Use -1 to disable the intervalled refresh (not recommended). If there are no locally referenced topics (no topic objects created, no messages produced, no subscription or no assignment) then only the broker list will be refreshed every interval but no more often than every 10s.
      *
@@ -152,7 +166,7 @@ export interface GlobalConfig {
     /**
      * Disable the Nagle algorithm (TCP_NODELAY) on broker sockets.
      *
-     * @default false
+     * @default true
      */
     "socket.nagle.disable"?: boolean;
 
@@ -185,7 +199,7 @@ export interface GlobalConfig {
     "socket.connection.setup.timeout.ms"?: number;
 
     /**
-     * Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info).
+     * Close broker connections after the specified time of inactivity. Disable with 0. If this property is left at its default value some heuristics are performed to determine a suitable default value, this is currently limited to identifying brokers on Azure (see librdkafka issue #3109 for more info). Actual value can be lower, up to 2s lower, only if `connections.max.idle.ms` >= 4s, as jitter is added to avoid disconnecting all brokers at the same time.
      *
      * @default 0
      */
@@ -329,7 +343,7 @@ export interface GlobalConfig {
     "internal.termination.signal"?: number;
 
     /**
-     * Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version `broker.version.fallback` will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the `broker.version.fallback` fallback is used.
+     * **DEPRECATED** **Post-deprecation actions: remove this configuration property, brokers < 0.10.0 won't be supported anymore in librdkafka 3.x.** Request broker's supported API versions to adjust functionality to available protocol features. If set to false, or the ApiVersionRequest fails, the fallback version `broker.version.fallback` will be used. **NOTE**: Depends on broker version >=0.10.0. If the request is not supported by (an older) broker the `broker.version.fallback` fallback is used.
      *
      * @default true
      */
@@ -343,14 +357,14 @@ export interface GlobalConfig {
     "api.version.request.timeout.ms"?: number;
 
     /**
-     * Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade).
+     * **DEPRECATED** **Post-deprecation actions: remove this configuration property, brokers < 0.10.0 won't be supported anymore in librdkafka 3.x.** Dictates how long the `broker.version.fallback` fallback is used in the case the ApiVersionRequest fails. **NOTE**: The ApiVersionRequest is only issued when a new connection to the broker is made (such as after an upgrade).
      *
      * @default 0
      */
     "api.version.fallback.ms"?: number;
 
     /**
-     * Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests.
+     * **DEPRECATED** **Post-deprecation actions: remove this configuration property, brokers < 0.10.0 won't be supported anymore in librdkafka 3.x.** Older broker versions (before 0.10.0) provide no way for a client to query for supported protocol features (ApiVersionRequest, see `api.version.request`) making it impossible for the client to know what features it may use. As a workaround a user may set this property to the expected broker version and the client will automatically adjust its feature set accordingly if the ApiVersionRequest fails (or is disabled). The fallback broker version will be used for `api.version.fallback.ms`. Valid values are: 0.9.0, 0.8.2, 0.8.1, 0.8.0. Any other value >= 0.10, such as 0.10.2.1, enables ApiVersionRequests.
      *
      * @default 0.10.0
      */
@@ -425,6 +439,16 @@ export interface GlobalConfig {
      */
     "ssl.ca.location"?: string;
 
+    /**
+     * File or directory path to CA certificate(s) for verifying HTTPS endpoints, like `sasl.oauthbearer.token.endpoint.url` used for OAUTHBEARER/OIDC authentication. Mutually exclusive with `https.ca.pem`. Defaults: On Windows the system's CA certificates are automatically looked up in the Windows Root certificate store. On Mac OSX this configuration defaults to `probe`. It is recommended to install openssl using Homebrew, to provide CA certificates. On Linux install the distribution's ca-certificates package. If OpenSSL is statically linked or `https.ca.location` is set to `probe` a list of standard paths will be probed and the first one found will be used as the default CA certificate location path. If OpenSSL is dynamically linked the OpenSSL library's default path will be used (see `OPENSSLDIR` in `openssl version -a`).
+     */
+    "https.ca.location"?: string;
+
+    /**
+     * CA certificate string (PEM format) for verifying HTTPS endpoints. Mutually exclusive with `https.ca.location`. Optional: see `https.ca.location`.
+     */
+    "https.ca.pem"?: string;
+
     /**
      * CA certificate string (PEM format) for verifying the broker's key.
      */
@@ -584,6 +608,16 @@ export interface GlobalConfig {
      */
     "sasl.oauthbearer.client.id"?: string;
 
+    /**
+     * Alias for `sasl.oauthbearer.client.id`: Public identifier for the application. Must be unique across all clients that the authorization server handles. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.client.credentials.client.id"?: string;
+
+    /**
+     * Alias for `sasl.oauthbearer.client.secret`: Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc".
+     */
+    "sasl.oauthbearer.client.credentials.client.secret"?: string;
+
     /**
      * Client secret only known to the application and the authorization server. This should be a sufficiently random string that is not guessable. Only used when `sasl.oauthbearer.method` is set to "oidc".
      */
@@ -604,6 +638,88 @@ export interface GlobalConfig {
      */
     "sasl.oauthbearer.token.endpoint.url"?: string;
 
+    /**
+     * OAuth grant type to use when communicating with the identity provider.
+     *
+     * @default client_credentials
+     */
+    "sasl.oauthbearer.grant.type"?: 'client_credentials' | 'urn:ietf:params:oauth:grant-type:jwt-bearer';
+
+    /**
+     * Algorithm the client should use to sign the assertion sent to the identity provider and in the OAuth alg header in the JWT assertion.
+     *
+     * @default RS256
+     */
+    "sasl.oauthbearer.assertion.algorithm"?: 'RS256' | 'ES256';
+
+    /**
+     * Path to client's private key (PEM) used for authentication when using the JWT assertion.
+     */
+    "sasl.oauthbearer.assertion.private.key.file"?: string;
+
+    /**
+     * Private key passphrase for `sasl.oauthbearer.assertion.private.key.file` or `sasl.oauthbearer.assertion.private.key.pem`.
+     */
+    "sasl.oauthbearer.assertion.private.key.passphrase"?: string;
+
+    /**
+     * Client's private key (PEM) used for authentication when using the JWT assertion.
+     */
+    "sasl.oauthbearer.assertion.private.key.pem"?: string;
+
+    /**
+     * Path to the assertion file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     */
+    "sasl.oauthbearer.assertion.file"?: string;
+
+    /**
+     * JWT audience claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     */
+    "sasl.oauthbearer.assertion.claim.aud"?: string;
+
+    /**
+     * Assertion expiration time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     *
+     * @default 300
+     */
+    "sasl.oauthbearer.assertion.claim.exp.seconds"?: number;
+
+    /**
+     * JWT issuer claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     */
+    "sasl.oauthbearer.assertion.claim.iss"?: string;
+
+    /**
+     * JWT ID claim. When set to `true`, a random UUID is generated. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     *
+     * @default false
+     */
+    "sasl.oauthbearer.assertion.claim.jti.include"?: boolean;
+
+    /**
+     * Assertion not before time in seconds. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     *
+     * @default 60
+     */
+    "sasl.oauthbearer.assertion.claim.nbf.seconds"?: number;
+
+    /**
+     * JWT subject claim. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     */
+    "sasl.oauthbearer.assertion.claim.sub"?: string;
+
+    /**
+     * Path to the JWT template file. Only used when `sasl.oauthbearer.method` is set to "oidc" and JWT assertion is needed.
+     */
+    "sasl.oauthbearer.assertion.jwt.template.file"?: string;
+
+    /**
+     * Type of metadata-based authentication to use for OAUTHBEARER/OIDC `azure_imds` authenticates using the Azure IMDS endpoint. Sets a default value for `sasl.oauthbearer.token.endpoint.url` if missing. Configuration values specific of chosen authentication type can be passed through `sasl.oauthbearer.config`.
+     *
+     * @default none
+     */
+    "sasl.oauthbearer.metadata.authentication.type"?: 'none' | 'azure_imds';
+
     /**
      * List of plugin libraries to load (; separated). The library search path is platform dependent (see dlopen(3) for Unix and LoadLibrary() for Windows). If no filename extension is specified the platform-specific extension (such as .dll or .so) will be appended automatically.
      */
@@ -796,28 +912,28 @@ export interface ConsumerGlobalConfig extends GlobalConfig {
     "group.instance.id"?: string;
 
     /**
-     * The name of one or more partition assignment strategies. The elected group leader will use a strategy supported by all members of the group to assign partitions to group members. If there is more than one eligible strategy, preference is determined by the order of this list (strategies earlier in the list have higher priority). Cooperative and non-cooperative (eager) strategies must not be mixed. Available strategies: range, roundrobin, cooperative-sticky.
+     * The name of one or more partition assignment strategies. The elected group leader will use a strategy supported by all members of the group to assign partitions to group members. If there is more than one eligible strategy, preference is determined by the order of this list (strategies earlier in the list have higher priority). Cooperative and non-cooperative (eager)strategies must not be mixed. `partition.assignment.strategy` is not supported for `group.protocol=consumer`. Use `group.remote.assignor` instead. Available strategies: range, roundrobin, cooperative-sticky.
      *
      * @default range,roundrobin
      */
     "partition.assignment.strategy"?: string;
 
     /**
-     * Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the **broker** configuration properties `group.min.session.timeout.ms` and `group.max.session.timeout.ms`. Also see `max.poll.interval.ms`.
+     * Client group session and failure detection timeout. The consumer sends periodic heartbeats (heartbeat.interval.ms) to indicate its liveness to the broker. If no hearts are received by the broker for a group member within the session timeout, the broker will remove the consumer from the group and trigger a rebalance. The allowed range is configured with the **broker** configuration properties `group.min.session.timeout.ms` and `group.max.session.timeout.ms`. `session.timeout.ms` is not supported for `group.protocol=consumer`. It is set with the broker configuration property `group.consumer.session.timeout.ms` by default or can be configured through the AdminClient IncrementalAlterConfigs API. The allowed range is configured with the broker configuration properties `group.consumer.min.session.timeout.ms` and `group.consumer.max.session.timeout.ms`. Also see `max.poll.interval.ms`.
      *
      * @default 45000
      */
     "session.timeout.ms"?: number;
 
     /**
-     * Group session keepalive heartbeat interval.
+     * Group session keepalive heartbeat interval. `heartbeat.interval.ms` is not supported for `group.protocol=consumer`. It is set with the broker configuration property `group.consumer.heartbeat.interval.ms` by default or can be configured through the AdminClient IncrementalAlterConfigs API. The allowed range is configured with the broker configuration properties `group.consumer.min.heartbeat.interval.ms` and `group.consumer.max.heartbeat.interval.ms`.
      *
      * @default 3000
      */
     "heartbeat.interval.ms"?: number;
 
     /**
-     * Group protocol type for the `classic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`.
+     * Group protocol type for the `classic` group protocol. NOTE: Currently, the only supported group protocol type is `consumer`. `group.protocol.type` is not supported for `group.protocol=consumer`
      *
      * @default consumer
      */

package/deps/librdkafka/.semaphore/run-all-tests.yml

@@ -0,0 +1,77 @@
+version: v1.0
+name: run-all-tests
+
+agent:
+  machine:
+    type: s1-prod-ubuntu24-04-amd64-00
+
+execution_time_limit:
+  hours: 3
+
+global_job_config:
+  prologue:
+    commands:
+      - checkout
+      - '[[ -z "$GIT_REF" ]] || git checkout $GIT_REF'
+      - wget -O rapidjson-dev.deb https://launchpad.net/ubuntu/+archive/primary/+files/rapidjson-dev_1.1.0+dfsg2-3_all.deb
+      - sudo dpkg -i rapidjson-dev.deb
+      - sudo apt update
+      - sudo apt remove -y needrestart
+      - sudo apt install -y valgrind
+      - python3 -m pip install -U pip
+      - python3 -m pip -V
+      - (cd tests && python3 -m pip install -r requirements.txt)
+      - ./configure --install-deps --enable-werror --enable-devel
+      - make -j all
+      - make -j -C tests build
+      - sem-version java 17
+
+blocks:
+  - name: "Run all tests (x86_64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-amd64-1
+      prologue:
+        commands:
+          - if [[ "$TEST_ARCHES" != *"x86_64"* ]]; then exit 0; fi
+      jobs:
+        - name: "PLAINTEXT cluster (x86_64)"
+          env_vars:
+            - name: TEST_SSL
+              value: "False"
+          commands:
+            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi
+            - ./tests/run-all-tests.sh 
+        - name: "SSL cluster (x86_64)"
+          env_vars:
+            - name: TEST_SSL
+              value: "True"
+          commands:
+            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi
+            - ./tests/run-all-tests.sh
+  - name: "Run all tests (aarch64)"
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-arm64-1
+      prologue:
+        commands:
+          - if [[ "$TEST_ARCHES" != *"aarch64"* ]]; then exit 0; fi
+      jobs:
+        - name: "PLAINTEXT cluster (aarch64)"
+          env_vars:
+            - name: TEST_SSL
+              value: "False"
+          commands:
+            - if [[ "$TEST_TYPE" != *"plaintext"* ]]; then exit 0; fi
+            - ./tests/run-all-tests.sh
+        - name: "SSL cluster (aarch64)"
+          env_vars:
+            - name: TEST_SSL
+              value: "True"
+          commands:
+            - if [[ "$TEST_TYPE" != *"ssl"* ]]; then exit 0; fi
+            - ./tests/run-all-tests.sh

package/deps/librdkafka/.semaphore/semaphore-integration.yml

@@ -0,0 +1,250 @@
+version: v1.0
+name: 'librdkafka integration and all architectures tests pipeline'
+agent:
+  machine:
+    type: s1-prod-ubuntu24-04-arm64-00
+execution_time_limit:
+  hours: 3
+
+auto_cancel:
+  running:
+    when: "branch != 'master'"
+
+global_job_config:
+  prologue:
+    commands:
+      - checkout
+      - mkdir artifacts
+      - mkdir dest
+  env_vars:
+    - name: CACHE_TAG
+      value: '7'
+    - name: CI
+      value: 'true'
+    - name: KAFKA_VERSION
+      value: '4.0.0'
+    - name: CP_VERSION
+      value: '8.0.0'
+    # Shows plain output from docker build (no progress bars)
+    - name: BUILDKIT_PROGRESS
+      value: plain
+
+blocks:
+  - name: 'OSX arm64/m1'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-macos-15-arm64-8
+      env_vars:
+        - name: ARTIFACT_KEY
+          value: p-librdkafka__plat-osx__arch-arm64__lnk-all
+      epilogue:
+        commands:
+          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
+      jobs:
+        - name: 'Build'
+          commands:
+            - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
+            - make -j all examples check
+            - examples/rdkafka_example -X builtin.features
+            - otool -L src/librdkafka.dylib
+            - otool -L src-cpp/librdkafka++.dylib
+            - make -j -C tests build
+            - make -C tests run_local_quick
+            - DESTDIR="$PWD/dest" make install
+            - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
+
+
+  - name: 'OSX x64'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-macos-15-amd64-6
+      env_vars:
+        - name: ARTIFACT_KEY
+          value: p-librdkafka__plat-osx__arch-x64__lnk-all
+      epilogue:
+        commands:
+          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
+      jobs:
+        - name: 'Build'
+          commands:
+            - ./configure --install-deps --source-deps-only --enable-static --disable-lz4-ext --enable-strip
+            - make -j all examples check
+            - examples/rdkafka_example -X builtin.features
+            - otool -L src/librdkafka.dylib
+            - otool -L src-cpp/librdkafka++.dylib
+            - make -j -C tests build
+            - make -C tests run_local_quick
+            - DESTDIR="$PWD/dest" make install
+            - (cd dest && tar cvzf ../artifacts/librdkafka.tgz .)
+
+  - name: 'Linux Ubuntu amd64: integration tests'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-amd64-1
+      env_vars:
+        - name: CFLAGS
+          value: -std=gnu90 # Test minimum C standard, default in CentOS 7
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
+      jobs:
+        - name: 'Build and integration tests with "classic" protocol'
+          commands:
+            - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG}
+            - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG}
+              ./packaging/tools/run-integration-tests.sh
+              ${KAFKA_VERSION} ${CP_VERSION} classic
+            - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION}
+        - name: 'Build and integration tests with "consumer" protocol'
+          commands:
+            - cache restore trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG}
+            - packaging/tools/run-in-docker.sh test-runner-amd64-${CACHE_TAG}
+              ./packaging/tools/run-integration-tests.sh
+              ${KAFKA_VERSION} ${CP_VERSION} consumer
+            - cache store trivup-kafka-${KAFKA_VERSION}-${CACHE_TAG} tests/tmp-KafkaCluster/KafkaCluster/KafkaBrokerApp/kafka/${KAFKA_VERSION}
+
+
+  - name: 'Linux arm64: local quick tests'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-ubuntu24-04-arm64-1
+      prologue:
+        commands:
+          - '[[ -z $DOCKERHUB_APIKEY ]] || docker login --username $DOCKERHUB_USER --password $DOCKERHUB_APIKEY'
+      epilogue:
+        commands:
+          - '[[ -z $SEMAPHORE_GIT_TAG_NAME ]] || artifact push workflow artifacts/ --destination artifacts/${ARTIFACT_KEY}/'
+      jobs:
+        - name: 'Build: centos8 glibc +gssapi'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-std__extra-gssapi
+          commands:
+            - packaging/tools/build-release-artifacts.sh quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
+
+        - name: 'Build: centos8 glibc'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-linux__dist-centos8__arch-arm64__lnk-all
+          commands:
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi quay.io/pypa/manylinux_2_28_aarch64:2024.07.01-1 artifacts/librdkafka.tgz
+
+        - name: 'Build: alpine musl +gssapi'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all__extra-gssapi
+          commands:
+            - packaging/tools/build-release-artifacts.sh alpine:3.16.9 artifacts/librdkafka.tgz
+
+        - name: 'Build: alpine musl'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-linux__dist-alpine__arch-arm64__lnk-all
+          commands:
+            - packaging/tools/build-release-artifacts.sh --disable-gssapi alpine:3.16.9 artifacts/librdkafka.tgz
+
+
+  - name: 'Windows x64: MinGW-w64'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-windows
+      env_vars:
+        - name: CHERE_INVOKING
+          value: 'yes'
+        - name: MSYSTEM
+          value: UCRT64
+      prologue:
+        commands:
+          # Set up msys2
+          - "& .\\win32\\setup-msys2.ps1"
+      epilogue:
+        commands:
+          - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
+      jobs:
+        - name: 'Build: MinGW-w64 Dynamic'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-std
+          commands:
+            - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh ./artifacts/librdkafka.tgz'
+
+        - name: 'Build: MinGW-w64 Static'
+          env_vars:
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-windows__dist-mingw__arch-x64__lnk-static
+          commands:
+            - C:\msys64\usr\bin\bash -lc './packaging/mingw-w64/semaphoreci-build.sh --static ./artifacts/librdkafka.tgz'
+
+  - name: 'Windows x64: Windows SDK 10.0 / MSVC v142 / VS 2019'
+    dependencies: []
+    task:
+      agent:
+        machine:
+          type: s1-prod-windows
+      env_vars:
+        # Disable vcpkg telemetry
+        - name: VCPKG_DISABLE_METRICS
+          value: 'yes'
+      prologue:
+        commands:
+          # install vcpkg in the parent directory.
+          - cd ..
+          # Setup vcpkg
+          - "& .\\librdkafka\\win32\\setup-vcpkg.ps1"
+          - cd librdkafka
+      epilogue:
+        commands:
+          - Get-ChildItem . -include *.dll -recurse
+          - Get-ChildItem . -include *.lib -recurse
+          - if ($env:SEMAPHORE_GIT_TAG_NAME -ne "") { artifact push workflow artifacts/ --destination artifacts/$Env:ARTIFACT_KEY/ }
+      jobs:
+        - name: 'Build: MSVC x64'
+          env_vars:
+            - name: triplet
+              value: x64-windows
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-windows__dist-msvc__arch-x64__lnk-std
+          commands:
+            - "& .\\win32\\msbuild.ps1 -platform x64"
+            - "& .\\win32\\package-zip.ps1 -platform x64"
+        - name: 'Build: MSVC x86'
+          env_vars:
+            - name: triplet
+              value: x86-windows
+            - name: ARTIFACT_KEY
+              value: p-librdkafka__plat-windows__dist-msvc__arch-x86__lnk-std
+          commands:
+            - "& .\\win32\\msbuild.ps1 -platform Win32"
+            - "& .\\win32\\package-zip.ps1 -platform Win32"
+
+promotions:
+  - name: Run all tests on master commits
+    pipeline_file: run-all-tests.yml
+    parameters:
+      env_vars:
+        - required: true
+          name: TEST_KAFKA_GIT_REF
+          default_value: 4.0.0
+        - required: true
+          name: TEST_TYPE
+          default_value: plaintext,ssl
+        - required: true
+          name: TEST_ARCHES
+          default_value: x86_64,aarch64
+        - required: true
+          name: TEST_PARALLEL
+          default_value: "1"
+    auto_promote_on:
+      - result: passed
+        branch:
+        - "master"