@redpanda-data/docs-extensions-and-macros 4.4.2 → 4.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +585 -133
- package/cli-utils/convert-doc-links.js +58 -0
- package/cli-utils/generate-cluster-docs.sh +1 -1
- package/cli-utils/self-managed-docs-branch.js +90 -0
- package/cli-utils/start-cluster.sh +12 -2
- package/docker-compose/25.1/bootstrap.yml +67 -0
- package/docker-compose/25.1/docker-compose.yml +414 -0
- package/docker-compose/25.1/generate-profiles.yaml +77 -0
- package/docker-compose/25.1/rpk-profile.yaml +24 -0
- package/docker-compose/25.1/transactions-schema.json +37 -0
- package/docker-compose/25.1/transactions.md +46 -0
- package/docker-compose/25.1/transform/README.adoc +73 -0
- package/docker-compose/25.1/transform/go.mod +5 -0
- package/docker-compose/25.1/transform/go.sum +2 -0
- package/docker-compose/25.1/transform/regex.wasm +0 -0
- package/docker-compose/25.1/transform/transform.go +122 -0
- package/docker-compose/25.1/transform/transform.yaml +33 -0
- package/docker-compose/bootstrap.yml +0 -12
- package/package.json +1 -1
- package/tools/fetch-from-github.js +2 -2
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
const { URL } = require('url');
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Converts a docs.redpanda.com URL, optionally suffixed with a label in brackets, into an Antora xref resource ID string.
|
|
5
|
+
*
|
|
6
|
+
* If the input includes a label in square brackets (e.g., `[Label]`), the label is preserved and appended to the resulting xref.
|
|
7
|
+
*
|
|
8
|
+
* @param {string} input - A docs.redpanda.com URL, optionally followed by a label in square brackets.
|
|
9
|
+
* @returns {string} The corresponding Antora xref resource ID, with the label preserved if present.
|
|
10
|
+
*
|
|
11
|
+
* @throws {Error} If the input is not a valid URL or does not belong to docs.redpanda.com.
|
|
12
|
+
*/
|
|
13
|
+
function urlToXref(input) {
|
|
14
|
+
// Peel off an optional “[label]”
|
|
15
|
+
let urlPart = input;
|
|
16
|
+
let label = '';
|
|
17
|
+
const mLabel = input.match(/^(.*)\[([^\]]+)\]$/);
|
|
18
|
+
if (mLabel) {
|
|
19
|
+
urlPart = mLabel[1];
|
|
20
|
+
label = mLabel[2];
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
//Parse & validate
|
|
24
|
+
let url;
|
|
25
|
+
try {
|
|
26
|
+
url = new URL(urlPart);
|
|
27
|
+
} catch {
|
|
28
|
+
throw new Error(`Invalid URL: ${input}`);
|
|
29
|
+
}
|
|
30
|
+
if (!/docs\.redpanda\.com$/.test(url.hostname)) {
|
|
31
|
+
throw new Error(`Not a docs.redpanda.com URL: ${input}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Strip any leading “/docs”, “/docs/vX.Y”, “/docs/current”, “/vX.Y” or “/current”
|
|
35
|
+
let p = url.pathname.replace(
|
|
36
|
+
/^\/(?:docs(?:\/(?:v?\d+\.\d+|current))?|v?\d+\.\d+|current)\/?/,
|
|
37
|
+
''
|
|
38
|
+
);
|
|
39
|
+
// Drop trailing slash
|
|
40
|
+
p = p.replace(/\/$/, '');
|
|
41
|
+
const segments = p.split('/').filter(Boolean);
|
|
42
|
+
|
|
43
|
+
// Build module + path + .adoc
|
|
44
|
+
let xref;
|
|
45
|
+
if (segments.length === 0) {
|
|
46
|
+
xref = 'xref:index.adoc';
|
|
47
|
+
} else {
|
|
48
|
+
const moduleName = segments.shift();
|
|
49
|
+
const pagePath = segments.join('/');
|
|
50
|
+
const fileName = (pagePath || moduleName) + '.adoc';
|
|
51
|
+
xref = `xref:${moduleName}:${fileName}`;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Re-attach label if there was one
|
|
55
|
+
return label ? `${xref}[${label}]` : xref;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
module.exports = { urlToXref };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
const https = require('https');
|
|
2
|
+
const yaml = require('js-yaml');
|
|
3
|
+
const { spawnSync } = require('child_process');
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Retrieves the current Self-Managed documentation version from the remote antora.yml file.
|
|
7
|
+
*
|
|
8
|
+
* @returns {Promise<string>} Resolves with the version string (e.g., "25.1").
|
|
9
|
+
*
|
|
10
|
+
* @throws {Error} If the antora.yml file cannot be fetched, parsed, or if the version field is missing.
|
|
11
|
+
*/
|
|
12
|
+
function fetchRemoteAntoraVersion() {
|
|
13
|
+
const url = 'https://raw.githubusercontent.com/redpanda-data/docs/main/antora.yml'
|
|
14
|
+
return new Promise((resolve, reject) => {
|
|
15
|
+
https.get(url, res => {
|
|
16
|
+
if (res.statusCode !== 200) {
|
|
17
|
+
return reject(new Error(`Failed to fetch antora.yml: ${res.statusCode}`))
|
|
18
|
+
}
|
|
19
|
+
let body = ''
|
|
20
|
+
res.on('data', chunk => (body += chunk))
|
|
21
|
+
res.on('end', () => {
|
|
22
|
+
try {
|
|
23
|
+
const cfg = yaml.load(body)
|
|
24
|
+
if (cfg.version == null) {
|
|
25
|
+
throw new Error('version field missing')
|
|
26
|
+
}
|
|
27
|
+
const version = String(cfg.version).trim()
|
|
28
|
+
resolve(version)
|
|
29
|
+
} catch (err) {
|
|
30
|
+
reject(err)
|
|
31
|
+
}
|
|
32
|
+
})
|
|
33
|
+
}).on('error', reject)
|
|
34
|
+
})
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Determines the appropriate documentation branch for a given operator tag based on the remote Antora version.
|
|
39
|
+
*
|
|
40
|
+
* Normalizes the input tag, extracts the major.minor version, and applies version-specific logic to select the correct branch. Verifies that the chosen branch exists in the remote repository.
|
|
41
|
+
*
|
|
42
|
+
* @param {string} operatorTag - The operator tag to evaluate (e.g., "operator/v25.1.2" or "v25.1.2").
|
|
43
|
+
* @returns {Promise<string>} The name of the documentation branch to use.
|
|
44
|
+
*
|
|
45
|
+
* @throws {Error} If the tag cannot be parsed or if the determined branch does not exist in the remote repository.
|
|
46
|
+
*/
|
|
47
|
+
async function determineDocsBranch(operatorTag) {
|
|
48
|
+
// Strip any "operator/" prefix
|
|
49
|
+
const TAG = operatorTag.replace(/^(?:operator|release)\//, '')
|
|
50
|
+
// Pull in the remote Antora version
|
|
51
|
+
const ANTORA = await fetchRemoteAntoraVersion()
|
|
52
|
+
// Extract v<major>.<minor>
|
|
53
|
+
const mm = TAG.match(/^v(\d+\.\d+)/)
|
|
54
|
+
const filtered = mm ? `v${mm[1]}` : null
|
|
55
|
+
|
|
56
|
+
if (!filtered) {
|
|
57
|
+
throw new Error(`Could not parse major.minor from ${TAG}`)
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let branch
|
|
61
|
+
if (filtered === 'v2.4') {
|
|
62
|
+
if (ANTORA === '25.1') {
|
|
63
|
+
branch = 'main'
|
|
64
|
+
} else {
|
|
65
|
+
branch = 'v/24.3'
|
|
66
|
+
}
|
|
67
|
+
} else if (filtered === `v${ANTORA}`) {
|
|
68
|
+
branch = 'main'
|
|
69
|
+
} else {
|
|
70
|
+
branch = `v/${filtered.slice(1)}`
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Verify branch exists
|
|
74
|
+
const repo = 'https://github.com/redpanda-data/docs.git'
|
|
75
|
+
const ref = `refs/heads/${branch}`
|
|
76
|
+
const ok = spawnSync('git', ['ls-remote', '--exit-code', '--heads', repo, ref], {
|
|
77
|
+
stdio: 'ignore'
|
|
78
|
+
}).status === 0
|
|
79
|
+
|
|
80
|
+
if (!ok) {
|
|
81
|
+
throw new Error(`Docs branch ${branch} not found in ${repo}`)
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
return branch
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
module.exports = {
|
|
88
|
+
fetchRemoteAntoraVersion,
|
|
89
|
+
determineDocsBranch
|
|
90
|
+
}
|
|
@@ -20,12 +20,22 @@ if [[ "$TAG" != "latest" ]]; then
|
|
|
20
20
|
MAJOR_MINOR="$(echo "$TAG" | sed -E 's/^v?([0-9]+\.[0-9]+).*$/\1/')"
|
|
21
21
|
fi
|
|
22
22
|
|
|
23
|
+
# Conditionally override quickstart dir for >= 25.1
|
|
24
|
+
if [[ "$MAJOR_MINOR" =~ ^([0-9]+)\.([0-9]+)$ ]]; then
|
|
25
|
+
major="${BASH_REMATCH[1]}"
|
|
26
|
+
minor="${BASH_REMATCH[2]}"
|
|
27
|
+
|
|
28
|
+
if (( major > 25 )) || (( major == 25 && minor >= 1 )); then
|
|
29
|
+
QUICKSTART_DIR="$PACKAGE_ROOT/docker-compose/25.1"
|
|
30
|
+
fi
|
|
31
|
+
fi
|
|
32
|
+
|
|
23
33
|
# Fetch quickstart into package root if needed
|
|
24
34
|
if [[ ! -d "$QUICKSTART_DIR" ]]; then
|
|
25
35
|
echo "📥 Fetching Redpanda quickstart for ${MAJOR_MINOR}…"
|
|
26
36
|
if [[ "$TAG" == "latest" ]]; then
|
|
27
|
-
|
|
28
|
-
|
|
37
|
+
curl -sSLf --retry 3 https://docs.redpanda.com/redpanda-quickstart.tar.gz \
|
|
38
|
+
| tar -C "$PACKAGE_ROOT" -xzf -
|
|
29
39
|
else
|
|
30
40
|
curl -sSLf --retry 3 "https://docs.redpanda.com/${MAJOR_MINOR}-redpanda-quickstart.tar.gz" \
|
|
31
41
|
| tar -C "$PACKAGE_ROOT" -xzf -
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# =================================================================
|
|
2
|
+
# This file defines initial cluster properties for a Redpanda cluster.
|
|
3
|
+
# Some of these settings are intended for quickstart development and evaluation
|
|
4
|
+
# and are not suitable for production environments.
|
|
5
|
+
#
|
|
6
|
+
# For more information on bootstrap files, see:
|
|
7
|
+
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#configure-a-bootstrap-file
|
|
8
|
+
# =================================================================
|
|
9
|
+
|
|
10
|
+
#
|
|
11
|
+
# Enable SASL authentication for the Kafka and Admin APIs.
|
|
12
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#admin_api_require_auth
|
|
13
|
+
admin_api_require_auth: true
|
|
14
|
+
# At least one superuser is required to be able to create other SASL users
|
|
15
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#superusers
|
|
16
|
+
superusers:
|
|
17
|
+
- superuser
|
|
18
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_sasl
|
|
19
|
+
enable_sasl: true
|
|
20
|
+
# Allow topics to be created on first access.
|
|
21
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#auto_create_topics_enabled
|
|
22
|
+
auto_create_topics_enabled: true
|
|
23
|
+
# Enable data transforms.
|
|
24
|
+
# https://docs.redpanda.com/current/develop/data-transforms/how-transforms-work/
|
|
25
|
+
data_transforms_enabled: true
|
|
26
|
+
# Enable audit logging (enterprise feature).
|
|
27
|
+
# https://docs.redpanda.com/current/manage/audit-logging/
|
|
28
|
+
audit_enabled: true
|
|
29
|
+
# Enable Tiered Storage (enterprise feature).
|
|
30
|
+
# https://docs.redpanda.com/current/manage/tiered-storage/
|
|
31
|
+
cloud_storage_enabled: true
|
|
32
|
+
cloud_storage_region: local
|
|
33
|
+
cloud_storage_access_key: minio
|
|
34
|
+
cloud_storage_secret_key: redpandaTieredStorage7
|
|
35
|
+
cloud_storage_api_endpoint: minio
|
|
36
|
+
cloud_storage_api_endpoint_port: 9000
|
|
37
|
+
cloud_storage_disable_tls: true
|
|
38
|
+
cloud_storage_bucket: redpanda
|
|
39
|
+
cloud_storage_enable_remote_write: true
|
|
40
|
+
cloud_storage_enable_remote_read: true
|
|
41
|
+
# Forces segments to be uploaded to Tiered Storage faster for the purposes of the quickstart
|
|
42
|
+
# https://docs.redpanda.com/current/reference/properties/object-storage-properties/#cloud_storage_segment_max_upload_interval_sec
|
|
43
|
+
cloud_storage_segment_max_upload_interval_sec: 60
|
|
44
|
+
# Continuous Data Balancing (enterprise feature) continuously monitors your node and rack availability and disk usage. This enables self-healing clusters that dynamically balance partitions, ensuring smooth operations and optimal cluster performance.
|
|
45
|
+
# https://docs.redpanda.com/current/manage/cluster-maintenance/continuous-data-balancing/
|
|
46
|
+
partition_autobalancing_mode: continuous
|
|
47
|
+
# Enable Redpanda to collect consumer group metrics.
|
|
48
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_consumer_group_metrics
|
|
49
|
+
enable_consumer_group_metrics:
|
|
50
|
+
- "group"
|
|
51
|
+
- "partition"
|
|
52
|
+
- "consumer_lag"
|
|
53
|
+
# Lower the interval for the autogeneration of consumer group metrics.
|
|
54
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#consumer_group_lag_collection_interval_sec
|
|
55
|
+
consumer_group_lag_collection_interval_sec: 60
|
|
56
|
+
# Enable Redpanda to collect host metrics.
|
|
57
|
+
# https://docs.redpanda.com/current/reference/properties/cluster-properties/#enable_host_metrics
|
|
58
|
+
enable_host_metrics: true
|
|
59
|
+
# Enable for Iceberg metrics
|
|
60
|
+
iceberg_enabled: true
|
|
61
|
+
# Set up Iceberg REST catalog configuration
|
|
62
|
+
iceberg_catalog_type: rest
|
|
63
|
+
iceberg_rest_catalog_endpoint: http://catalog:8181
|
|
64
|
+
# Credentials are required, but the catalog ignores them
|
|
65
|
+
iceberg_rest_catalog_client_id: catalog
|
|
66
|
+
iceberg_rest_catalog_client_secret: catalog123
|
|
67
|
+
iceberg_catalog_commit_interval_ms: 5000
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
name: redpanda-cluster
|
|
2
|
+
networks:
|
|
3
|
+
redpanda_network:
|
|
4
|
+
driver: bridge
|
|
5
|
+
volumes:
|
|
6
|
+
redpanda-0: null
|
|
7
|
+
redpanda-1: null
|
|
8
|
+
redpanda-2: null
|
|
9
|
+
minio: null
|
|
10
|
+
services:
|
|
11
|
+
##################
|
|
12
|
+
# Redpanda Brokers #
|
|
13
|
+
##################
|
|
14
|
+
redpanda-0:
|
|
15
|
+
command:
|
|
16
|
+
- redpanda
|
|
17
|
+
- start
|
|
18
|
+
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
|
|
19
|
+
# Address the broker advertises to clients that connect to the Kafka API.
|
|
20
|
+
# Use the internal addresses to connect to the Redpanda brokers
|
|
21
|
+
# from inside the same Docker network.
|
|
22
|
+
# Use the external addresses to connect to the Redpanda brokers
|
|
23
|
+
# from outside the Docker network.
|
|
24
|
+
- --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
|
|
25
|
+
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
|
|
26
|
+
# Address the broker advertises to clients that connect to the HTTP Proxy.
|
|
27
|
+
- --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
|
|
28
|
+
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
|
|
29
|
+
# Redpanda brokers use the RPC API to communicate with each other internally.
|
|
30
|
+
- --rpc-addr redpanda-0:33145
|
|
31
|
+
- --advertise-rpc-addr redpanda-0:33145
|
|
32
|
+
# Mode dev-container uses well-known configuration properties for development in containers.
|
|
33
|
+
- --mode dev-container
|
|
34
|
+
# Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
|
|
35
|
+
- --smp 1
|
|
36
|
+
- --default-log-level=info
|
|
37
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
38
|
+
container_name: redpanda-0
|
|
39
|
+
# Sets the username and password of the bootstrap SCRAM superuser
|
|
40
|
+
# See https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/production-deployment/#bootstrap-a-user-account
|
|
41
|
+
environment:
|
|
42
|
+
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
43
|
+
volumes:
|
|
44
|
+
- redpanda-0:/var/lib/redpanda/data
|
|
45
|
+
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
46
|
+
networks:
|
|
47
|
+
- redpanda_network
|
|
48
|
+
ports:
|
|
49
|
+
- 18081:18081
|
|
50
|
+
- 18082:18082
|
|
51
|
+
- 19092:19092
|
|
52
|
+
- 19644:9644
|
|
53
|
+
healthcheck:
|
|
54
|
+
test: ["CMD", "rpk", "cluster", "info", "-X", "user=superuser", "-X", "pass=secretpassword"]
|
|
55
|
+
interval: 10s
|
|
56
|
+
timeout: 15s
|
|
57
|
+
retries: 10
|
|
58
|
+
depends_on:
|
|
59
|
+
minio:
|
|
60
|
+
condition: service_healthy
|
|
61
|
+
redpanda-1:
|
|
62
|
+
command:
|
|
63
|
+
- redpanda
|
|
64
|
+
- start
|
|
65
|
+
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
|
|
66
|
+
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
|
|
67
|
+
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
|
|
68
|
+
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
|
|
69
|
+
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
|
|
70
|
+
- --rpc-addr redpanda-1:33145
|
|
71
|
+
- --advertise-rpc-addr redpanda-1:33145
|
|
72
|
+
- --mode dev-container
|
|
73
|
+
- --smp 1
|
|
74
|
+
- --default-log-level=info
|
|
75
|
+
- --seeds redpanda-0:33145
|
|
76
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
77
|
+
container_name: redpanda-1
|
|
78
|
+
environment:
|
|
79
|
+
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
80
|
+
volumes:
|
|
81
|
+
- redpanda-1:/var/lib/redpanda/data
|
|
82
|
+
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
83
|
+
networks:
|
|
84
|
+
- redpanda_network
|
|
85
|
+
ports:
|
|
86
|
+
- 28081:28081
|
|
87
|
+
- 28082:28082
|
|
88
|
+
- 29092:29092
|
|
89
|
+
- 29644:9644
|
|
90
|
+
depends_on:
|
|
91
|
+
- redpanda-0
|
|
92
|
+
- minio
|
|
93
|
+
redpanda-2:
|
|
94
|
+
command:
|
|
95
|
+
- redpanda
|
|
96
|
+
- start
|
|
97
|
+
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
|
|
98
|
+
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
|
|
99
|
+
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
|
|
100
|
+
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
|
|
101
|
+
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
|
|
102
|
+
- --rpc-addr redpanda-2:33145
|
|
103
|
+
- --advertise-rpc-addr redpanda-2:33145
|
|
104
|
+
- --mode dev-container
|
|
105
|
+
- --smp 1
|
|
106
|
+
- --default-log-level=info
|
|
107
|
+
- --seeds redpanda-0:33145
|
|
108
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
109
|
+
container_name: redpanda-2
|
|
110
|
+
environment:
|
|
111
|
+
RP_BOOTSTRAP_USER: "superuser:secretpassword"
|
|
112
|
+
volumes:
|
|
113
|
+
- redpanda-2:/var/lib/redpanda/data
|
|
114
|
+
- ./bootstrap.yml:/etc/redpanda/.bootstrap.yaml
|
|
115
|
+
networks:
|
|
116
|
+
- redpanda_network
|
|
117
|
+
ports:
|
|
118
|
+
- 38081:38081
|
|
119
|
+
- 38082:38082
|
|
120
|
+
- 39092:39092
|
|
121
|
+
- 39644:9644
|
|
122
|
+
depends_on:
|
|
123
|
+
- redpanda-0
|
|
124
|
+
- minio
|
|
125
|
+
####################
|
|
126
|
+
# Redpanda Console #
|
|
127
|
+
####################
|
|
128
|
+
console:
|
|
129
|
+
container_name: redpanda-console
|
|
130
|
+
image: docker.redpanda.com/redpandadata/${CONSOLE_DOCKER_REPO:-console}:${REDPANDA_CONSOLE_VERSION:-latest}
|
|
131
|
+
networks:
|
|
132
|
+
- redpanda_network
|
|
133
|
+
entrypoint: /bin/sh
|
|
134
|
+
command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml && /app/console'
|
|
135
|
+
volumes:
|
|
136
|
+
- ./config:/tmp/config/
|
|
137
|
+
environment:
|
|
138
|
+
CONFIG_FILEPATH: ${CONFIG_FILEPATH:-/tmp/config.yml}
|
|
139
|
+
CONSOLE_CONFIG_FILE: |
|
|
140
|
+
# Configure a connection to the Redpanda cluster
|
|
141
|
+
# See https://docs.redpanda.com/current/console/config/connect-to-redpanda/
|
|
142
|
+
kafka:
|
|
143
|
+
brokers: ["redpanda-0:9092"]
|
|
144
|
+
sasl:
|
|
145
|
+
enabled: true
|
|
146
|
+
impersonateUser: true
|
|
147
|
+
schemaRegistry:
|
|
148
|
+
enabled: true
|
|
149
|
+
urls: ["http://redpanda-0:8081","http://redpanda-1:8081","http://redpanda-2:8081"]
|
|
150
|
+
authentication:
|
|
151
|
+
impersonateUser: true
|
|
152
|
+
redpanda:
|
|
153
|
+
adminApi:
|
|
154
|
+
enabled: true
|
|
155
|
+
urls: ["http://redpanda-0:9644","http://redpanda-1:9644","http://redpanda-2:9644"]
|
|
156
|
+
authentication:
|
|
157
|
+
basic:
|
|
158
|
+
username: superuser
|
|
159
|
+
password: secretpassword
|
|
160
|
+
impersonateUser: false
|
|
161
|
+
console:
|
|
162
|
+
# Configures Redpanda Console to fetch topic documentation from GitHub and display it in the UI.
|
|
163
|
+
# See https://docs.redpanda.com/current/console/config/topic-documentation/
|
|
164
|
+
topicDocumentation:
|
|
165
|
+
enabled: true
|
|
166
|
+
git:
|
|
167
|
+
enabled: true
|
|
168
|
+
repository:
|
|
169
|
+
url: https://github.com/redpanda-data/docs
|
|
170
|
+
branch: main
|
|
171
|
+
baseDirectory: tests/docker-compose
|
|
172
|
+
authentication:
|
|
173
|
+
jwtSigningKey: vazxnT+ZHtxKslK6QlDGovcYnSjTk/lKMmZ+mHrBVE+YdVDkLgSuP6AszAKe9Gvq
|
|
174
|
+
basic:
|
|
175
|
+
enabled: true
|
|
176
|
+
authorization:
|
|
177
|
+
roleBindings:
|
|
178
|
+
- roleName: admin
|
|
179
|
+
users:
|
|
180
|
+
- loginType: basic
|
|
181
|
+
name: superuser
|
|
182
|
+
ports:
|
|
183
|
+
- 8080:8080
|
|
184
|
+
depends_on:
|
|
185
|
+
redpanda-0:
|
|
186
|
+
condition: service_healthy
|
|
187
|
+
####################
|
|
188
|
+
# Redpanda Connect #
|
|
189
|
+
####################
|
|
190
|
+
connect:
|
|
191
|
+
container_name: redpanda-connect
|
|
192
|
+
image: docker.redpanda.com/redpandadata/connect
|
|
193
|
+
networks:
|
|
194
|
+
- redpanda_network
|
|
195
|
+
entrypoint: /bin/sh
|
|
196
|
+
depends_on:
|
|
197
|
+
redpanda-0:
|
|
198
|
+
condition: service_healthy
|
|
199
|
+
command: -c 'echo "$$CONNECT_CFG_FILE" > /tmp/connect.yml; /redpanda-connect -c /tmp/connect.yml'
|
|
200
|
+
environment:
|
|
201
|
+
# This Redpanda Connect configuration creates fake data,
|
|
202
|
+
# processes it, and writes the output to a set of topics.
|
|
203
|
+
#
|
|
204
|
+
# Input:
|
|
205
|
+
# - Uses Redpanda Connect's generate input to generate fake data.
|
|
206
|
+
# See https://docs.redpanda.com/redpanda-connect/components/inputs/generate/
|
|
207
|
+
# Pipeline:
|
|
208
|
+
# - Bloblang mapping to batch each input and map 1 message to 'logins'
|
|
209
|
+
# topic, and a random number (1-3) of messages to 'transaction' topic
|
|
210
|
+
# - Unarchive processor to parse the JSON array and extract each
|
|
211
|
+
# element into its own message.
|
|
212
|
+
# See https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/
|
|
213
|
+
# Output:
|
|
214
|
+
# - kafka_franz output to write the messages to the Redpanda brokers.
|
|
215
|
+
# See https://docs.redpanda.com/redpanda-connect/components/outputs/kafka_franz/
|
|
216
|
+
CONNECT_CFG_FILE: |
|
|
217
|
+
input:
|
|
218
|
+
generate:
|
|
219
|
+
interval: 1s
|
|
220
|
+
mapping: |
|
|
221
|
+
let first_name = fake("first_name")
|
|
222
|
+
let last_name = fake("last_name")
|
|
223
|
+
|
|
224
|
+
root.user_id = counter()
|
|
225
|
+
root.name = $$first_name + " " + $$last_name
|
|
226
|
+
root.email = ($$first_name.slice(0,1) + $$last_name + "@" + fake("domain_name")).lowercase()
|
|
227
|
+
root.ip = fake("ipv4")
|
|
228
|
+
root.login_time = now()
|
|
229
|
+
pipeline:
|
|
230
|
+
processors:
|
|
231
|
+
- mapping: |
|
|
232
|
+
root = range(0, random_int(min:2, max:4)).map_each(cust -> this)
|
|
233
|
+
- unarchive:
|
|
234
|
+
format: "json_array"
|
|
235
|
+
- mapping: |
|
|
236
|
+
if batch_index() == 0 {
|
|
237
|
+
meta topic = "logins"
|
|
238
|
+
root = this
|
|
239
|
+
} else {
|
|
240
|
+
meta topic = "transactions"
|
|
241
|
+
root.user_id = this.user_id
|
|
242
|
+
root.email = this.email
|
|
243
|
+
root.index = batch_index() - 1
|
|
244
|
+
root.product_url = fake("url")
|
|
245
|
+
root.price = fake("amount_with_currency")
|
|
246
|
+
root.timestamp = now()
|
|
247
|
+
}
|
|
248
|
+
output:
|
|
249
|
+
kafka_franz:
|
|
250
|
+
seed_brokers: [ "redpanda-0:9092" ]
|
|
251
|
+
topic: $${! metadata("topic") }
|
|
252
|
+
sasl:
|
|
253
|
+
- mechanism: SCRAM-SHA-256
|
|
254
|
+
password: secretpassword
|
|
255
|
+
username: superuser
|
|
256
|
+
####################
|
|
257
|
+
# rpk container to create the edu-filtered-domains topic #
|
|
258
|
+
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-create/
|
|
259
|
+
####################
|
|
260
|
+
createtopic:
|
|
261
|
+
command:
|
|
262
|
+
- topic
|
|
263
|
+
- create
|
|
264
|
+
- edu-filtered-domains
|
|
265
|
+
- -X user=superuser
|
|
266
|
+
- -X pass=secretpassword
|
|
267
|
+
- -X brokers=redpanda-0:9092
|
|
268
|
+
- --topic-config
|
|
269
|
+
- redpanda.iceberg.mode=key_value
|
|
270
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
271
|
+
networks:
|
|
272
|
+
- redpanda_network
|
|
273
|
+
depends_on:
|
|
274
|
+
redpanda-0:
|
|
275
|
+
condition: service_healthy
|
|
276
|
+
####################
|
|
277
|
+
# rpk container to register the schema #
|
|
278
|
+
# See https://docs.redpanda.com/current/manage/schema-reg/schema-reg-api/
|
|
279
|
+
####################
|
|
280
|
+
registerschema:
|
|
281
|
+
command:
|
|
282
|
+
- registry
|
|
283
|
+
- schema
|
|
284
|
+
- create
|
|
285
|
+
- transactions
|
|
286
|
+
- --schema
|
|
287
|
+
- /etc/redpanda/transactions-schema.json
|
|
288
|
+
- -X user=superuser
|
|
289
|
+
- -X pass=secretpassword
|
|
290
|
+
- -X registry.hosts=redpanda-0:8081
|
|
291
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
292
|
+
# Mount the local directory that contains your schema to the container.
|
|
293
|
+
volumes:
|
|
294
|
+
- ./transactions-schema.json:/etc/redpanda/transactions-schema.json
|
|
295
|
+
networks:
|
|
296
|
+
- redpanda_network
|
|
297
|
+
depends_on:
|
|
298
|
+
redpanda-0:
|
|
299
|
+
condition: service_healthy
|
|
300
|
+
####################
|
|
301
|
+
# rpk container to deploy a consumer group #
|
|
302
|
+
# See https://docs.redpanda.com/current/reference/rpk/rpk-topic/rpk-topic-consume/
|
|
303
|
+
####################
|
|
304
|
+
consumergroup:
|
|
305
|
+
command:
|
|
306
|
+
- topic
|
|
307
|
+
- consume
|
|
308
|
+
- transactions
|
|
309
|
+
- --group
|
|
310
|
+
- transactions-consumer
|
|
311
|
+
- -X user=superuser
|
|
312
|
+
- -X pass=secretpassword
|
|
313
|
+
- -X brokers=redpanda-0:9092
|
|
314
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
315
|
+
networks:
|
|
316
|
+
- redpanda_network
|
|
317
|
+
depends_on:
|
|
318
|
+
- deploytransform
|
|
319
|
+
####################
|
|
320
|
+
# rpk container to deploy the pre-built data transform #
|
|
321
|
+
# See https://docs.redpanda.com/current/develop/data-transforms/deploy/
|
|
322
|
+
####################
|
|
323
|
+
deploytransform:
|
|
324
|
+
command:
|
|
325
|
+
- transform
|
|
326
|
+
- deploy
|
|
327
|
+
- --file=/etc/redpanda/regex.wasm
|
|
328
|
+
- --name=regex
|
|
329
|
+
- --input-topic=logins
|
|
330
|
+
- --output-topic=edu-filtered-domains
|
|
331
|
+
- --var=PATTERN="[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.edu"
|
|
332
|
+
- --var=MATCH_VALUE=true
|
|
333
|
+
- -X user=superuser
|
|
334
|
+
- -X pass=secretpassword
|
|
335
|
+
- -X admin.hosts=redpanda-0:9644
|
|
336
|
+
image: docker.redpanda.com/redpandadata/${REDPANDA_DOCKER_REPO:-redpanda}:${REDPANDA_VERSION:-latest}
|
|
337
|
+
volumes:
|
|
338
|
+
- ./transform/regex.wasm:/etc/redpanda/regex.wasm
|
|
339
|
+
networks:
|
|
340
|
+
- redpanda_network
|
|
341
|
+
depends_on:
|
|
342
|
+
redpanda-0:
|
|
343
|
+
condition: service_healthy
|
|
344
|
+
createtopic:
|
|
345
|
+
condition: service_started
|
|
346
|
+
####################
|
|
347
|
+
# MinIO for Tiered Storage #
|
|
348
|
+
# See https://min.io/
|
|
349
|
+
#
|
|
350
|
+
# NOTE: MinIO is included in this quickstart for development and evaluation purposes only.
|
|
351
|
+
# It is not supported for production deployments of Redpanda.
|
|
352
|
+
#
|
|
353
|
+
# For production environments, use one of the supported object storage providers:
|
|
354
|
+
# https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/manual/production/requirements/#object-storage-providers-for-tiered-storage
|
|
355
|
+
####################
|
|
356
|
+
minio:
|
|
357
|
+
container_name: minio
|
|
358
|
+
image: minio/minio
|
|
359
|
+
command: server --console-address ":9001" /data
|
|
360
|
+
ports:
|
|
361
|
+
- 9000:9000
|
|
362
|
+
- 9001:9001
|
|
363
|
+
environment:
|
|
364
|
+
MINIO_ROOT_USER: minio
|
|
365
|
+
MINIO_ROOT_PASSWORD: redpandaTieredStorage7
|
|
366
|
+
MINIO_SERVER_URL: "http://minio:9000"
|
|
367
|
+
MINIO_REGION_NAME: local
|
|
368
|
+
MINIO_DOMAIN: minio
|
|
369
|
+
volumes:
|
|
370
|
+
- minio:/data
|
|
371
|
+
networks:
|
|
372
|
+
redpanda_network:
|
|
373
|
+
aliases:
|
|
374
|
+
- redpanda.minio
|
|
375
|
+
healthcheck:
|
|
376
|
+
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/ready"]
|
|
377
|
+
interval: 10s
|
|
378
|
+
timeout: 5s
|
|
379
|
+
retries: 3
|
|
380
|
+
mc:
|
|
381
|
+
depends_on:
|
|
382
|
+
minio:
|
|
383
|
+
condition: service_healthy
|
|
384
|
+
image: minio/mc
|
|
385
|
+
container_name: mc
|
|
386
|
+
networks:
|
|
387
|
+
- redpanda_network
|
|
388
|
+
environment:
|
|
389
|
+
- AWS_ACCESS_KEY_ID=minio
|
|
390
|
+
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
|
391
|
+
- AWS_REGION=local
|
|
392
|
+
entrypoint: >
|
|
393
|
+
/bin/sh -c "
|
|
394
|
+
until (/usr/bin/mc config host add minio http://minio:9000 minio redpandaTieredStorage7) do echo '...waiting...' && sleep 1; done;
|
|
395
|
+
/usr/bin/mc mb minio/redpanda;
|
|
396
|
+
/usr/bin/mc policy set public minio/redpanda;
|
|
397
|
+
tail -f /dev/null
|
|
398
|
+
"
|
|
399
|
+
catalog:
|
|
400
|
+
image: tabulario/iceberg-rest
|
|
401
|
+
container_name: iceberg-rest
|
|
402
|
+
networks:
|
|
403
|
+
- redpanda_network
|
|
404
|
+
depends_on:
|
|
405
|
+
- minio
|
|
406
|
+
ports:
|
|
407
|
+
- 8181:8181
|
|
408
|
+
environment:
|
|
409
|
+
- AWS_ACCESS_KEY_ID=minio
|
|
410
|
+
- AWS_SECRET_ACCESS_KEY=redpandaTieredStorage7
|
|
411
|
+
- AWS_REGION=local
|
|
412
|
+
- CATALOG_WAREHOUSE=s3://redpanda/
|
|
413
|
+
- CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
|
|
414
|
+
- CATALOG_S3_ENDPOINT=http://minio:9000
|