@redpanda-data/docs-extensions-and-macros 5.51.2 → 5.53.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools-mcp.js +4 -0
- package/bin/doc-tools.js +119 -0
- package/bin/mcp-tools/rpcn-docs.js +1 -0
- package/extensions/REFERENCE.adoc +19 -4
- package/extensions/generate-rp-connect-categories.js +76 -25
- package/extensions/generate-rp-connect-info.js +150 -40
- package/extensions/modify-connect-tag-playbook.js +183 -17
- package/extensions/util/connect-catalog.js +153 -0
- package/macros/rp-connect-components.js +23 -8
- package/package.json +1 -1
- package/tools/lint-strings/diff.js +4 -1
- package/tools/lint-strings/engine.js +5 -0
- package/tools/lint-strings/fixtures/connect/lint_connect.go +1 -1
- package/tools/lint-strings/fixtures/connect/repo/go.mod.txt +5 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/cli/lint.go +19 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/azure/common.go +20 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/azure/docs.go +8 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/azure/input_thing.go +32 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/azure/output_thing.go +12 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/widget/bloblang.go +21 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/impl/widget/scanner.go +12 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/plugins/info.csv +3 -0
- package/tools/lint-strings/fixtures/connect/repo/internal/retries/retries.go +13 -0
- package/tools/lint-strings/go-index.js +1143 -0
- package/tools/lint-strings/index.js +28 -2
- package/tools/lint-strings/rules/common.js +10 -2
- package/tools/lint-strings/surfaces/connect.js +1484 -137
- package/tools/lint-strings/surfaces/rpk.js +1 -0
- package/tools/redpanda-connect/generate-rpcn-connector-docs.js +23 -13
- package/tools/redpanda-connect/report-delta.js +1 -0
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +19 -13
- package/tools/redpanda-connect/templates/connector.hbs +20 -2
- package/tools/redpanda-release-notes/generate-release-notes.js +242 -0
- package/tools/redpanda-release-notes/parse-release-body.js +241 -0
- package/tools/redpanda-release-notes/render-release-notes.js +70 -0
- package/tools/rpk-docs/helpers/index.js +32 -35
package/bin/doc-tools-mcp.js
CHANGED
|
@@ -221,6 +221,10 @@ const tools = [
|
|
|
221
221
|
type: 'boolean',
|
|
222
222
|
description: 'Include Bloblang functions and methods in generation (optional, defaults to false)'
|
|
223
223
|
},
|
|
224
|
+
no_partials: {
|
|
225
|
+
type: 'boolean',
|
|
226
|
+
description: 'Skip the generated partials, config snippets, and Bloblang reference because the connect repo publishes them. What\'s new, diffs, drafts, nav, and the version bump still run (optional, defaults to false)'
|
|
227
|
+
},
|
|
224
228
|
data_dir: {
|
|
225
229
|
type: 'string',
|
|
226
230
|
description: 'Directory where versioned connect JSON files live (optional)'
|
package/bin/doc-tools.js
CHANGED
|
@@ -898,6 +898,7 @@ automation
|
|
|
898
898
|
.option('--template-bloblang <path>', 'Custom Handlebars template for bloblang function/method partials')
|
|
899
899
|
.option('--overrides <path>', 'Optional JSON file with overrides', 'docs-data/overrides.json')
|
|
900
900
|
.option('--include-bloblang', 'Include Bloblang functions and methods in generation')
|
|
901
|
+
.option('--no-partials', 'Skip the generated partials, config snippets, and Bloblang reference because the connect repo publishes them. What\'s new, diffs, drafts, nav, and the version bump still run')
|
|
901
902
|
.option('--prune-orphaned-descriptions', 'Allow the description-partial orphan sweep to blank more than 10% of the tree. Only use this when the connector dataset is confirmed complete: an incomplete dataset looks identical to a mass upstream deletion and the sweep blanks published content')
|
|
902
903
|
.option('--cloud-version <version>', 'Cloud binary version (default: auto-detect latest)')
|
|
903
904
|
.option('--cgo-version <version>', 'cgo binary version (default: same as cloud-version)')
|
|
@@ -2001,6 +2002,124 @@ automation
|
|
|
2001
2002
|
}
|
|
2002
2003
|
})
|
|
2003
2004
|
|
|
2005
|
+
/**
|
|
2006
|
+
* generate redpanda-release-notes
|
|
2007
|
+
*
|
|
2008
|
+
* @description
|
|
2009
|
+
* Converts a Self-Managed rpchangelog release body into a candidate AsciiDoc
|
|
2010
|
+
* section and inserts it into the Redpanda Release Notes page, newest-first.
|
|
2011
|
+
* Applies the mechanical guards: GA tags only (a prerelease is a no-op), the
|
|
2012
|
+
* `:earliest-tracked-version:` floor, and idempotency (an already-present
|
|
2013
|
+
* version is a no-op). The section carries de-noised source prose as plain
|
|
2014
|
+
* bullets — it is a CANDIDATE for the curation step that rewrites voice, adds
|
|
2015
|
+
* `Area::` labels, normalizes units, and phrases CVEs.
|
|
2016
|
+
*
|
|
2017
|
+
* @why
|
|
2018
|
+
* Release notes are the one reference surface whose source of truth is
|
|
2019
|
+
* engineer-authored prose rather than code, so the deterministic part
|
|
2020
|
+
* (parsing, de-noising, de-duplicating backports, ordering, insertion) is
|
|
2021
|
+
* split from the editorial part. This command is the deterministic half; it is
|
|
2022
|
+
* unit-tested and makes no editorial change.
|
|
2023
|
+
*
|
|
2024
|
+
* @example
|
|
2025
|
+
* # Phase 1: build the candidate the curation skill will edit (no page touched).
|
|
2026
|
+
* # Paths must stay inside the repository; write the fetched body somewhere local.
|
|
2027
|
+
* npx doc-tools generate redpanda-release-notes \
|
|
2028
|
+
* --tag v26.2.3 --date 2026-09-15 --body docs-data/v26.2.3-body.md --section-only > candidate.adoc
|
|
2029
|
+
*
|
|
2030
|
+
* # (the sm-release-notes skill curates candidate.adoc into curated.adoc)
|
|
2031
|
+
*
|
|
2032
|
+
* # Phase 2: insert the curated section into the page under the guards
|
|
2033
|
+
* npx doc-tools generate redpanda-release-notes \
|
|
2034
|
+
* --tag v26.2.3 --section-file curated.adoc
|
|
2035
|
+
*
|
|
2036
|
+
* @requirements
|
|
2037
|
+
* - Phase 1 needs --body (+ --date); phase 2 needs --section-file. Fetching the
|
|
2038
|
+
* body from the private streaming-enterprise release, and running the curation
|
|
2039
|
+
* skill between the two phases, are wired by the docs-repo workflow, not here.
|
|
2040
|
+
*/
|
|
2041
|
+
automation
|
|
2042
|
+
.command('redpanda-release-notes')
|
|
2043
|
+
.description('Generate a Self-Managed release-notes section from an rpchangelog release body')
|
|
2044
|
+
.requiredOption('--tag <tag>', 'GA release tag, such as v26.2.3')
|
|
2045
|
+
.option('--body <file>', 'Phase 1: rpchangelog release body markdown to build a candidate from (relative to repo root, must stay inside the repository)')
|
|
2046
|
+
.option('--section-file <file>', 'Phase 2: a curated section (from the sm-release-notes skill) to insert (relative to repo root, must stay inside the repository)')
|
|
2047
|
+
.option('--date <date>', 'Authored release date as YYYY-MM-DD (required with --body; confirmed on the PR)')
|
|
2048
|
+
.option('--page <file>', 'Target release-notes page (relative to repo root, must stay inside the repository)', 'modules/reference/pages/releases/redpanda.adoc')
|
|
2049
|
+
.option('--section-only', 'Phase 1 only: print the candidate section and do not read or modify the page')
|
|
2050
|
+
.option('--dry-run', 'Print output to stdout instead of writing the page')
|
|
2051
|
+
.action(async (options) => {
|
|
2052
|
+
const {
|
|
2053
|
+
generateReleaseNotes,
|
|
2054
|
+
buildReleaseSection,
|
|
2055
|
+
isGaTag,
|
|
2056
|
+
} = require('../tools/redpanda-release-notes/generate-release-notes.js')
|
|
2057
|
+
try {
|
|
2058
|
+
const repoRoot = findRepoRoot()
|
|
2059
|
+
|
|
2060
|
+
// Exactly one input: --body (phase 1, build a candidate) XOR --section-file
|
|
2061
|
+
// (phase 2, insert the curated section). Curation sits between them.
|
|
2062
|
+
const hasBody = Boolean(options.body)
|
|
2063
|
+
const hasSection = Boolean(options.sectionFile)
|
|
2064
|
+
if (hasBody === hasSection) {
|
|
2065
|
+
throw new Error('Provide exactly one of --body (phase 1) or --section-file (phase 2).')
|
|
2066
|
+
}
|
|
2067
|
+
if (hasBody && !options.date) {
|
|
2068
|
+
throw new Error('--date is required with --body.')
|
|
2069
|
+
}
|
|
2070
|
+
|
|
2071
|
+
const readInside = (rel, label) => {
|
|
2072
|
+
const abs = resolveInsideRepo(repoRoot, rel, label)
|
|
2073
|
+
if (!fs.existsSync(abs)) throw new Error(`File not found for ${label}: ${abs}`)
|
|
2074
|
+
return abs
|
|
2075
|
+
}
|
|
2076
|
+
|
|
2077
|
+
// Phase-1 candidate preview: emit the candidate for the curation step. No
|
|
2078
|
+
// page is read or written — this is the LLM step's input, nothing else.
|
|
2079
|
+
if (options.sectionOnly) {
|
|
2080
|
+
if (!hasBody) throw new Error('--section-only applies to --body (candidate generation).')
|
|
2081
|
+
// Same GA-only contract as the insert path: a non-GA tag is a clean
|
|
2082
|
+
// no-op, not an "invalid version" throw from buildReleaseSection.
|
|
2083
|
+
if (!isGaTag(options.tag)) {
|
|
2084
|
+
console.log(`[release-notes] Skipped: not a GA tag: ${options.tag}`)
|
|
2085
|
+
return
|
|
2086
|
+
}
|
|
2087
|
+
const body = fs.readFileSync(readInside(options.body, '--body'), 'utf8')
|
|
2088
|
+
const section = buildReleaseSection({ body, version: options.tag, date: options.date })
|
|
2089
|
+
process.stdout.write(section)
|
|
2090
|
+
return
|
|
2091
|
+
}
|
|
2092
|
+
|
|
2093
|
+
const pagePath = readInside(options.page, '--page')
|
|
2094
|
+
const pageContent = fs.readFileSync(pagePath, 'utf8')
|
|
2095
|
+
const genArgs = { tag: options.tag, date: options.date, pageContent }
|
|
2096
|
+
if (hasBody) {
|
|
2097
|
+
genArgs.body = fs.readFileSync(readInside(options.body, '--body'), 'utf8')
|
|
2098
|
+
} else {
|
|
2099
|
+
genArgs.section = fs.readFileSync(readInside(options.sectionFile, '--section-file'), 'utf8')
|
|
2100
|
+
}
|
|
2101
|
+
const result = generateReleaseNotes(genArgs)
|
|
2102
|
+
|
|
2103
|
+
if (result.status === 'skipped') {
|
|
2104
|
+
// A guard miss is a clean no-op, not an error: the workflow runs this on
|
|
2105
|
+
// every release and most runs have nothing to add.
|
|
2106
|
+
console.log(`[release-notes] Skipped: ${result.reason}`)
|
|
2107
|
+
return
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
if (options.dryRun) {
|
|
2111
|
+
process.stdout.write(result.content)
|
|
2112
|
+
console.log(`\nDone: (dry-run) updated page for ${options.tag} printed to stdout.`)
|
|
2113
|
+
} else {
|
|
2114
|
+
fs.writeFileSync(pagePath, result.content, 'utf8')
|
|
2115
|
+
console.log(`Done: Inserted the ${options.tag} section into ${pagePath}`)
|
|
2116
|
+
}
|
|
2117
|
+
} catch (err) {
|
|
2118
|
+
console.error(`Error: Failed to generate release notes: ${err.message}`)
|
|
2119
|
+
process.exit(1)
|
|
2120
|
+
}
|
|
2121
|
+
})
|
|
2122
|
+
|
|
2004
2123
|
/**
|
|
2005
2124
|
* generate crd-spec
|
|
2006
2125
|
*
|
|
@@ -41,6 +41,7 @@ function generateRpConnectDocs(args = {}) {
|
|
|
41
41
|
if (args.draft_missing) cmdArgs.push('--draft-missing');
|
|
42
42
|
if (args.update_whats_new) cmdArgs.push('--update-whats-new');
|
|
43
43
|
if (args.include_bloblang) cmdArgs.push('--include-bloblang');
|
|
44
|
+
if (args.no_partials) cmdArgs.push('--no-partials');
|
|
44
45
|
if (args.skip_cloud_detection) cmdArgs.push('--skip-cloud-detection');
|
|
45
46
|
if (args.skip_binary_analysis) cmdArgs.push('--skip-binary-analysis');
|
|
46
47
|
if (args.skip_cloud_analysis) cmdArgs.push('--skip-cloud-analysis');
|
|
@@ -404,13 +404,28 @@ You can optionally display pages only if they match the component and version of
|
|
|
404
404
|
|
|
405
405
|
== Redpanda Connect tag modifier
|
|
406
406
|
|
|
407
|
-
This extension
|
|
407
|
+
This extension sources the Redpanda Connect reference partials (field reference, examples, metadata, descriptions, config snippets, and the Bloblang reference) from the `docs/` directory of the connect repository, at the latest Connect release.
|
|
408
408
|
|
|
409
|
-
|
|
409
|
+
Add the connect repository as a content source next to rp-connect-docs:
|
|
410
410
|
|
|
411
|
-
|
|
411
|
+
```yaml
|
|
412
|
+
content:
|
|
413
|
+
sources:
|
|
414
|
+
- url: https://github.com/redpanda-data/connect
|
|
415
|
+
tags: latest # replaced with the latest release tag at build time
|
|
416
|
+
start_path: docs
|
|
417
|
+
```
|
|
418
|
+
|
|
419
|
+
At build time the extension:
|
|
420
|
+
|
|
421
|
+
* Pins a source that sets `tags: latest` to the latest Connect release tag and clears its branches (a source with any other refs, such as a PR branch for a preview, is left as configured), so the site never shows fields from unreleased code on connect's `main` branch. It asks the GitHub API for the latest release and falls back to the highest stable `vX.Y.Z` tag from `git ls-remote`. If neither works, the build stops with an error.
|
|
422
|
+
* Keeps only the `modules/components/partials/` and `modules/components/examples/` files of the `connect` component from that source. Any file that another source, such as rp-connect-docs, already provides is skipped, and so is content from older Connect tags that name their component `redpanda-connect`. Antora fails the build on a duplicate page or partial, so you can add the source to a playbook before or after rp-connect-docs stops committing its own generated copies.
|
|
423
|
+
|
|
424
|
+
A local clone whose directory is named `connect` is filtered the same way but not pinned, so a local playbook can use `branches: HEAD` to test uncommitted changes.
|
|
425
|
+
|
|
426
|
+
=== Environment variables
|
|
412
427
|
|
|
413
|
-
|
|
428
|
+
- `REDPANDA_GITHUB_TOKEN` (optional): A GitHub token for the release lookup. Without it, the extension sends unauthenticated API requests, which GitHub can rate limit. The `git ls-remote` fallback needs no token.
|
|
414
429
|
|
|
415
430
|
=== Configuration options
|
|
416
431
|
|
|
@@ -1,6 +1,65 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
4
|
+
const { normalizeType, typeFromRelative } = require('./util/connect-catalog')
|
|
5
|
+
|
|
6
|
+
// Page :type: values for each data type. Pages use metrics where the data says metric.
|
|
7
|
+
const PAGE_TYPE = { metric: 'metrics' }
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Indexes translated catalog rows (from generate-rp-connect-info) by
|
|
11
|
+
* `name:type`. Names are not unique across types: parquet is a certified input
|
|
12
|
+
* and a community processor, and sql is a certified cache and a community
|
|
13
|
+
* output, so a name-only key lets one type's row overwrite another's.
|
|
14
|
+
*/
|
|
15
|
+
function buildRowLookup (rows) {
|
|
16
|
+
const lookup = new Map()
|
|
17
|
+
for (const row of rows || []) {
|
|
18
|
+
const name = String(row.connector || '').trim()
|
|
19
|
+
const type = normalizeType(row.type)
|
|
20
|
+
if (name && type) lookup.set(`${name}:${type}`, row)
|
|
21
|
+
}
|
|
22
|
+
return lookup
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* The catalog facts for one connector page, from its data row when there is
|
|
27
|
+
* one and from the page's own attributes otherwise.
|
|
28
|
+
* - type: the page's :type:, or the type its directory implies when the data
|
|
29
|
+
* has a row for that name and type (drafts no longer write :type:)
|
|
30
|
+
* - deprecated: the data row's deprecated flag (info.csv deprecated column, or
|
|
31
|
+
* catalog.json status), so a hand-kept :status: cannot hide a live component
|
|
32
|
+
* or list a deprecated one. Pages without a data row fall back to :status:.
|
|
33
|
+
* - categories: catalog.json categories when the row has them, else :categories:
|
|
34
|
+
*/
|
|
35
|
+
function resolveComponent ({ name, relative, pageType, pageStatus, pageCategories }, lookup) {
|
|
36
|
+
const dirType = typeFromRelative(relative)
|
|
37
|
+
const key = (t) => `${name}:${normalizeType(t)}`
|
|
38
|
+
let type = pageType || null
|
|
39
|
+
let row = type ? lookup.get(key(type)) : undefined
|
|
40
|
+
if (!type && dirType && lookup.has(key(dirType))) {
|
|
41
|
+
row = lookup.get(key(dirType))
|
|
42
|
+
type = PAGE_TYPE[dirType] || dirType
|
|
43
|
+
}
|
|
44
|
+
if (!type) return null
|
|
45
|
+
const deprecated = row
|
|
46
|
+
? row.deprecated === 'y' || row.status === 'deprecated'
|
|
47
|
+
: pageStatus === 'deprecated'
|
|
48
|
+
const categories = row && Array.isArray(row.categories) && row.categories.length
|
|
49
|
+
? row.categories.join(', ')
|
|
50
|
+
: pageCategories
|
|
51
|
+
return {
|
|
52
|
+
type,
|
|
53
|
+
row,
|
|
54
|
+
deprecated,
|
|
55
|
+
categories,
|
|
56
|
+
supportLevel: row ? (row.support_level || 'community').toLowerCase() : null,
|
|
57
|
+
isEnterprise: row ? row.is_licensed === 'Yes' : false
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
module.exports.buildRowLookup = buildRowLookup
|
|
62
|
+
module.exports.resolveComponent = resolveComponent
|
|
4
63
|
|
|
5
64
|
/**
|
|
6
65
|
* Redpanda Connect Category Aggregation Extension
|
|
@@ -38,19 +97,9 @@ module.exports.register = function ({ config }) {
|
|
|
38
97
|
return
|
|
39
98
|
}
|
|
40
99
|
|
|
41
|
-
|
|
42
|
-
const supportLookup = new Map()
|
|
43
|
-
for (const row of csvData.data) {
|
|
44
|
-
const connector = row.connector
|
|
45
|
-
if (connector) {
|
|
46
|
-
supportLookup.set(connector, {
|
|
47
|
-
supportLevel: (row.support_level || 'community').toLowerCase(),
|
|
48
|
-
isEnterprise: row.is_licensed === 'Yes'
|
|
49
|
-
})
|
|
50
|
-
}
|
|
51
|
-
}
|
|
100
|
+
const rowLookup = buildRowLookup(csvData.data)
|
|
52
101
|
|
|
53
|
-
logger.info(`Loaded support data for ${
|
|
102
|
+
logger.info(`Loaded support data for ${rowLookup.size} connectors from CSV`)
|
|
54
103
|
|
|
55
104
|
const connectCategoriesData = {}
|
|
56
105
|
const flatComponentsData = []
|
|
@@ -71,8 +120,6 @@ module.exports.register = function ({ config }) {
|
|
|
71
120
|
const attrs = file.asciidoc?.attributes || {}
|
|
72
121
|
|
|
73
122
|
// Get attributes - prefer API, fallback to content parsing
|
|
74
|
-
const fileType = attrs.type || extractAttribute(file, 'type')
|
|
75
|
-
const categories = attrs.categories || extractAttribute(file, 'categories')
|
|
76
123
|
const status = attrs.status || extractAttribute(file, 'status')
|
|
77
124
|
const driverSupport = attrs['driver-support'] || extractAttribute(file, 'driver-support')
|
|
78
125
|
const cacheSupport = attrs['cache-support'] || extractAttribute(file, 'cache-support')
|
|
@@ -81,23 +128,27 @@ module.exports.register = function ({ config }) {
|
|
|
81
128
|
const pubUrl = file.pub.url
|
|
82
129
|
const name = file.src.stem
|
|
83
130
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
131
|
+
const resolved = resolveComponent({
|
|
132
|
+
name,
|
|
133
|
+
relative: file.src.relative,
|
|
134
|
+
pageType: attrs.type || extractAttribute(file, 'type'),
|
|
135
|
+
pageStatus: status,
|
|
136
|
+
pageCategories: attrs.categories || extractAttribute(file, 'categories')
|
|
137
|
+
}, rowLookup)
|
|
138
|
+
if (!resolved) continue
|
|
139
|
+
const { type: fileType, categories, isEnterprise } = resolved
|
|
87
140
|
|
|
88
141
|
// Skip deprecated components
|
|
89
|
-
if (
|
|
142
|
+
if (resolved.deprecated) continue
|
|
90
143
|
|
|
91
|
-
|
|
92
|
-
const csvInfo = supportLookup.get(name)
|
|
93
|
-
const isEnterprise = csvInfo?.isEnterprise || false
|
|
144
|
+
let componentStatus = status || 'community'
|
|
94
145
|
|
|
95
|
-
// Determine status from
|
|
96
|
-
if (
|
|
97
|
-
if (
|
|
146
|
+
// Determine status from the data row's support level (data takes precedence)
|
|
147
|
+
if (resolved.supportLevel) {
|
|
148
|
+
if (resolved.supportLevel === 'certified' || resolved.supportLevel === 'enterprise') {
|
|
98
149
|
componentStatus = 'certified'
|
|
99
150
|
} else {
|
|
100
|
-
componentStatus =
|
|
151
|
+
componentStatus = resolved.supportLevel
|
|
101
152
|
}
|
|
102
153
|
}
|
|
103
154
|
|
|
@@ -4,6 +4,7 @@ const { raiseListenerLimit } = require('./util/raise-listener-limit')
|
|
|
4
4
|
const fs = require('fs')
|
|
5
5
|
const path = require('path')
|
|
6
6
|
const Papa = require('papaparse')
|
|
7
|
+
const catalogUtil = require('./util/connect-catalog')
|
|
7
8
|
|
|
8
9
|
// Default configuration - can be overridden via playbook config
|
|
9
10
|
const DEFAULTS = {
|
|
@@ -12,6 +13,31 @@ const DEFAULTS = {
|
|
|
12
13
|
githubRepo: 'connect'
|
|
13
14
|
}
|
|
14
15
|
|
|
16
|
+
// Connector pages include the generated field reference from the connect repo
|
|
17
|
+
// (see the modify-connect-tag-playbook extension). If a playbook has the
|
|
18
|
+
// Connect pages but none of those partials, every connector page would publish
|
|
19
|
+
// without its fields, and Antora only logs the missing includes. Fail the
|
|
20
|
+
// build instead so the playbook gets fixed before anything is published.
|
|
21
|
+
function assertConnectReferencePresent (contentCatalog) {
|
|
22
|
+
const component = contentCatalog.getComponents().find((c) => c.name === 'connect')
|
|
23
|
+
if (!component) return
|
|
24
|
+
// Only connector pages (inputs/kafka.adoc and so on) include field
|
|
25
|
+
// partials. Overview pages at the module root don't.
|
|
26
|
+
const pages = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'page' })
|
|
27
|
+
.filter((p) => p.src.relative.includes('/'))
|
|
28
|
+
if (!pages.length) return
|
|
29
|
+
const fields = contentCatalog.findBy({ component: 'connect', module: 'components', family: 'partial' })
|
|
30
|
+
.filter((f) => f.src.relative.startsWith('fields/'))
|
|
31
|
+
if (fields.length) return
|
|
32
|
+
throw new Error(
|
|
33
|
+
'The connect component has connector pages but no generated field partials (components:partial$fields/*). ' +
|
|
34
|
+
'Add the connect repository as a content source (url: https://github.com/redpanda-data/connect, tags: latest, start_path: docs) ' +
|
|
35
|
+
'and register the modify-connect-tag-playbook extension, or check that the latest Connect release includes docs/modules/components.'
|
|
36
|
+
)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
module.exports.assertConnectReferencePresent = assertConnectReferencePresent
|
|
40
|
+
|
|
15
41
|
module.exports.register = function ({ config }) {
|
|
16
42
|
raiseListenerLimit(this)
|
|
17
43
|
const logger = this.getLogger('redpanda-connect-info-extension')
|
|
@@ -38,6 +64,7 @@ module.exports.register = function ({ config }) {
|
|
|
38
64
|
|
|
39
65
|
// Use 'on' and return the promise so Antora waits for async completion
|
|
40
66
|
this.on('contentClassified', ({ contentCatalog }) => {
|
|
67
|
+
assertConnectReferencePresent(contentCatalog)
|
|
41
68
|
return processContent(contentCatalog)
|
|
42
69
|
})
|
|
43
70
|
|
|
@@ -65,12 +92,8 @@ module.exports.register = function ({ config }) {
|
|
|
65
92
|
const pages = contentCatalog.getPages()
|
|
66
93
|
|
|
67
94
|
try {
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
|
|
71
|
-
// Fetch CSV data (from local file first, then GitHub as fallback)
|
|
72
|
-
const csvData = await fetchCSV(localCsvPath, connectVersion, logger)
|
|
73
|
-
const parsedData = Papa.parse(csvData, { header: true, skipEmptyLines: true })
|
|
95
|
+
const rawRows = await loadCatalogRows(contentCatalog)
|
|
96
|
+
const parsedData = { data: rawRows }
|
|
74
97
|
const enrichedData = translateCsvData(parsedData, pages, logger)
|
|
75
98
|
parsedData.data = enrichedData
|
|
76
99
|
translatedRows = enrichedData
|
|
@@ -107,8 +130,46 @@ module.exports.register = function ({ config }) {
|
|
|
107
130
|
}
|
|
108
131
|
}
|
|
109
132
|
|
|
133
|
+
// Raw catalog rows (info.csv column names) for this build.
|
|
134
|
+
//
|
|
135
|
+
// The generated partials/platforms/catalog.json from the connect content
|
|
136
|
+
// source comes first: it is pinned to the same ref as the reference content
|
|
137
|
+
// and carries status, categories, and cgo data that info.csv lacks. info.csv
|
|
138
|
+
// still supplies the SQL driver rows, which are not components and so are
|
|
139
|
+
// not in the catalog. Without catalog.json, info.csv supplies every row.
|
|
140
|
+
async function loadCatalogRows (contentCatalog) {
|
|
141
|
+
const catalogFile = catalogUtil.findConnectCatalogFile(contentCatalog)
|
|
142
|
+
let catalogRows = null
|
|
143
|
+
if (catalogFile) {
|
|
144
|
+
try {
|
|
145
|
+
catalogRows = catalogUtil.catalogEntriesToCsvRows(JSON.parse(catalogFile.contents.toString('utf8')))
|
|
146
|
+
logger.info(`Loaded ${catalogRows.length} components from ${describeFile(catalogFile)}`)
|
|
147
|
+
} catch (error) {
|
|
148
|
+
logger.warn(`Could not read ${describeFile(catalogFile)}, so falling back to info.csv: ${error.message}`)
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
let csvRows = []
|
|
153
|
+
try {
|
|
154
|
+
const csvText = await fetchCSV(localCsvPath, contentCatalog)
|
|
155
|
+
csvRows = Papa.parse(csvText, { header: true, skipEmptyLines: true }).data
|
|
156
|
+
} catch (error) {
|
|
157
|
+
if (!catalogRows) throw error
|
|
158
|
+
logger.warn(`Could not fetch info.csv for the SQL driver rows, so the SQL driver support list is empty: ${error.message}`)
|
|
159
|
+
}
|
|
160
|
+
if (!catalogRows) return csvRows
|
|
161
|
+
const isDriver = (row) => String(row.type || '').trim().toLowerCase() === 'sql_driver'
|
|
162
|
+
return [...catalogRows, ...csvRows.filter(isDriver)]
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
function describeFile (file) {
|
|
166
|
+
const origin = file.src.origin || {}
|
|
167
|
+
const ref = origin.tag || origin.branch || origin.refname
|
|
168
|
+
return `${file.src.component}:${file.src.module}:partial$${file.src.relative}${ref ? ` (${ref})` : ''}`
|
|
169
|
+
}
|
|
170
|
+
|
|
110
171
|
// Fetch CSV from GitHub or local file (local file for testing/override only)
|
|
111
|
-
async function fetchCSV (localPath,
|
|
172
|
+
async function fetchCSV (localPath, contentCatalog) {
|
|
112
173
|
// Priority 1: Use explicitly provided CSV path (for testing/override)
|
|
113
174
|
if (localPath && fs.existsSync(localPath)) {
|
|
114
175
|
if (path.extname(localPath).toLowerCase() !== '.csv') {
|
|
@@ -118,29 +179,50 @@ module.exports.register = function ({ config }) {
|
|
|
118
179
|
return fs.readFileSync(localPath, 'utf8')
|
|
119
180
|
}
|
|
120
181
|
|
|
121
|
-
// Priority 2: Fetch from GitHub
|
|
122
|
-
|
|
123
|
-
|
|
182
|
+
// Priority 2: Fetch from GitHub at the connect ref of this build
|
|
183
|
+
const target = resolveCsvRef(contentCatalog)
|
|
184
|
+
logger.info(`Fetching ${csvPath} from ${target.owner}/${target.repo} at ${target.ref} (${target.source})`)
|
|
185
|
+
return fetchCsvFromGitHub(target)
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// The ref to read info.csv from, in order:
|
|
189
|
+
// 1. the tag modify-connect-tag-playbook resolved for the connect content source
|
|
190
|
+
// 2. the ref of the connect content source in the content catalog
|
|
191
|
+
// 3. latest-connect-version in antora.yml in the working directory
|
|
192
|
+
// 4. main, with a warning, because the catalog can then disagree with the
|
|
193
|
+
// reference content
|
|
194
|
+
function resolveCsvRef (contentCatalog) {
|
|
195
|
+
const base = { owner: githubOwner, repo: githubRepo }
|
|
196
|
+
const shared = catalogUtil.getResolvedConnectRef()
|
|
197
|
+
if (shared) return { ...base, ref: shared, source: 'the tag modify-connect-tag-playbook resolved' }
|
|
198
|
+
const origin = catalogUtil.connectOriginRef(contentCatalog)
|
|
199
|
+
if (origin && origin.ref) return { ...base, ...origin, source: 'the connect content source' }
|
|
200
|
+
const connectVersion = getAntoraValue('asciidoc.attributes.latest-connect-version')
|
|
201
|
+
const normalizedVersion = connectVersion ? String(connectVersion).trim().replace(/^v/, '') : ''
|
|
202
|
+
if (normalizedVersion) return { ...base, ref: `v${normalizedVersion}`, source: 'latest-connect-version in antora.yml' }
|
|
203
|
+
logger.warn(
|
|
204
|
+
'No connect content source or latest-connect-version found, so info.csv is read from connect main. ' +
|
|
205
|
+
'Catalog badges can then disagree with the reference content. Add the connect content source and the ' +
|
|
206
|
+
'modify-connect-tag-playbook extension to the playbook.'
|
|
207
|
+
)
|
|
208
|
+
return { ...base, ref: 'main', source: 'fallback' }
|
|
124
209
|
}
|
|
125
210
|
|
|
126
211
|
// Fetch CSV data from GitHub
|
|
127
|
-
async function fetchCsvFromGitHub (
|
|
212
|
+
async function fetchCsvFromGitHub ({ owner, repo, ref }) {
|
|
128
213
|
const octokit = await loadOctokit()
|
|
129
|
-
// Normalize version: trim whitespace and remove leading 'v' if present
|
|
130
|
-
const normalizedVersion = connectVersion ? connectVersion.trim().replace(/^v/, '') : ''
|
|
131
|
-
// Use version tag if valid, otherwise fallback to main branch
|
|
132
|
-
const ref = normalizedVersion ? `v${normalizedVersion}` : 'main'
|
|
133
|
-
|
|
134
214
|
try {
|
|
135
215
|
const { data: fileContent } = await octokit.rest.repos.getContent({
|
|
136
|
-
owner
|
|
137
|
-
repo
|
|
216
|
+
owner,
|
|
217
|
+
repo,
|
|
138
218
|
path: csvPath,
|
|
139
|
-
ref
|
|
219
|
+
ref
|
|
140
220
|
})
|
|
141
221
|
return Buffer.from(fileContent.content, 'base64').toString('utf8')
|
|
142
222
|
} catch (error) {
|
|
143
|
-
|
|
223
|
+
// The caller decides how loud this is: fatal to the catalog without
|
|
224
|
+
// catalog.json, a warning with it.
|
|
225
|
+
error.message = `${owner}/${repo} ${csvPath} at ${ref}: ${error.message}`
|
|
144
226
|
throw error
|
|
145
227
|
}
|
|
146
228
|
}
|
|
@@ -206,8 +288,9 @@ module.exports.register = function ({ config }) {
|
|
|
206
288
|
|
|
207
289
|
return parsedData.data.map(row => {
|
|
208
290
|
// Create a new object with trimmed keys and values
|
|
291
|
+
// Rows from catalog.json also carry arrays (categories, commercial_names)
|
|
209
292
|
const trimmedRow = Object.fromEntries(
|
|
210
|
-
Object.entries(row).map(([key, value]) => [key.trim(), (value || '').trim()])
|
|
293
|
+
Object.entries(row).map(([key, value]) => [key.trim(), typeof value === 'string' || value == null ? (value || '').trim() : value])
|
|
211
294
|
)
|
|
212
295
|
|
|
213
296
|
// Map fields from the trimmed row to the desired output
|
|
@@ -216,8 +299,13 @@ module.exports.register = function ({ config }) {
|
|
|
216
299
|
const commercialName = trimmedRow.commercial_name
|
|
217
300
|
const availableConnectVersion = trimmedRow.version
|
|
218
301
|
const deprecated = (trimmedRow.deprecated || '').toLowerCase() === 'y' ? 'y' : 'n'
|
|
302
|
+
// is_cloud_supported is the standard Cloud pipeline flag and cloud_ai the
|
|
303
|
+
// GPU pipeline flag. A component in either one is available in Cloud
|
|
304
|
+
// (catalogUtil.isCloudAvailable); gpu_only and no_gpu say which.
|
|
219
305
|
const isCloudSupported = (trimmedRow.cloud || '').toLowerCase() === 'y' ? 'y' : 'n'
|
|
220
306
|
const cloudAi = (trimmedRow.cloud_with_gpu || '').toLowerCase() === 'y' ? 'y' : 'n'
|
|
307
|
+
const cloudAvailable = isCloudSupported === 'y' || cloudAi === 'y'
|
|
308
|
+
const { gpu_only: gpuOnly, no_gpu: noGpu } = catalogUtil.gpuFlags(isCloudSupported, cloudAi)
|
|
221
309
|
|
|
222
310
|
// Handle enterprise to certified conversion and set enterprise license flag
|
|
223
311
|
const originalSupport = (trimmedRow.support || '').toLowerCase()
|
|
@@ -239,13 +327,13 @@ module.exports.register = function ({ config }) {
|
|
|
239
327
|
// - Only run in Redpanda Cloud (not in self-managed rpk connect)
|
|
240
328
|
// - Have docs in cloud-docs repo but not in rp-connect-docs pages
|
|
241
329
|
// - Are marked with cloud: y in CSV but don't ship with OSS binary
|
|
242
|
-
const isCloudOnly =
|
|
330
|
+
const isCloudOnly = cloudAvailable && !redpandaConnectUrl && redpandaCloudUrl
|
|
243
331
|
|
|
244
332
|
// Only warn about missing self-managed docs if it's NOT cloud-only
|
|
245
333
|
if (!redpandaConnectUrl && !isCloudOnly) {
|
|
246
334
|
logger.warn(`Self-Managed docs missing for: ${connector} of type: ${type}`)
|
|
247
335
|
}
|
|
248
|
-
if (
|
|
336
|
+
if (cloudAvailable && !redpandaCloudUrl && redpandaConnectUrl) {
|
|
249
337
|
logger.warn(`Cloud docs missing for: ${connector} of type: ${type}`)
|
|
250
338
|
}
|
|
251
339
|
}
|
|
@@ -259,7 +347,14 @@ module.exports.register = function ({ config }) {
|
|
|
259
347
|
deprecated,
|
|
260
348
|
is_cloud_supported: isCloudSupported,
|
|
261
349
|
cloud_ai: cloudAi,
|
|
350
|
+
gpu_only: gpuOnly,
|
|
351
|
+
no_gpu: noGpu,
|
|
262
352
|
is_licensed: isLicensed,
|
|
353
|
+
// From catalog.json only; empty or absent for info.csv rows
|
|
354
|
+
status: trimmedRow.status || (deprecated === 'y' ? 'deprecated' : ''),
|
|
355
|
+
categories: Array.isArray(trimmedRow.categories) ? trimmedRow.categories : undefined,
|
|
356
|
+
commercial_names: Array.isArray(trimmedRow.commercial_names) ? trimmedRow.commercial_names : undefined,
|
|
357
|
+
cgo_only: trimmedRow.cgo_only === 'y' ? 'y' : 'n',
|
|
263
358
|
redpandaConnectUrl,
|
|
264
359
|
redpandaCloudUrl
|
|
265
360
|
}
|
|
@@ -274,20 +369,23 @@ module.exports.register = function ({ config }) {
|
|
|
274
369
|
const csvCommercialNames = new Map()
|
|
275
370
|
|
|
276
371
|
for (const row of parsedData.data) {
|
|
277
|
-
const { connector, commercial_name: commercialName } = row
|
|
278
|
-
if (!connector
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
const
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
372
|
+
const { connector, commercial_name: commercialName, commercial_names: commercialNames } = row
|
|
373
|
+
if (!connector) continue
|
|
374
|
+
// catalog.json rows list every commercial name; info.csv rows have one
|
|
375
|
+
const names = Array.isArray(commercialNames) && commercialNames.length ? commercialNames : [commercialName]
|
|
376
|
+
for (const name of names) {
|
|
377
|
+
// Skip N/A and empty values
|
|
378
|
+
const trimmedName = String(name || '').trim()
|
|
379
|
+
if (trimmedName.toLowerCase() === 'n/a' || trimmedName === '') continue
|
|
380
|
+
|
|
381
|
+
if (!csvCommercialNames.has(connector)) {
|
|
382
|
+
csvCommercialNames.set(connector, new Set())
|
|
383
|
+
}
|
|
287
384
|
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
385
|
+
// Add the commercial name if it's different from the connector name
|
|
386
|
+
if (trimmedName.toLowerCase() !== connector.toLowerCase()) {
|
|
387
|
+
csvCommercialNames.get(connector).add(trimmedName)
|
|
388
|
+
}
|
|
291
389
|
}
|
|
292
390
|
}
|
|
293
391
|
|
|
@@ -370,7 +468,13 @@ module.exports.register = function ({ config }) {
|
|
|
370
468
|
* - page-cloud-available + page-cloud-available-url: on Self-Managed pages with a Cloud variant
|
|
371
469
|
* - page-self-managed-available + page-self-managed-available-url: on Cloud pages with a
|
|
372
470
|
* Self-Managed variant
|
|
373
|
-
* - page-self-managed-only: on Self-Managed pages
|
|
471
|
+
* - page-self-managed-only: on Self-Managed pages whose type is not available in Cloud
|
|
472
|
+
* - page-cloud-gpu-only: on both variants when the type runs only in Cloud GPU pipelines
|
|
473
|
+
* - page-cloud-no-gpu: on both variants when the type runs in Cloud, but not in GPU pipelines
|
|
474
|
+
*
|
|
475
|
+
* Availability is decided per row (name and type), not per connector: the http_server
|
|
476
|
+
* input runs in Cloud and the http_server output does not. Only the Type dropdown is
|
|
477
|
+
* built from all of the connector's rows.
|
|
374
478
|
*/
|
|
375
479
|
function setStickyBarPageAttributes (contentCatalog, rows, logger) {
|
|
376
480
|
const pagesByUrl = new Map()
|
|
@@ -388,9 +492,13 @@ module.exports.register = function ({ config }) {
|
|
|
388
492
|
}
|
|
389
493
|
|
|
390
494
|
let decoratedCount = 0
|
|
495
|
+
const setGpuFlags = (attrs, row) => {
|
|
496
|
+
if (row.gpu_only === 'y') attrs['page-cloud-gpu-only'] = 'true'
|
|
497
|
+
if (row.no_gpu === 'y') attrs['page-cloud-no-gpu'] = 'true'
|
|
498
|
+
}
|
|
391
499
|
for (const connectorRows of rowsByConnector.values()) {
|
|
392
|
-
const isCloudSupported = connectorRows.some((row) => row.is_cloud_supported === 'y')
|
|
393
500
|
for (const row of connectorRows) {
|
|
501
|
+
const isCloudAvailable = catalogUtil.isCloudAvailable(row)
|
|
394
502
|
// Self-Managed (Connect) variant of this connector page
|
|
395
503
|
const connectPage = row.redpandaConnectUrl && pagesByUrl.get(row.redpandaConnectUrl)
|
|
396
504
|
if (connectPage) {
|
|
@@ -398,10 +506,11 @@ module.exports.register = function ({ config }) {
|
|
|
398
506
|
if (connectorRows.length > 1) {
|
|
399
507
|
attrs['page-context-switcher'] = buildContextSwitcher(connectorRows, row, 'connect')
|
|
400
508
|
}
|
|
401
|
-
if (
|
|
509
|
+
if (isCloudAvailable && row.redpandaCloudUrl) {
|
|
402
510
|
attrs['page-cloud-available'] = 'true'
|
|
403
511
|
attrs['page-cloud-available-url'] = row.redpandaCloudUrl
|
|
404
|
-
|
|
512
|
+
setGpuFlags(attrs, row)
|
|
513
|
+
} else if (!isCloudAvailable) {
|
|
405
514
|
attrs['page-self-managed-only'] = 'true'
|
|
406
515
|
}
|
|
407
516
|
decoratedCount++
|
|
@@ -417,6 +526,7 @@ module.exports.register = function ({ config }) {
|
|
|
417
526
|
attrs['page-self-managed-available'] = 'true'
|
|
418
527
|
attrs['page-self-managed-available-url'] = row.redpandaConnectUrl
|
|
419
528
|
}
|
|
529
|
+
setGpuFlags(attrs, row)
|
|
420
530
|
decoratedCount++
|
|
421
531
|
}
|
|
422
532
|
}
|