@redpanda-data/docs-extensions-and-macros 5.35.0 → 5.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/doc-tools.js +268 -6
- package/cli-utils/enterprise-sources.js +3 -3
- package/cli-utils/github-token.js +53 -13
- package/cli-utils/kapa-credentials.js +109 -0
- package/cli-utils/octokit-client.js +2 -2
- package/cli-utils/self-managed-docs-branch.js +2 -2
- package/docs-data/kapa-source-groups.json +119 -0
- package/docs-data/kapa-source-groups.schema.json +73 -0
- package/extensions/kapa-source-groups.js +180 -0
- package/extensions/version-fetcher/set-latest-version.js +6 -3
- package/package.json +7 -3
- package/tools/generate-cli-docs.js +19 -4
- package/tools/get-redpanda-version.js +2 -2
- package/tools/kapa-source-groups/fetch-with-deadline.js +33 -0
- package/tools/kapa-source-groups/generate-kapa-source-groups.js +319 -0
- package/tools/kapa-source-groups/kapa-source-versions.js +83 -0
- package/tools/kapa-source-groups/published-segments.js +98 -0
- package/tools/property-extractor/rp-util-fetch.js +2 -2
- package/tools/rpk-docs/rpk-docs-handler.js +2 -2
package/bin/doc-tools.js
CHANGED
|
@@ -122,9 +122,9 @@ programCli
|
|
|
122
122
|
* - A GitHub token with access to redpanda-data/streaming-enterprise, which is
|
|
123
123
|
* private. Redpanda releases are published there now, and the old public repo
|
|
124
124
|
* is frozen, so this command exits rather than report a stale version when it
|
|
125
|
-
* has no token. Resolved from
|
|
126
|
-
*
|
|
127
|
-
*
|
|
125
|
+
* has no token. Resolved from REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN,
|
|
126
|
+
* GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, GH_TOKEN, or GIT_CREDENTIALS, in that
|
|
127
|
+
* priority order (an API call, so the git credential is the last resort)
|
|
128
128
|
*/
|
|
129
129
|
programCli
|
|
130
130
|
.command('get-redpanda-version')
|
|
@@ -1908,7 +1908,7 @@ automation
|
|
|
1908
1908
|
*
|
|
1909
1909
|
* @requirements
|
|
1910
1910
|
* - GitHub token with access to redpanda-data/cloudv2-infra repository
|
|
1911
|
-
* - Token is resolved from
|
|
1911
|
+
* - Token is resolved from REDPANDA_GITHUB_TOKEN, ACTIONS_BOT_TOKEN, GITHUB_TOKEN, VBOT_GITHUB_API_TOKEN, GH_TOKEN, or GIT_CREDENTIALS, in that priority order (an API call, so the git credential is the last resort)
|
|
1912
1912
|
* - Internet connection to access GitHub API
|
|
1913
1913
|
*/
|
|
1914
1914
|
automation
|
|
@@ -1925,7 +1925,7 @@ automation
|
|
|
1925
1925
|
.option('--dry-run', 'Print output to stdout instead of writing file')
|
|
1926
1926
|
.action(async (options, command) => {
|
|
1927
1927
|
const { generateCloudRegions } = require('../tools/cloud-regions/generate-cloud-regions.js')
|
|
1928
|
-
const {
|
|
1928
|
+
const { getGitHubApiToken } = require('../cli-utils/github-token')
|
|
1929
1929
|
|
|
1930
1930
|
try {
|
|
1931
1931
|
// The default output path holds the unfiltered table, so a filtered run
|
|
@@ -1947,7 +1947,7 @@ automation
|
|
|
1947
1947
|
const absOutput = options.dryRun
|
|
1948
1948
|
? undefined
|
|
1949
1949
|
: resolveInsideRepo(repoRoot, options.output, '--output')
|
|
1950
|
-
const token =
|
|
1950
|
+
const token = getGitHubApiToken()
|
|
1951
1951
|
if (!token) {
|
|
1952
1952
|
throw new Error('GitHub token is required to fetch from private cloudv2-infra repo.')
|
|
1953
1953
|
}
|
|
@@ -2312,6 +2312,107 @@ automation
|
|
|
2312
2312
|
}
|
|
2313
2313
|
})
|
|
2314
2314
|
|
|
2315
|
+
/**
|
|
2316
|
+
* generate kapa-source-groups
|
|
2317
|
+
*
|
|
2318
|
+
* @description
|
|
2319
|
+
* Reads the source-group tree and the per-source group assignments from Kapa's ingestion API
|
|
2320
|
+
* and writes a mapping from each docs URL version segment to the Kapa source group that scopes
|
|
2321
|
+
* retrieval to that version. Consumers pass the group id as `sourceGroupIdsInclude` (Agent SDK),
|
|
2322
|
+
* `sourceGroupIDsInclude` (Chat SDK, note the capital ID) or `_meta.source_group_ids_include`
|
|
2323
|
+
* (hosted MCP server).
|
|
2324
|
+
*
|
|
2325
|
+
* The mapping is keyed on the URL segment, not Antora's page.version, because the two disagree
|
|
2326
|
+
* for the latest release: latest_version_segment: 'current' publishes 26.2 at
|
|
2327
|
+
* /streaming/current/ while page.version reads 26.2.
|
|
2328
|
+
*
|
|
2329
|
+
* This command only ever reads from Kapa. Kapa publishes no write API ("we do not provide an API
|
|
2330
|
+
* endpoint for uploading or managing sources"), so the groups and their source assignments are
|
|
2331
|
+
* created by hand in the Kapa dashboard under Sources > Manage groups. Run this afterwards to
|
|
2332
|
+
* record what is there.
|
|
2333
|
+
*
|
|
2334
|
+
* Refuses rather than writing a misleading mapping when: no groups exist, no group has children,
|
|
2335
|
+
* any version group has no sources assigned, or --default-segment is not one of the groups. An
|
|
2336
|
+
* empty version group is worse than no group, because scoping a query to it returns only the
|
|
2337
|
+
* global sources and the reader silently gets no version-specific content.
|
|
2338
|
+
*
|
|
2339
|
+
* @why
|
|
2340
|
+
* Kapa indexes one separately crawled source per published docs version. Without retrieval
|
|
2341
|
+
* scoping a reader gets answers drawn from any of them: a measured call for "What is a Redpanda
|
|
2342
|
+
* topic partition?" returned 12 sections, six of which were the same get-started/architecture
|
|
2343
|
+
* page at six different versions. Source groups fix that at retrieval time, but the group UUIDs
|
|
2344
|
+
* live in Kapa while the version list lives in Antora, so something has to join the two. Because
|
|
2345
|
+
* a new docs version appears with no file change in docs-site (the playbook globs branches: v/*),
|
|
2346
|
+
* a hand-maintained mapping goes stale silently; generating it and diff-checking the result in CI
|
|
2347
|
+
* is what makes drift loud.
|
|
2348
|
+
*
|
|
2349
|
+
* @example
|
|
2350
|
+
* # Write the mapping to its default location
|
|
2351
|
+
* export KAPA_API_KEY=... KAPA_PROJECT_ID=...
|
|
2352
|
+
* npx doc-tools generate kapa-source-groups
|
|
2353
|
+
*
|
|
2354
|
+
* # Preview without writing
|
|
2355
|
+
* npx doc-tools generate kapa-source-groups --dry-run
|
|
2356
|
+
*
|
|
2357
|
+
* # Scope unversioned pages to something other than "current"
|
|
2358
|
+
* npx doc-tools generate kapa-source-groups --default-segment 26.1
|
|
2359
|
+
*
|
|
2360
|
+
* # Disambiguate when more than one group has children
|
|
2361
|
+
* npx doc-tools generate kapa-source-groups --parent-group Streaming
|
|
2362
|
+
*
|
|
2363
|
+
* @requirements
|
|
2364
|
+
* - Kapa API key and project ID, set via KAPA_API_KEY and KAPA_PROJECT_ID
|
|
2365
|
+
* - Create the key in the Kapa platform under Configuration > API Keys; the project ID is the
|
|
2366
|
+
* UUID in the dashboard URL (https://app.kapa.ai/<project-id>)
|
|
2367
|
+
* - The parent group and its version sub groups must already exist in Kapa, with each version
|
|
2368
|
+
* group holding at least one source
|
|
2369
|
+
* - Internet connection to reach https://api.kapa.ai
|
|
2370
|
+
*/
|
|
2371
|
+
automation
|
|
2372
|
+
.command('kapa-source-groups')
|
|
2373
|
+
.description('Generate the docs version segment to Kapa source group mapping from Kapa\'s ingestion API')
|
|
2374
|
+
.option('--output <file>', 'Output file (relative to repo root, must stay inside the repository)', 'docs-data/kapa-source-groups.json')
|
|
2375
|
+
.option('--default-segment <segment>', 'Segment used for pages with no version of their own', 'current')
|
|
2376
|
+
.option('--parent-group <name>', 'Restrict to one parent group by name (default: the only group that has children)')
|
|
2377
|
+
.option('--dry-run', 'Print output to stdout instead of writing file')
|
|
2378
|
+
.action(async (options) => {
|
|
2379
|
+
const { generateKapaSourceGroups } = require('../tools/kapa-source-groups/generate-kapa-source-groups.js')
|
|
2380
|
+
const { requireKapaCredentials } = require('../cli-utils/kapa-credentials')
|
|
2381
|
+
|
|
2382
|
+
try {
|
|
2383
|
+
const repoRoot = findRepoRoot()
|
|
2384
|
+
// resolveInsideRepo for the same reason every other generator uses it: this
|
|
2385
|
+
// command is reachable from the MCP server, where a write option that escapes
|
|
2386
|
+
// the repo is a write-anywhere primitive.
|
|
2387
|
+
const absOutput = options.dryRun ? undefined : resolveInsideRepo(repoRoot, options.output, '--output')
|
|
2388
|
+
|
|
2389
|
+
// Fail loud on a missing credential rather than emitting an empty mapping:
|
|
2390
|
+
// unauthenticated would look identical to "Kapa has no source groups", and the
|
|
2391
|
+
// drift check would then report false drift forever.
|
|
2392
|
+
const { apiKey, projectId } = requireKapaCredentials()
|
|
2393
|
+
|
|
2394
|
+
const out = await generateKapaSourceGroups({
|
|
2395
|
+
apiKey,
|
|
2396
|
+
projectId,
|
|
2397
|
+
defaultSegment: options.defaultSegment,
|
|
2398
|
+
parentGroupName: options.parentGroup,
|
|
2399
|
+
})
|
|
2400
|
+
|
|
2401
|
+
if (options.dryRun) {
|
|
2402
|
+
process.stdout.write(out)
|
|
2403
|
+
return
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
fs.mkdirSync(path.dirname(absOutput), { recursive: true })
|
|
2407
|
+
fs.writeFileSync(absOutput, out, 'utf8')
|
|
2408
|
+
const segments = Object.keys(JSON.parse(out).segments)
|
|
2409
|
+
console.log(`Done: Wrote ${absOutput} (${segments.length} version segments: ${segments.join(', ')})`)
|
|
2410
|
+
} catch (err) {
|
|
2411
|
+
console.error(`Error: Failed to generate Kapa source groups: ${err.message}`)
|
|
2412
|
+
process.exit(1)
|
|
2413
|
+
}
|
|
2414
|
+
})
|
|
2415
|
+
|
|
2315
2416
|
const validation = new Command('validate').description('Validate docs data against internal sources of truth')
|
|
2316
2417
|
|
|
2317
2418
|
/**
|
|
@@ -2734,6 +2835,167 @@ overridesGroup
|
|
|
2734
2835
|
})
|
|
2735
2836
|
|
|
2736
2837
|
programCli.addCommand(automation)
|
|
2838
|
+
/**
|
|
2839
|
+
* validate kapa-source-groups
|
|
2840
|
+
*
|
|
2841
|
+
* @description
|
|
2842
|
+
* Checks that every streaming docs version currently published has a Kapa source, so Ask AI
|
|
2843
|
+
* can scope answers to it. Exit status is meaningful: 0 every version is covered, 1 a version
|
|
2844
|
+
* is missing, 2 the command itself failed (Kapa or the sitemap unreachable, bad credentials).
|
|
2845
|
+
*
|
|
2846
|
+
* A version counts as covered when Kapa has a source named `Documentation (X)` for it, that
|
|
2847
|
+
* source is assigned to a source group, AND the committed mapping (docs-data/kapa-source-groups.json)
|
|
2848
|
+
* has a segment for it whose group_id is a group that source actually sits in. Four gaps are
|
|
2849
|
+
* reported, each with its own line:
|
|
2850
|
+
*
|
|
2851
|
+
* - missing: the version is published but Kapa has no `Documentation (X)` source.
|
|
2852
|
+
* - unassigned: the source exists but is in no group, so it is global in Kapa and returned for
|
|
2853
|
+
* EVERY query on every version.
|
|
2854
|
+
* - unmapped: Kapa has the grouped source but the committed mapping has no segment for it. Every
|
|
2855
|
+
* deployed surface reads the mapping, not Kapa, so readers on that version silently get the
|
|
2856
|
+
* default segment until the mapping is regenerated and released.
|
|
2857
|
+
* - mismatched: the mapping has the segment but its group_id is not a group the version's source
|
|
2858
|
+
* sits in, which is what a group deleted and recreated in the dashboard, or a source moved into
|
|
2859
|
+
* another version's group, looks like. Every Ask AI query on that version is then scoped to a
|
|
2860
|
+
* group that no longer holds its docs, until the mapping is regenerated and released.
|
|
2861
|
+
*
|
|
2862
|
+
* The prerelease segment (/streaming/beta/) is ignored: a beta publishes there for the whole
|
|
2863
|
+
* pre-GA cycle and is not expected to have a Kapa group of its own.
|
|
2864
|
+
*
|
|
2865
|
+
* @why
|
|
2866
|
+
* The latest release always publishes at /streaming/current/, and the `Documentation (current)`
|
|
2867
|
+
* crawl follows it, so latest is never the version that goes missing. The one that goes missing
|
|
2868
|
+
* is the version that just got archived: when 26.3 ships, 26.2 moves to /streaming/26.2/ and
|
|
2869
|
+
* needs a crawl of its own that nobody has made yet. Kapa has no write API, so that crawl is
|
|
2870
|
+
* created by hand in the dashboard, and nothing else notices when it is forgotten. Readers on the
|
|
2871
|
+
* archived version then silently fall back to the default segment.
|
|
2872
|
+
*
|
|
2873
|
+
* Run it on a schedule so the gap opens an issue rather than waiting for a bug report.
|
|
2874
|
+
*
|
|
2875
|
+
* @example
|
|
2876
|
+
* export KAPA_API_KEY=... KAPA_PROJECT_ID=...
|
|
2877
|
+
* npx doc-tools validate kapa-source-groups
|
|
2878
|
+
*
|
|
2879
|
+
* # Check a preview or staging site instead of production
|
|
2880
|
+
* npx doc-tools validate kapa-source-groups --site-url https://deploy-preview-123--redpanda-documentation.netlify.app
|
|
2881
|
+
*
|
|
2882
|
+
* @requirements
|
|
2883
|
+
* - Kapa API key and project ID, set via KAPA_API_KEY and KAPA_PROJECT_ID
|
|
2884
|
+
* - Internet connection to reach https://api.kapa.ai and the docs site's sitemap
|
|
2885
|
+
*/
|
|
2886
|
+
validation
|
|
2887
|
+
.command('kapa-source-groups')
|
|
2888
|
+
.description('Check every published streaming version has a grouped Kapa source and the committed mapping points at that group (exit 0 in sync, 1 gap, 2 error)')
|
|
2889
|
+
.option('--site-url <url>', 'Docs site whose sitemap says which versions are published', 'https://docs.redpanda.com')
|
|
2890
|
+
.option('--mapping <file>', 'Committed mapping every surface reads', 'docs-data/kapa-source-groups.json')
|
|
2891
|
+
.action(async (options) => {
|
|
2892
|
+
const { fetchKapaSourceVersions } = require('../tools/kapa-source-groups/kapa-source-versions.js')
|
|
2893
|
+
const { fetchPublishedSegments, isPrereleaseSegment } = require('../tools/kapa-source-groups/published-segments.js')
|
|
2894
|
+
const { requireKapaCredentials } = require('../cli-utils/kapa-credentials')
|
|
2895
|
+
|
|
2896
|
+
// Printed only where a gap has actually been established, and required by
|
|
2897
|
+
// kapa-source-groups-drift.sh before it files an issue. Exit 1 alone is not
|
|
2898
|
+
// trustworthy: Commander exits 1 on a usage error and an uncaught throw
|
|
2899
|
+
// exits 1 too, so without this a typo'd flag would file an issue quoting a
|
|
2900
|
+
// stack trace as the report.
|
|
2901
|
+
const SENTINEL = 'KAPA_DRIFT_CONFIRMED'
|
|
2902
|
+
|
|
2903
|
+
// Exit 2 is reserved for "could not find out", so the scheduled job can tell
|
|
2904
|
+
// "a version is missing" apart from "Kapa was down" and not file a false issue.
|
|
2905
|
+
let published, kapa
|
|
2906
|
+
try {
|
|
2907
|
+
const { apiKey, projectId } = requireKapaCredentials()
|
|
2908
|
+
;[published, kapa] = await Promise.all([
|
|
2909
|
+
fetchPublishedSegments({ siteUrl: options.siteUrl }),
|
|
2910
|
+
fetchKapaSourceVersions({ apiKey, projectId }),
|
|
2911
|
+
])
|
|
2912
|
+
} catch (err) {
|
|
2913
|
+
console.error(`Error: Could not check Kapa sources: ${err.message}`)
|
|
2914
|
+
process.exit(2)
|
|
2915
|
+
}
|
|
2916
|
+
|
|
2917
|
+
// The committed mapping is what every surface actually reads (the Antora
|
|
2918
|
+
// extension, docs-ui, the /api/ proxy, the MCP server). Kapa having a
|
|
2919
|
+
// grouped source is necessary but not sufficient: if nobody regenerated and
|
|
2920
|
+
// released the mapping, readers on that version still get the default
|
|
2921
|
+
// segment and nothing else would ever say so.
|
|
2922
|
+
let mappedSegments, mappedGroupIds
|
|
2923
|
+
try {
|
|
2924
|
+
const mappingPath = path.resolve(options.mapping)
|
|
2925
|
+
const mapping = JSON.parse(fs.readFileSync(mappingPath, 'utf8'))
|
|
2926
|
+
if (!mapping || typeof mapping.segments !== 'object' || mapping.segments === null) {
|
|
2927
|
+
throw new Error('no segments object')
|
|
2928
|
+
}
|
|
2929
|
+
mappedSegments = new Set(Object.keys(mapping.segments))
|
|
2930
|
+
mappedGroupIds = new Map(Object.entries(mapping.segments)
|
|
2931
|
+
.map(([seg, entry]) => [seg, entry && entry.group_id != null ? String(entry.group_id) : null]))
|
|
2932
|
+
} catch (err) {
|
|
2933
|
+
console.error(`Error: Could not read the committed mapping at ${options.mapping}: ${err.message}`)
|
|
2934
|
+
process.exit(2)
|
|
2935
|
+
}
|
|
2936
|
+
|
|
2937
|
+
// A beta publishes at /streaming/beta/ for the whole pre-GA cycle and has no
|
|
2938
|
+
// group of its own by design. Reporting it would file a false issue weekly.
|
|
2939
|
+
const prerelease = published.filter(isPrereleaseSegment)
|
|
2940
|
+
const versions = published.filter((v) => !isPrereleaseSegment(v))
|
|
2941
|
+
|
|
2942
|
+
const missing = versions.filter((v) => !kapa.covered.has(v) && !kapa.unassigned.has(v))
|
|
2943
|
+
const unassigned = versions.filter((v) => kapa.unassigned.has(v))
|
|
2944
|
+
const unmapped = versions.filter((v) => kapa.covered.has(v) && !mappedSegments.has(v))
|
|
2945
|
+
// The mapping is what gets sent with every query, so its group_id has to be a
|
|
2946
|
+
// group the version's source is in right now. A group deleted and recreated
|
|
2947
|
+
// keeps its name and gets a new id; a source moved to another version's
|
|
2948
|
+
// group leaves the old id pointing at a group without these docs. Kapa
|
|
2949
|
+
// reports neither, and the three checks above pass in both cases.
|
|
2950
|
+
const mismatched = versions.filter((v) => {
|
|
2951
|
+
if (!kapa.covered.has(v) || !mappedSegments.has(v)) return false
|
|
2952
|
+
const want = mappedGroupIds.get(v)
|
|
2953
|
+
const have = kapa.groups.get(v) || new Set()
|
|
2954
|
+
return !want || !have.has(want)
|
|
2955
|
+
})
|
|
2956
|
+
|
|
2957
|
+
if (prerelease.length) {
|
|
2958
|
+
console.log(` (ignoring prerelease segment${prerelease.length > 1 ? 's' : ''} ${prerelease.join(', ')}, ` +
|
|
2959
|
+
'which publishes for the pre-GA cycle and is not expected to have a Kapa group)')
|
|
2960
|
+
}
|
|
2961
|
+
|
|
2962
|
+
if (missing.length === 0 && unassigned.length === 0 && unmapped.length === 0 && mismatched.length === 0) {
|
|
2963
|
+
console.log(`✓ Every published streaming version (${versions.join(', ')}) has a Kapa source in a group, ` +
|
|
2964
|
+
`and ${options.mapping} points each one at that group.`)
|
|
2965
|
+
process.exit(0)
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
console.log('✗ Kapa and the committed mapping are out of sync for published streaming docs versions.\n')
|
|
2969
|
+
for (const v of missing) {
|
|
2970
|
+
console.log(` - ${v}: published at /streaming/${v}/ but Kapa has no "Documentation (${v})" source. ` +
|
|
2971
|
+
'Readers on this version fall back to the default segment.')
|
|
2972
|
+
}
|
|
2973
|
+
for (const v of unassigned) {
|
|
2974
|
+
console.log(` - ${v}: "Documentation (${v})" exists but is not in a source group, so it is global ` +
|
|
2975
|
+
'and returned for every query on every version.')
|
|
2976
|
+
}
|
|
2977
|
+
for (const v of unmapped) {
|
|
2978
|
+
console.log(` - ${v}: Kapa has a grouped "Documentation (${v})" source, but ${options.mapping} has no ` +
|
|
2979
|
+
`"${v}" segment. Readers on this version fall back to the default segment until the mapping is ` +
|
|
2980
|
+
'regenerated and released.')
|
|
2981
|
+
}
|
|
2982
|
+
for (const v of mismatched) {
|
|
2983
|
+
const want = mappedGroupIds.get(v)
|
|
2984
|
+
const have = [...(kapa.groups.get(v) || [])]
|
|
2985
|
+
console.log(` - ${v}: ${options.mapping} scopes this version to group ${want || '(no group_id)'}, but Kapa has ` +
|
|
2986
|
+
`"Documentation (${v})" in ${have.length === 1 ? 'group' : 'groups'} ${have.join(', ')}. Every Ask AI query on ` +
|
|
2987
|
+
'this version is sent with a group that no longer holds its docs until the mapping is regenerated and released.')
|
|
2988
|
+
}
|
|
2989
|
+
if (missing.length || unassigned.length) {
|
|
2990
|
+
console.log('\nIn the Kapa dashboard: Sources > Add source for the crawl, then assign it to the version ' +
|
|
2991
|
+
'group under Sources > Manage groups. Then regenerate the mapping:\n doc-tools generate kapa-source-groups')
|
|
2992
|
+
} else {
|
|
2993
|
+
console.log('\nRegenerate the mapping and release docs-extensions-and-macros:\n doc-tools generate kapa-source-groups')
|
|
2994
|
+
}
|
|
2995
|
+
console.log(`\n${SENTINEL}`)
|
|
2996
|
+
process.exit(1)
|
|
2997
|
+
})
|
|
2998
|
+
|
|
2737
2999
|
programCli.addCommand(validation)
|
|
2738
3000
|
programCli.addCommand(overridesGroup)
|
|
2739
3001
|
programCli.parse(process.argv)
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
|
|
13
13
|
const fs = require('fs')
|
|
14
14
|
const path = require('path')
|
|
15
|
-
const {
|
|
15
|
+
const { getGitHubApiToken } = require('./github-token')
|
|
16
16
|
|
|
17
17
|
const RAW = 'https://raw.githubusercontent.com'
|
|
18
18
|
|
|
@@ -62,7 +62,7 @@ async function discardBody (resp) {
|
|
|
62
62
|
* check runs on a weekly cron where nobody is watching live.
|
|
63
63
|
*
|
|
64
64
|
* @param {object} [deps]
|
|
65
|
-
* @param {string|null} [deps.token] - GitHub token (defaults to
|
|
65
|
+
* @param {string|null} [deps.token] - GitHub token (defaults to getGitHubApiToken())
|
|
66
66
|
* @param {Function} [deps.fetchImpl] - fetch implementation (defaults to global fetch)
|
|
67
67
|
* @param {Function} [deps.warn] - warning sink (defaults to console.warn)
|
|
68
68
|
* @param {Function} [deps.sleep] - delay implementation, for tests
|
|
@@ -72,7 +72,7 @@ async function discardBody (resp) {
|
|
|
72
72
|
* check that never ran. Failures without a sourceName throw instead.
|
|
73
73
|
*/
|
|
74
74
|
function createSourceFetcher (deps = {}) {
|
|
75
|
-
const ghToken = deps.token !== undefined ? deps.token :
|
|
75
|
+
const ghToken = deps.token !== undefined ? deps.token : getGitHubApiToken()
|
|
76
76
|
const fetchImpl = deps.fetchImpl || ((...args) => fetch(...args))
|
|
77
77
|
const warn = deps.warn || ((msg) => console.warn(msg))
|
|
78
78
|
const sleep = deps.sleep || ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
|
|
@@ -47,9 +47,25 @@ function getTokenFromGitCredentials(credentials = process.env.GIT_CREDENTIALS) {
|
|
|
47
47
|
return null;
|
|
48
48
|
}
|
|
49
49
|
|
|
50
|
+
const API_TOKEN_VARS = [
|
|
51
|
+
'REDPANDA_GITHUB_TOKEN',
|
|
52
|
+
'ACTIONS_BOT_TOKEN',
|
|
53
|
+
'GITHUB_TOKEN',
|
|
54
|
+
'VBOT_GITHUB_API_TOKEN',
|
|
55
|
+
'GH_TOKEN'
|
|
56
|
+
];
|
|
57
|
+
|
|
58
|
+
function firstApiTokenVar() {
|
|
59
|
+
for (const name of API_TOKEN_VARS) {
|
|
60
|
+
if (process.env[name]) return process.env[name];
|
|
61
|
+
}
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
|
|
50
65
|
/**
|
|
51
|
-
* Get GitHub token
|
|
52
|
-
*
|
|
66
|
+
* Get the GitHub token for git operations (clone, fetch, authenticated URLs).
|
|
67
|
+
*
|
|
68
|
+
* Checks, in priority order:
|
|
53
69
|
* 1. GIT_CREDENTIALS - Antora's credential store contents (github.com entry)
|
|
54
70
|
* 2. REDPANDA_GITHUB_TOKEN - Custom Redpanda token
|
|
55
71
|
* 3. ACTIONS_BOT_TOKEN - GitHub Actions bot token
|
|
@@ -57,21 +73,44 @@ function getTokenFromGitCredentials(credentials = process.env.GIT_CREDENTIALS) {
|
|
|
57
73
|
* 5. VBOT_GITHUB_API_TOKEN - Legacy bot token
|
|
58
74
|
* 6. GH_TOKEN - GitHub CLI default
|
|
59
75
|
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
76
|
+
* GIT_CREDENTIALS wins here on purpose: it is the credential Antora itself
|
|
77
|
+
* clones the content repos with, so anything else that clones in the same
|
|
78
|
+
* build should use the same identity. It is a git credential, not an API
|
|
79
|
+
* one; for REST calls use getGitHubApiToken().
|
|
64
80
|
*
|
|
65
81
|
* @returns {string|null} GitHub token or null if not found
|
|
66
82
|
*/
|
|
67
83
|
function getGitHubToken() {
|
|
68
|
-
return getTokenFromGitCredentials() ||
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
84
|
+
return getTokenFromGitCredentials() || firstApiTokenVar();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Get the GitHub token for REST API calls (Octokit, api.github.com,
|
|
89
|
+
* raw.githubusercontent.com).
|
|
90
|
+
*
|
|
91
|
+
* Same variables as getGitHubToken(), but GIT_CREDENTIALS comes LAST:
|
|
92
|
+
*
|
|
93
|
+
* 1. REDPANDA_GITHUB_TOKEN
|
|
94
|
+
* 2. ACTIONS_BOT_TOKEN
|
|
95
|
+
* 3. GITHUB_TOKEN
|
|
96
|
+
* 4. VBOT_GITHUB_API_TOKEN
|
|
97
|
+
* 5. GH_TOKEN
|
|
98
|
+
* 6. GIT_CREDENTIALS
|
|
99
|
+
*
|
|
100
|
+
* Why the different order: on the Netlify Antora sites GIT_CREDENTIALS holds
|
|
101
|
+
* a token scoped only to cloning the private docs content repos. Letting it
|
|
102
|
+
* outrank REDPANDA_GITHUB_TOKEN meant the release lookup against the private
|
|
103
|
+
* streaming-enterprise repo authenticated with a token that cannot see that
|
|
104
|
+
* repo, got a 404, and every latest-redpanda-* attribute went unset in the
|
|
105
|
+
* production build even though a token with access was sitting right there.
|
|
106
|
+
* An explicitly named API token always beats the git credential; the git
|
|
107
|
+
* credential remains a fallback so a build with only GIT_CREDENTIALS still
|
|
108
|
+
* authenticates.
|
|
109
|
+
*
|
|
110
|
+
* @returns {string|null} GitHub token or null if not found
|
|
111
|
+
*/
|
|
112
|
+
function getGitHubApiToken() {
|
|
113
|
+
return firstApiTokenVar() || getTokenFromGitCredentials();
|
|
75
114
|
}
|
|
76
115
|
|
|
77
116
|
/**
|
|
@@ -106,6 +145,7 @@ function hasGitHubToken() {
|
|
|
106
145
|
|
|
107
146
|
module.exports = {
|
|
108
147
|
getGitHubToken,
|
|
148
|
+
getGitHubApiToken,
|
|
109
149
|
getTokenFromGitCredentials,
|
|
110
150
|
getAuthenticatedGitHubUrl,
|
|
111
151
|
hasGitHubToken
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Kapa API Credentials Utility
|
|
3
|
+
*
|
|
4
|
+
* Provides a consistent way to retrieve Kapa API credentials from environment
|
|
5
|
+
* variables, so no generator reads process.env.KAPA_* directly. Mirrors
|
|
6
|
+
* cli-utils/github-token.js: resolve here, return null when absent, and let the
|
|
7
|
+
* calling command decide whether a missing credential is fatal.
|
|
8
|
+
*
|
|
9
|
+
* The variable names are not new. They are already the established pair in
|
|
10
|
+
* redpanda-data/docs-site (netlify/functions/kapa-session.mjs) and in
|
|
11
|
+
* docs-team-standards/scripts/docs_analytics/kapa_probe.py, so a third spelling
|
|
12
|
+
* would just be a fourth place to get it wrong.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
// Kapa's public API. Both the ingestion and query endpoints live here, and both
|
|
16
|
+
// authenticate with the same project API key -- there is no separate ingestion
|
|
17
|
+
// key and no scope model.
|
|
18
|
+
const KAPA_API_BASE = 'https://api.kapa.ai';
|
|
19
|
+
|
|
20
|
+
// Kapa's OpenAPI schema spells the header X-API-KEY; their prose pages also use
|
|
21
|
+
// X-API-Key in places. HTTP header names are case-insensitive so either works,
|
|
22
|
+
// but pin the canonical spelling from the schema.
|
|
23
|
+
const KAPA_API_KEY_HEADER = 'X-API-KEY';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Get the Kapa API key.
|
|
27
|
+
*
|
|
28
|
+
* @param {object} [env] - Environment to read (defaults to process.env)
|
|
29
|
+
* @returns {string|null} The API key, or null when unset/blank
|
|
30
|
+
*/
|
|
31
|
+
function getKapaApiKey (env = process.env) {
|
|
32
|
+
const key = (env.KAPA_API_KEY || '').trim()
|
|
33
|
+
return key || null
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Get the Kapa project ID.
|
|
38
|
+
*
|
|
39
|
+
* @param {object} [env] - Environment to read (defaults to process.env)
|
|
40
|
+
* @returns {string|null} The project ID, or null when unset/blank
|
|
41
|
+
*/
|
|
42
|
+
function getKapaProjectId (env = process.env) {
|
|
43
|
+
const id = (env.KAPA_PROJECT_ID || '').trim()
|
|
44
|
+
return id || null
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* True when both credentials are present.
|
|
49
|
+
*
|
|
50
|
+
* @param {object} [env] - Environment to read (defaults to process.env)
|
|
51
|
+
* @returns {boolean}
|
|
52
|
+
*/
|
|
53
|
+
function hasKapaCredentials (env = process.env) {
|
|
54
|
+
return Boolean(getKapaApiKey(env) && getKapaProjectId(env))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Resolve both credentials, or throw with a remediation hint naming exactly
|
|
59
|
+
* what is missing.
|
|
60
|
+
*
|
|
61
|
+
* Generators that write published content call this and let it throw: an
|
|
62
|
+
* unauthenticated run would otherwise produce an empty mapping that looks
|
|
63
|
+
* identical to "Kapa has no source groups", and the drift check would then
|
|
64
|
+
* report false drift forever. Fail loud, like cloud-regions does for its
|
|
65
|
+
* GitHub token.
|
|
66
|
+
*
|
|
67
|
+
* @param {object} [env] - Environment to read (defaults to process.env)
|
|
68
|
+
* @returns {{apiKey: string, projectId: string, apiBase: string, headerName: string}}
|
|
69
|
+
* @throws {Error} When either credential is missing
|
|
70
|
+
*/
|
|
71
|
+
function requireKapaCredentials (env = process.env) {
|
|
72
|
+
const apiKey = getKapaApiKey(env)
|
|
73
|
+
const projectId = getKapaProjectId(env)
|
|
74
|
+
|
|
75
|
+
const missing = []
|
|
76
|
+
if (!apiKey) missing.push('KAPA_API_KEY')
|
|
77
|
+
if (!projectId) missing.push('KAPA_PROJECT_ID')
|
|
78
|
+
|
|
79
|
+
if (missing.length) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`Kapa credentials are required: ${missing.join(' and ')} ${missing.length > 1 ? 'are' : 'is'} not set. ` +
|
|
82
|
+
'Create an API key in the Kapa platform under Configuration > API Keys, and find the project ID ' +
|
|
83
|
+
'in the dashboard URL (https://app.kapa.ai/<project-id>). ' +
|
|
84
|
+
'In CI these come from AWS Secrets Manager via OIDC, not from GitHub secrets directly.'
|
|
85
|
+
)
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
return { apiKey, projectId, apiBase: KAPA_API_BASE, headerName: KAPA_API_KEY_HEADER }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Build the auth headers for a Kapa API request.
|
|
93
|
+
*
|
|
94
|
+
* @param {string} apiKey
|
|
95
|
+
* @returns {object} Headers suitable for fetch()
|
|
96
|
+
*/
|
|
97
|
+
function kapaAuthHeaders (apiKey) {
|
|
98
|
+
return { [KAPA_API_KEY_HEADER]: apiKey, Accept: 'application/json' }
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
module.exports = {
|
|
102
|
+
KAPA_API_BASE,
|
|
103
|
+
KAPA_API_KEY_HEADER,
|
|
104
|
+
getKapaApiKey,
|
|
105
|
+
getKapaProjectId,
|
|
106
|
+
hasKapaCredentials,
|
|
107
|
+
requireKapaCredentials,
|
|
108
|
+
kapaAuthHeaders,
|
|
109
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
'use strict'
|
|
2
2
|
|
|
3
3
|
const { Octokit } = require('@octokit/rest')
|
|
4
|
-
const {
|
|
4
|
+
const { getGitHubApiToken } = require('./github-token')
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* Shared Octokit client instance for GitHub API access
|
|
@@ -14,7 +14,7 @@ const { getGitHubToken } = require('./github-token')
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
// Get authentication token from environment
|
|
17
|
-
const token =
|
|
17
|
+
const token = getGitHubApiToken()
|
|
18
18
|
|
|
19
19
|
// Configure Octokit options
|
|
20
20
|
const octokitOptions = {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
const yaml = require('js-yaml');
|
|
2
2
|
const { spawnSync } = require('child_process');
|
|
3
|
-
const {
|
|
3
|
+
const { getGitHubApiToken } = require('./github-token');
|
|
4
4
|
|
|
5
5
|
const ANTORA_URL = 'https://raw.githubusercontent.com/redpanda-data/docs/main/antora.yml'
|
|
6
6
|
|
|
@@ -23,7 +23,7 @@ const REJECTED_HINT = ' A GitHub token was sent but rejected, so it may be expir
|
|
|
23
23
|
*/
|
|
24
24
|
async function fetchRemoteAntoraVersion(deps = {}) {
|
|
25
25
|
const fetchImpl = deps.fetchImpl || fetch
|
|
26
|
-
const ghToken = 'token' in deps ? deps.token :
|
|
26
|
+
const ghToken = 'token' in deps ? deps.token : getGitHubApiToken()
|
|
27
27
|
|
|
28
28
|
let resp = await fetchImpl(ANTORA_URL, ghToken ? { headers: { Authorization: `Bearer ${ghToken}` } } : undefined)
|
|
29
29
|
const hint = ghToken ? REJECTED_HINT : TOKEN_HINT
|