@redpanda-data/docs-extensions-and-macros 5.0.2 → 5.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/version-fetcher/set-latest-version.js +131 -0
- package/package.json +1 -1
- package/tools/property-extractor/constant_resolver.py +169 -1
- package/tools/property-extractor/property_extractor.py +1 -1
- package/tools/property-extractor/transformers.py +54 -1
- package/tools/redpanda-connect/helpers/buildConfigYaml.js +14 -9
- package/tools/redpanda-connect/rpcn-connector-docs-handler.js +73 -41
|
@@ -135,12 +135,97 @@ module.exports.register = function ({ config }) {
|
|
|
135
135
|
}
|
|
136
136
|
});
|
|
137
137
|
|
|
138
|
+
// Find latest available Redpanda properties JSON in content catalog
|
|
139
|
+
// This ensures we use a version that actually exists, even if latest-redpanda-tag
|
|
140
|
+
// points to a newer release that doesn't have docs/attachments yet
|
|
141
|
+
// Component name is 'streaming' (or 'ROOT' for backwards compatibility)
|
|
142
|
+
const propertiesFiles = contentCatalog.findBy({
|
|
143
|
+
family: 'attachment'
|
|
144
|
+
}).filter(f => {
|
|
145
|
+
if (!f?.src?.stem || !f?.src?.component) return false;
|
|
146
|
+
const isStreamingComponent = f.src.component === 'streaming' || f.src.component === 'ROOT';
|
|
147
|
+
const isReferenceModule = f.src.module === 'reference';
|
|
148
|
+
const isPropertiesFile = f.src.stem.startsWith('redpanda-properties-');
|
|
149
|
+
return isStreamingComponent && isReferenceModule && isPropertiesFile;
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
let availablePropertiesTag = null;
|
|
153
|
+
if (propertiesFiles.length > 0) {
|
|
154
|
+
const versions = propertiesFiles
|
|
155
|
+
.map(f => f.src.stem.replace('redpanda-properties-', ''))
|
|
156
|
+
// Only accept stable versions (no prerelease suffix like -backup, -rc1, etc.)
|
|
157
|
+
.filter(v => {
|
|
158
|
+
const normalized = v.replace(/^v/, '');
|
|
159
|
+
return semver.valid(normalized) && semver.prerelease(normalized) === null;
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
if (versions.length > 0) {
|
|
163
|
+
availablePropertiesTag = versions.sort((a, b) =>
|
|
164
|
+
semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))
|
|
165
|
+
)[0];
|
|
166
|
+
|
|
167
|
+
// Set on all component versions
|
|
168
|
+
components.forEach(component => {
|
|
169
|
+
component.versions.forEach(({ asciidoc }) => {
|
|
170
|
+
asciidoc.attributes['available-properties-tag'] = availablePropertiesTag;
|
|
171
|
+
});
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
logger.debug(`Found ${propertiesFiles.length} properties JSON files, latest available: ${availablePropertiesTag}`);
|
|
175
|
+
}
|
|
176
|
+
} else {
|
|
177
|
+
logger.debug('No properties JSON files found in content catalog');
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// Find latest available Connect JSON in content catalog
|
|
181
|
+
// Component name is 'connect' (or 'redpanda-connect' for backwards compatibility)
|
|
182
|
+
const connectFiles = contentCatalog.findBy({
|
|
183
|
+
family: 'attachment'
|
|
184
|
+
}).filter(f => {
|
|
185
|
+
if (!f?.src?.stem || !f?.src?.component) return false;
|
|
186
|
+
const isConnectComponent = f.src.component === 'connect' || f.src.component === 'redpanda-connect';
|
|
187
|
+
const isComponentsModule = f.src.module === 'components';
|
|
188
|
+
const isConnectFile = f.src.stem.startsWith('connect-');
|
|
189
|
+
return isConnectComponent && isComponentsModule && isConnectFile;
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
let availableConnectVersion = null;
|
|
193
|
+
if (connectFiles.length > 0) {
|
|
194
|
+
const versions = connectFiles
|
|
195
|
+
.map(f => f.src.stem.replace('connect-', ''))
|
|
196
|
+
// Only accept stable versions (no prerelease suffix)
|
|
197
|
+
.filter(v => semver.valid(v) && semver.prerelease(v) === null);
|
|
198
|
+
|
|
199
|
+
if (versions.length > 0) {
|
|
200
|
+
availableConnectVersion = versions.sort((a, b) =>
|
|
201
|
+
semver.rcompare(a, b)
|
|
202
|
+
)[0];
|
|
203
|
+
|
|
204
|
+
// Set on all component versions
|
|
205
|
+
components.forEach(component => {
|
|
206
|
+
component.versions.forEach(({ asciidoc }) => {
|
|
207
|
+
asciidoc.attributes['available-connect-version'] = availableConnectVersion;
|
|
208
|
+
});
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
logger.debug(`Found ${connectFiles.length} connect JSON files, latest available: ${availableConnectVersion}`);
|
|
212
|
+
}
|
|
213
|
+
} else {
|
|
214
|
+
logger.debug('No connect JSON files found in content catalog');
|
|
215
|
+
}
|
|
216
|
+
|
|
138
217
|
logger.info('Updated Redpanda documentation versions successfully:');
|
|
139
218
|
logger.info(`- Redpanda: ${latestVersions.redpanda.latestRedpandaRelease.version}${latestVersions.redpanda.latestRcRelease ? ', beta: ' + latestVersions.redpanda.latestRcRelease.version : ''}`);
|
|
140
219
|
logger.info(`- Connect: ${latestVersions.connect}`);
|
|
141
220
|
logger.info(`- Console: ${latestVersions.console.latestStableRelease}${latestVersions.console.latestBetaRelease ? ', beta: ' + latestVersions.console.latestBetaRelease : ''}`);
|
|
142
221
|
logger.info(`- Operator: ${latestVersions.operator?.latestStableRelease || 'unknown'}${latestVersions.operator?.latestBetaRelease ? ', beta: ' + latestVersions.operator.latestBetaRelease : ''}`);
|
|
143
222
|
logger.info(`- Helm chart: ${latestVersions.helmChart?.latestStableRelease || 'unknown'}${latestVersions.helmChart?.latestBetaRelease ? ', beta: ' + latestVersions.helmChart.latestBetaRelease : ''}`);
|
|
223
|
+
if (availablePropertiesTag) {
|
|
224
|
+
logger.info(`- Available properties JSON: ${availablePropertiesTag}`);
|
|
225
|
+
}
|
|
226
|
+
if (availableConnectVersion) {
|
|
227
|
+
logger.info(`- Available Connect JSON: ${availableConnectVersion}`);
|
|
228
|
+
}
|
|
144
229
|
} catch (error) {
|
|
145
230
|
logger.error(`Error updating versions: ${error}`);
|
|
146
231
|
}
|
|
@@ -174,4 +259,50 @@ module.exports.register = function ({ config }) {
|
|
|
174
259
|
}
|
|
175
260
|
});
|
|
176
261
|
}
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
// Export helper functions for testing
|
|
265
|
+
module.exports.filterPropertiesFiles = function(files) {
|
|
266
|
+
return files.filter(f => {
|
|
267
|
+
if (!f?.src?.stem || !f?.src?.component) return false;
|
|
268
|
+
const isStreamingComponent = f.src.component === 'streaming' || f.src.component === 'ROOT';
|
|
269
|
+
const isReferenceModule = f.src.module === 'reference';
|
|
270
|
+
const isPropertiesFile = f.src.stem.startsWith('redpanda-properties-');
|
|
271
|
+
return isStreamingComponent && isReferenceModule && isPropertiesFile;
|
|
272
|
+
});
|
|
273
|
+
};
|
|
274
|
+
|
|
275
|
+
module.exports.filterConnectFiles = function(files) {
|
|
276
|
+
return files.filter(f => {
|
|
277
|
+
if (!f?.src?.stem || !f?.src?.component) return false;
|
|
278
|
+
const isConnectComponent = f.src.component === 'connect' || f.src.component === 'redpanda-connect';
|
|
279
|
+
const isComponentsModule = f.src.module === 'components';
|
|
280
|
+
const isConnectFile = f.src.stem.startsWith('connect-');
|
|
281
|
+
return isConnectComponent && isComponentsModule && isConnectFile;
|
|
282
|
+
});
|
|
283
|
+
};
|
|
284
|
+
|
|
285
|
+
module.exports.extractLatestPropertiesVersion = function(files, semver) {
|
|
286
|
+
const versions = files
|
|
287
|
+
.map(f => f.src.stem.replace('redpanda-properties-', ''))
|
|
288
|
+
.filter(v => {
|
|
289
|
+
const normalized = v.replace(/^v/, '');
|
|
290
|
+
return semver.valid(normalized) && semver.prerelease(normalized) === null;
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
if (versions.length === 0) return null;
|
|
294
|
+
|
|
295
|
+
return versions.sort((a, b) =>
|
|
296
|
+
semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, ''))
|
|
297
|
+
)[0];
|
|
298
|
+
};
|
|
299
|
+
|
|
300
|
+
module.exports.extractLatestConnectVersion = function(files, semver) {
|
|
301
|
+
const versions = files
|
|
302
|
+
.map(f => f.src.stem.replace('connect-', ''))
|
|
303
|
+
.filter(v => semver.valid(v) && semver.prerelease(v) === null);
|
|
304
|
+
|
|
305
|
+
if (versions.length === 0) return null;
|
|
306
|
+
|
|
307
|
+
return versions.sort((a, b) => semver.rcompare(a, b))[0];
|
|
177
308
|
};
|
package/package.json
CHANGED
|
@@ -22,7 +22,8 @@ class ConstantResolver:
|
|
|
22
22
|
Args:
|
|
23
23
|
source_path: Path to the Redpanda src/v directory
|
|
24
24
|
"""
|
|
25
|
-
|
|
25
|
+
# Ensure source_path is a Path object
|
|
26
|
+
self.source_path = Path(source_path) if not isinstance(source_path, Path) else source_path
|
|
26
27
|
self._constant_cache: Dict[str, str] = {}
|
|
27
28
|
|
|
28
29
|
def resolve_constant(self, constant_name: str) -> Optional[str]:
|
|
@@ -371,6 +372,173 @@ class ConstantResolver:
|
|
|
371
372
|
|
|
372
373
|
return result
|
|
373
374
|
|
|
375
|
+
def _parse_size_literal(self, value: str) -> Optional[int]:
|
|
376
|
+
"""
|
|
377
|
+
Parse C++ size literals like 256_MiB, 20_GiB to integers.
|
|
378
|
+
|
|
379
|
+
Args:
|
|
380
|
+
value: The size literal string (e.g., "256_MiB", "20_GiB", "1024", "1'024_KiB", "1048576ULL")
|
|
381
|
+
|
|
382
|
+
Returns:
|
|
383
|
+
The integer value in bytes, or None if parsing fails
|
|
384
|
+
"""
|
|
385
|
+
if not value:
|
|
386
|
+
return None
|
|
387
|
+
|
|
388
|
+
value = value.strip()
|
|
389
|
+
|
|
390
|
+
# Remove C++ digit separators (apostrophes) - e.g., 1'024 -> 1024
|
|
391
|
+
value = value.replace("'", "")
|
|
392
|
+
|
|
393
|
+
# Binary suffixes (powers of 1024)
|
|
394
|
+
binary_suffixes = {
|
|
395
|
+
'_GiB': 1024**3,
|
|
396
|
+
'_MiB': 1024**2,
|
|
397
|
+
'_KiB': 1024,
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
# Decimal suffixes (powers of 1000)
|
|
401
|
+
decimal_suffixes = {
|
|
402
|
+
'_GB': 1000**3,
|
|
403
|
+
'_MB': 1000**2,
|
|
404
|
+
'_KB': 1000,
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
all_suffixes = {**binary_suffixes, **decimal_suffixes}
|
|
408
|
+
|
|
409
|
+
for suffix, multiplier in all_suffixes.items():
|
|
410
|
+
if value.endswith(suffix):
|
|
411
|
+
try:
|
|
412
|
+
num = int(value[:-len(suffix)])
|
|
413
|
+
return num * multiplier
|
|
414
|
+
except ValueError:
|
|
415
|
+
return None
|
|
416
|
+
|
|
417
|
+
# Try parsing as plain integer
|
|
418
|
+
# First strip C++ integer suffixes (U, L, UL, LL, ULL, etc.)
|
|
419
|
+
normalized = re.sub(r'[uUlLzZ]+$', '', value)
|
|
420
|
+
try:
|
|
421
|
+
return int(normalized)
|
|
422
|
+
except ValueError:
|
|
423
|
+
return None
|
|
424
|
+
|
|
425
|
+
def resolve_numeric_constant(self, constant_name: str) -> Optional[dict]:
|
|
426
|
+
"""
|
|
427
|
+
Resolve a C++ numeric constant to its actual value.
|
|
428
|
+
|
|
429
|
+
Handles patterns like:
|
|
430
|
+
- constexpr size_t DEFAULT_TOPIC_MEMORY_PER_PARTITION = 256_MiB;
|
|
431
|
+
- inline constexpr uint64_t identifier = 12345;
|
|
432
|
+
- constexpr auto identifier = 20_GiB;
|
|
433
|
+
|
|
434
|
+
Args:
|
|
435
|
+
constant_name: The constant name (e.g., "DEFAULT_TOPIC_MEMORY_PER_PARTITION")
|
|
436
|
+
|
|
437
|
+
Returns:
|
|
438
|
+
Dict with 'raw' (int) and 'friendly' (str) values, or None if not found
|
|
439
|
+
"""
|
|
440
|
+
# Check cache first
|
|
441
|
+
cache_key = f"numeric:{constant_name}"
|
|
442
|
+
if cache_key in self._constant_cache:
|
|
443
|
+
return self._constant_cache[cache_key]
|
|
444
|
+
|
|
445
|
+
# Strip namespace qualifier if present
|
|
446
|
+
if '::' in constant_name:
|
|
447
|
+
namespace, identifier = constant_name.rsplit('::', 1)
|
|
448
|
+
else:
|
|
449
|
+
namespace = None
|
|
450
|
+
identifier = constant_name
|
|
451
|
+
|
|
452
|
+
# Search patterns for numeric constants
|
|
453
|
+
numeric_patterns = [
|
|
454
|
+
# constexpr size_t/uint64_t/int64_t identifier = VALUE;
|
|
455
|
+
rf'(?:inline\s+)?constexpr\s+(?:size_t|uint64_t|int64_t|int|unsigned(?:\s+int)?|long(?:\s+long)?)\s+{re.escape(identifier)}\s*=\s*([^;]+);',
|
|
456
|
+
# constexpr auto identifier = VALUE;
|
|
457
|
+
rf'(?:inline\s+)?constexpr\s+auto\s+{re.escape(identifier)}\s*=\s*([^;]+);',
|
|
458
|
+
# constexpr type identifier{VALUE};
|
|
459
|
+
rf'(?:inline\s+)?constexpr\s+(?:size_t|uint64_t|int64_t|int|unsigned(?:\s+int)?)\s+{re.escape(identifier)}\s*\{{\s*([^}}]+)\s*\}}',
|
|
460
|
+
# static constexpr variants
|
|
461
|
+
rf'static\s+(?:inline\s+)?constexpr\s+(?:size_t|uint64_t|int64_t|int|unsigned(?:\s+int)?)\s+{re.escape(identifier)}\s*=\s*([^;]+);',
|
|
462
|
+
]
|
|
463
|
+
|
|
464
|
+
# Determine search directories based on namespace
|
|
465
|
+
if namespace == 'config':
|
|
466
|
+
search_dirs = ['config']
|
|
467
|
+
elif namespace == 'model':
|
|
468
|
+
search_dirs = ['model']
|
|
469
|
+
elif namespace == 'kafka':
|
|
470
|
+
search_dirs = ['kafka']
|
|
471
|
+
else:
|
|
472
|
+
# Search common locations
|
|
473
|
+
search_dirs = ['config', 'model', 'kafka', 'cluster']
|
|
474
|
+
|
|
475
|
+
for dir_name in search_dirs:
|
|
476
|
+
search_path = self.source_path / dir_name
|
|
477
|
+
if not search_path.exists():
|
|
478
|
+
continue
|
|
479
|
+
|
|
480
|
+
# Search both .h and .cc files
|
|
481
|
+
for file_path in list(search_path.rglob('*.h')) + list(search_path.rglob('*.cc')):
|
|
482
|
+
try:
|
|
483
|
+
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
|
|
484
|
+
content = f.read()
|
|
485
|
+
|
|
486
|
+
for pattern in numeric_patterns:
|
|
487
|
+
match = re.search(pattern, content)
|
|
488
|
+
if match:
|
|
489
|
+
raw_value = match.group(1).strip()
|
|
490
|
+
logger.debug(f"Found numeric constant {constant_name} = {raw_value} in {file_path}")
|
|
491
|
+
|
|
492
|
+
# Try to parse the value
|
|
493
|
+
parsed = self._parse_size_literal(raw_value)
|
|
494
|
+
if parsed is not None:
|
|
495
|
+
result = self._format_numeric_result(parsed)
|
|
496
|
+
self._constant_cache[cache_key] = result
|
|
497
|
+
logger.info(f"Resolved numeric constant {constant_name} = {result}")
|
|
498
|
+
return result
|
|
499
|
+
else:
|
|
500
|
+
# Couldn't parse, return raw string
|
|
501
|
+
logger.debug(f"Could not parse numeric value: {raw_value}")
|
|
502
|
+
|
|
503
|
+
except Exception as e:
|
|
504
|
+
logger.debug(f"Error reading {file_path}: {e}")
|
|
505
|
+
continue
|
|
506
|
+
|
|
507
|
+
logger.debug(f"Could not resolve numeric constant: {constant_name}")
|
|
508
|
+
return None
|
|
509
|
+
|
|
510
|
+
def _format_numeric_result(self, value: int) -> dict:
|
|
511
|
+
"""
|
|
512
|
+
Format a numeric value with both raw and human-friendly representations.
|
|
513
|
+
|
|
514
|
+
Args:
|
|
515
|
+
value: The raw numeric value in bytes
|
|
516
|
+
|
|
517
|
+
Returns:
|
|
518
|
+
Dict with 'raw' (int) and 'friendly' (str like "256 MiB (268435456)")
|
|
519
|
+
"""
|
|
520
|
+
# Define units in descending order
|
|
521
|
+
units = [
|
|
522
|
+
('GiB', 1024**3),
|
|
523
|
+
('MiB', 1024**2),
|
|
524
|
+
('KiB', 1024),
|
|
525
|
+
]
|
|
526
|
+
|
|
527
|
+
for unit_name, divisor in units:
|
|
528
|
+
if value >= divisor and value % divisor == 0:
|
|
529
|
+
friendly_num = value // divisor
|
|
530
|
+
friendly_str = f"{friendly_num} {unit_name} ({value})"
|
|
531
|
+
return {
|
|
532
|
+
'raw': value,
|
|
533
|
+
'friendly': friendly_str
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
# No unit applies, just return the raw number
|
|
537
|
+
return {
|
|
538
|
+
'raw': value,
|
|
539
|
+
'friendly': str(value)
|
|
540
|
+
}
|
|
541
|
+
|
|
374
542
|
|
|
375
543
|
def resolve_property_default(default_value: str, resolver: ConstantResolver) -> str:
|
|
376
544
|
"""
|
|
@@ -723,7 +723,7 @@ def transform_files_with_properties(files_with_properties):
|
|
|
723
723
|
NumericBoundsTransformer(type_transformer),
|
|
724
724
|
DurationBoundsTransformer(type_transformer),
|
|
725
725
|
SimpleDefaultValuesTransformer(),
|
|
726
|
-
FriendlyDefaultTransformer(),
|
|
726
|
+
FriendlyDefaultTransformer(constant_resolver),
|
|
727
727
|
ExperimentalTransformer(),
|
|
728
728
|
AliasTransformer(),
|
|
729
729
|
]
|
|
@@ -1570,9 +1570,12 @@ class FriendlyDefaultTransformer:
|
|
|
1570
1570
|
NUMERIC_LIMITS_PATTERN = r"std::numeric_limits<[^>]+>::max\(\)"
|
|
1571
1571
|
LEGACY_DEFAULT_PATTERN = r"legacy_default<[^>]+>\(([^,]+)"
|
|
1572
1572
|
BRACED_LIST_PATTERN = r"^\{(.*)\}$"
|
|
1573
|
+
# Pattern for identifiers that look like C++ constants (ALL_CAPS with underscores)
|
|
1574
|
+
CONSTANT_IDENTIFIER_PATTERN = r'^[A-Z][A-Z0-9_]*$'
|
|
1573
1575
|
|
|
1574
|
-
def __init__(self):
|
|
1576
|
+
def __init__(self, constant_resolver=None):
|
|
1575
1577
|
self._resolver = None
|
|
1578
|
+
self._constant_resolver = constant_resolver
|
|
1576
1579
|
|
|
1577
1580
|
def accepts(self, info, file_pair):
|
|
1578
1581
|
return bool(info.get("params") and len(info["params"]) > 2)
|
|
@@ -1592,6 +1595,40 @@ class FriendlyDefaultTransformer:
|
|
|
1592
1595
|
return identifier
|
|
1593
1596
|
return identifier
|
|
1594
1597
|
|
|
1598
|
+
def _resolve_numeric_constant(self, identifier):
|
|
1599
|
+
"""
|
|
1600
|
+
Try to resolve a C++ numeric constant using the ConstantResolver.
|
|
1601
|
+
|
|
1602
|
+
Args:
|
|
1603
|
+
identifier: The constant name (e.g., "DEFAULT_TOPIC_MEMORY_PER_PARTITION")
|
|
1604
|
+
|
|
1605
|
+
Returns:
|
|
1606
|
+
The friendly formatted value (e.g., "256 MiB (268435456)"), or None if not resolved
|
|
1607
|
+
"""
|
|
1608
|
+
if not self._constant_resolver:
|
|
1609
|
+
return None
|
|
1610
|
+
|
|
1611
|
+
try:
|
|
1612
|
+
result = self._constant_resolver.resolve_numeric_constant(identifier)
|
|
1613
|
+
if result:
|
|
1614
|
+
return result.get('friendly')
|
|
1615
|
+
except Exception as e:
|
|
1616
|
+
logger.debug(f"Error resolving numeric constant {identifier}: {e}")
|
|
1617
|
+
|
|
1618
|
+
return None
|
|
1619
|
+
|
|
1620
|
+
def _looks_like_constant(self, identifier):
|
|
1621
|
+
"""
|
|
1622
|
+
Check if an identifier looks like a C++ constant (ALL_CAPS_WITH_UNDERSCORES).
|
|
1623
|
+
|
|
1624
|
+
Args:
|
|
1625
|
+
identifier: The identifier to check
|
|
1626
|
+
|
|
1627
|
+
Returns:
|
|
1628
|
+
True if it looks like a constant, False otherwise
|
|
1629
|
+
"""
|
|
1630
|
+
return bool(re.match(self.CONSTANT_IDENTIFIER_PATTERN, identifier))
|
|
1631
|
+
|
|
1595
1632
|
def _parse_initializer_list(self, text):
|
|
1596
1633
|
"""Handle braced lists like {"a", "b"}."""
|
|
1597
1634
|
inner = re.sub(self.BRACED_LIST_PATTERN, r"\1", text.strip()).strip()
|
|
@@ -1778,10 +1815,26 @@ class FriendlyDefaultTransformer:
|
|
|
1778
1815
|
property["default"] = COMPUTED_CONSTANTS[d]
|
|
1779
1816
|
return property
|
|
1780
1817
|
|
|
1818
|
+
# ------------------------------------------------------------------
|
|
1819
|
+
# ALL_CAPS numeric constants (DEFAULT_TOPIC_MEMORY_PER_PARTITION, etc.)
|
|
1820
|
+
# Try to resolve via ConstantResolver before falling back
|
|
1821
|
+
# ------------------------------------------------------------------
|
|
1822
|
+
if self._looks_like_constant(d):
|
|
1823
|
+
resolved = self._resolve_numeric_constant(d)
|
|
1824
|
+
if resolved:
|
|
1825
|
+
property["default"] = resolved
|
|
1826
|
+
return property
|
|
1827
|
+
# If not resolved, fall through to normalized string fallback
|
|
1828
|
+
|
|
1781
1829
|
# ------------------------------------------------------------------
|
|
1782
1830
|
# Constant-like or symbolic identifiers (model::..., net::..., etc.)
|
|
1783
1831
|
# ------------------------------------------------------------------
|
|
1784
1832
|
if "::" in d and not d.startswith("std::"):
|
|
1833
|
+
# Try numeric resolution first for namespace-qualified constants
|
|
1834
|
+
resolved = self._resolve_numeric_constant(d)
|
|
1835
|
+
if resolved:
|
|
1836
|
+
property["default"] = resolved
|
|
1837
|
+
return property
|
|
1785
1838
|
# Try to resolve to string enum if possible
|
|
1786
1839
|
resolved = self._resolve_identifier(d)
|
|
1787
1840
|
property["default"] = resolved or d
|
|
@@ -31,19 +31,24 @@ module.exports = function buildConfigYaml(type, connectorName, children, include
|
|
|
31
31
|
lines.push(` label: ""`);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
// Count how many fields will be rendered to determine if we need {}
|
|
35
|
+
const fieldsToRender = children.filter(field => {
|
|
36
|
+
if (field.is_deprecated) return false;
|
|
37
|
+
if (!includeAdvanced && field.is_advanced) return false;
|
|
38
|
+
return true;
|
|
39
|
+
});
|
|
40
|
+
|
|
34
41
|
// Two‐space indent for connectorName heading
|
|
35
|
-
|
|
42
|
+
// If no fields will be rendered, use {} for valid YAML
|
|
43
|
+
if (fieldsToRender.length === 0) {
|
|
44
|
+
lines.push(` ${connectorName}: {}`);
|
|
45
|
+
} else {
|
|
46
|
+
lines.push(` ${connectorName}:`);
|
|
47
|
+
}
|
|
36
48
|
|
|
37
49
|
// Four‐space indent for children
|
|
38
50
|
const baseIndent = 4;
|
|
39
|
-
|
|
40
|
-
if (field.is_deprecated) {
|
|
41
|
-
return; // skip deprecated fields
|
|
42
|
-
}
|
|
43
|
-
if (!includeAdvanced && field.is_advanced) {
|
|
44
|
-
return; // skip advanced fields in "common" mode
|
|
45
|
-
}
|
|
46
|
-
|
|
51
|
+
fieldsToRender.forEach(field => {
|
|
47
52
|
// Check if this is an array-of-objects (e.g., client_certs[])
|
|
48
53
|
// These should render as empty arrays, not expanded object structures
|
|
49
54
|
if (field.kind === 'array' && field.type === 'object' && Array.isArray(field.children)) {
|
|
@@ -8,7 +8,8 @@ const { getAntoraValue, setAntoraValue } = require('../../cli-utils/antora-utils
|
|
|
8
8
|
const fetchFromGithub = require('../fetch-from-github.js')
|
|
9
9
|
const { generateRpcnConnectorDocs } = require('./generate-rpcn-connector-docs.js')
|
|
10
10
|
const { getRpkConnectVersion, printDeltaReport } = require('./report-delta')
|
|
11
|
-
const { discoverIntermediateReleases } = require('./github-release-utils')
|
|
11
|
+
const { discoverIntermediateReleases, findCloudVersionForDate } = require('./github-release-utils')
|
|
12
|
+
const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
|
|
12
13
|
const parseCSVConnectors = require('./parse-csv-connectors.js')
|
|
13
14
|
const semver = require('semver')
|
|
14
15
|
|
|
@@ -600,15 +601,17 @@ function logCollapsed (label, filesArray, maxToShow = 10) {
|
|
|
600
601
|
async function loadConnectorDataForVersion(version, dataDir, options = {}) {
|
|
601
602
|
const dataFile = path.join(dataDir, `connect-${version}.json`);
|
|
602
603
|
|
|
603
|
-
// If
|
|
604
|
-
|
|
604
|
+
// If forceFresh is set, always fetch even if file exists
|
|
605
|
+
// This ensures we have accurate connector lists for diff comparison
|
|
606
|
+
if (!options.forceFresh && fs.existsSync(dataFile)) {
|
|
605
607
|
console.log(`✓ Using existing data file: connect-${version}.json`);
|
|
606
608
|
const data = JSON.parse(fs.readFileSync(dataFile, 'utf8'));
|
|
607
609
|
return data;
|
|
608
610
|
}
|
|
609
611
|
|
|
610
|
-
//
|
|
611
|
-
|
|
612
|
+
// Fetch fresh data
|
|
613
|
+
const reason = options.forceFresh ? 'force refresh requested' : 'file not found';
|
|
614
|
+
console.log(`📥 Fetching data for ${version} (${reason})...`);
|
|
612
615
|
|
|
613
616
|
try {
|
|
614
617
|
// Try installing that specific version and fetching data
|
|
@@ -866,38 +869,62 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
866
869
|
console.log('─'.repeat(80) + '\n')
|
|
867
870
|
|
|
868
871
|
try {
|
|
869
|
-
// Load data for both versions
|
|
870
|
-
|
|
871
|
-
|
|
872
|
+
// Load data for both versions - FORCE FRESH to avoid stale/incomplete data
|
|
873
|
+
// This ensures we have accurate connector lists for both versions
|
|
874
|
+
console.log(`Loading connector data for ${fromVer} (fresh fetch)...`)
|
|
875
|
+
const oldData = await loadConnectorDataForVersion(fromVer, dataDir, { forceFresh: true })
|
|
872
876
|
|
|
873
|
-
console.log(`Loading connector data for ${toVer}...`)
|
|
874
|
-
const newData = await loadConnectorDataForVersion(toVer, dataDir)
|
|
877
|
+
console.log(`Loading connector data for ${toVer} (fresh fetch)...`)
|
|
878
|
+
const newData = await loadConnectorDataForVersion(toVer, dataDir, { forceFresh: true })
|
|
875
879
|
|
|
876
|
-
|
|
877
|
-
|
|
880
|
+
const analysisOptions = {
|
|
881
|
+
skipCloud: false,
|
|
882
|
+
skipCgo: false,
|
|
883
|
+
cgoVersion: options.cgoVersion || null
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
// Determine cloud version for OLD release date
|
|
887
|
+
const oldReleaseInfo = intermediateReleases.find(r => r.version === fromVer)
|
|
888
|
+
let cloudVersionForOldRelease = options.cloudVersion || null
|
|
889
|
+
|
|
890
|
+
if (!options.cloudVersion && oldReleaseInfo && oldReleaseInfo.date) {
|
|
891
|
+
cloudVersionForOldRelease = await findCloudVersionForDate(oldReleaseInfo.date, { useCache: true })
|
|
892
|
+
if (cloudVersionForOldRelease) {
|
|
893
|
+
console.log(` Using cloud version ${cloudVersionForOldRelease} for old release ${fromVer}`)
|
|
894
|
+
} else {
|
|
895
|
+
cloudVersionForOldRelease = fromVer
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
// Determine cloud version for NEW release date
|
|
900
|
+
const newReleaseInfo = intermediateReleases.find(r => r.version === toVer)
|
|
878
901
|
let cloudVersionForRelease = options.cloudVersion || null
|
|
879
902
|
|
|
880
|
-
if (!options.cloudVersion &&
|
|
881
|
-
|
|
882
|
-
cloudVersionForRelease = await findCloudVersionForDate(releaseInfo.date, { useCache: true })
|
|
903
|
+
if (!options.cloudVersion && newReleaseInfo && newReleaseInfo.date) {
|
|
904
|
+
cloudVersionForRelease = await findCloudVersionForDate(newReleaseInfo.date, { useCache: true })
|
|
883
905
|
if (cloudVersionForRelease) {
|
|
884
|
-
console.log(` Using cloud version ${cloudVersionForRelease}
|
|
906
|
+
console.log(` Using cloud version ${cloudVersionForRelease} for new release ${toVer}`)
|
|
885
907
|
} else {
|
|
886
|
-
console.log(` No cloud version found for release date, using OSS version ${toVer}`)
|
|
887
908
|
cloudVersionForRelease = toVer
|
|
888
909
|
}
|
|
889
910
|
}
|
|
890
911
|
|
|
891
|
-
// Run binary analysis for
|
|
892
|
-
console.log(`\nAnalyzing binaries for version ${
|
|
893
|
-
const
|
|
912
|
+
// Run binary analysis for BOTH versions to ensure symmetric comparison
|
|
913
|
+
console.log(`\nAnalyzing binaries for OLD version ${fromVer}...`)
|
|
914
|
+
const oldAnalysis = await analyzeAllBinaries(
|
|
915
|
+
fromVer,
|
|
916
|
+
cloudVersionForOldRelease,
|
|
917
|
+
dataDir,
|
|
918
|
+
analysisOptions
|
|
919
|
+
)
|
|
894
920
|
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
921
|
+
console.log(`Done: Binary analysis for ${fromVer}:`)
|
|
922
|
+
console.log(` - OSS version: ${oldAnalysis.ossVersion}`)
|
|
923
|
+
if (oldAnalysis.cgoOnly && oldAnalysis.cgoOnly.length > 0) {
|
|
924
|
+
console.log(` - CGO-only connectors: ${oldAnalysis.cgoOnly.length}`)
|
|
899
925
|
}
|
|
900
926
|
|
|
927
|
+
console.log(`\nAnalyzing binaries for NEW version ${toVer}...`)
|
|
901
928
|
const intermediateAnalysis = await analyzeAllBinaries(
|
|
902
929
|
toVer,
|
|
903
930
|
cloudVersionForRelease,
|
|
@@ -905,40 +932,46 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
905
932
|
analysisOptions
|
|
906
933
|
)
|
|
907
934
|
|
|
908
|
-
console.log(
|
|
909
|
-
console.log(`
|
|
935
|
+
console.log(`Done: Binary analysis for ${toVer}:`)
|
|
936
|
+
console.log(` - OSS version: ${intermediateAnalysis.ossVersion}`)
|
|
910
937
|
if (intermediateAnalysis.cloudVersion) {
|
|
911
|
-
console.log(`
|
|
938
|
+
console.log(` - Cloud version: ${intermediateAnalysis.cloudVersion}`)
|
|
912
939
|
}
|
|
913
940
|
if (intermediateAnalysis.comparison) {
|
|
914
|
-
console.log(`
|
|
915
|
-
console.log(`
|
|
941
|
+
console.log(` - Connectors in cloud: ${intermediateAnalysis.comparison.inCloud.length}`)
|
|
942
|
+
console.log(` - Self-hosted only: ${intermediateAnalysis.comparison.notInCloud.length}`)
|
|
916
943
|
if (intermediateAnalysis.comparison.cloudOnly) {
|
|
917
|
-
console.log(`
|
|
944
|
+
console.log(` - Cloud-only: ${intermediateAnalysis.comparison.cloudOnly.length}`)
|
|
918
945
|
}
|
|
919
946
|
}
|
|
947
|
+
if (intermediateAnalysis.cgoOnly && intermediateAnalysis.cgoOnly.length > 0) {
|
|
948
|
+
console.log(` - CGO-only connectors: ${intermediateAnalysis.cgoOnly.length}`)
|
|
949
|
+
}
|
|
920
950
|
|
|
921
|
-
// Augment newData with binary analysis
|
|
922
|
-
// This ensures
|
|
923
|
-
const { augmentedData:
|
|
951
|
+
// Augment BOTH oldData and newData with their respective binary analysis
|
|
952
|
+
// This ensures symmetric comparison - both have CGO/cloud connectors and metadata
|
|
953
|
+
const { augmentedData: augmentedOldData, addedCgoCount: oldCgoCount, addedCloudOnlyCount: oldCloudOnlyCount } =
|
|
954
|
+
augmentConnectorData(oldData, oldAnalysis)
|
|
955
|
+
|
|
956
|
+
const { augmentedData: augmentedNewData, addedCgoCount: newCgoCount, addedCloudOnlyCount: newCloudOnlyCount } =
|
|
924
957
|
augmentConnectorData(newData, intermediateAnalysis)
|
|
925
958
|
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
}
|
|
959
|
+
console.log(` - Augmented oldData: +${oldCgoCount} CGO-only, +${oldCloudOnlyCount} cloud-only`)
|
|
960
|
+
console.log(` - Augmented newData: +${newCgoCount} CGO-only, +${newCloudOnlyCount} cloud-only`)
|
|
929
961
|
|
|
930
|
-
// Generate diff
|
|
931
|
-
console.log(`\nGenerating diff: ${fromVer}
|
|
962
|
+
// Generate diff with BOTH augmented (symmetric comparison)
|
|
963
|
+
console.log(`\nGenerating diff: ${fromVer} -> ${toVer}...`)
|
|
932
964
|
const { generateConnectorDiffJson } = require('./report-delta.js')
|
|
933
965
|
|
|
934
966
|
const diffData = generateConnectorDiffJson(
|
|
935
|
-
|
|
967
|
+
augmentedOldData,
|
|
936
968
|
augmentedNewData,
|
|
937
969
|
{
|
|
938
970
|
oldVersion: fromVer,
|
|
939
971
|
newVersion: toVer,
|
|
940
972
|
timestamp,
|
|
941
|
-
binaryAnalysis: intermediateAnalysis
|
|
973
|
+
binaryAnalysis: intermediateAnalysis,
|
|
974
|
+
oldBinaryAnalysis: oldAnalysis
|
|
942
975
|
}
|
|
943
976
|
)
|
|
944
977
|
|
|
@@ -1136,7 +1169,6 @@ async function handleRpcnConnectorDocs (options) {
|
|
|
1136
1169
|
|
|
1137
1170
|
try {
|
|
1138
1171
|
console.log('\nAnalyzing connector binaries...')
|
|
1139
|
-
const { analyzeAllBinaries } = require('./connector-binary-analyzer.js')
|
|
1140
1172
|
|
|
1141
1173
|
if (fs.existsSync(cleanOssDataPath)) {
|
|
1142
1174
|
if (fs.existsSync(expectedPath)) {
|