@tdk-landscape/tdk-cli-core 1.3.14 → 1.3.22

package/discovery/config.star

@@ -0,0 +1,258 @@
+#!/usr/bin/env starlark
+# =============================================================================
+# 🗺️ TOPOLOGIES - DISCOVERY CONFIG
+# =============================================================================
+
+load("./constants.star", "RESOURCES_ROOT")
+load("../engine/topologies/platform/docker/constants.star", "PlatformDockerConstants")
+
+# NOTE: DEFAULTS and focus lists are now loaded dynamically in resource_registry.star
+# from the project's spec.master (using TDK_PROJECT_ROOT environment variable).
+# This allows per-project resource configuration.
+
+# Default empty values - will be populated by _load_project_defaults() in resource_registry.star
+DEFAULTS = {}
+FOCUS_PRE_ALPHA = []
+FOCUS_ALPHA = []
+FOCUS_BETA = []
+
+# Export a function to load project-specific defaults
+def load_project_defaults(project_root):
+    """
+    Load DEFAULTS and focus lists from project spec.master.
+    Called by resource_registry.star during initialization.
+    
+    Returns struct with loaded values or None if spec.master not found.
+    """
+    if not project_root:
+        return None
+    
+    # Check multiple locations for spec.master (in order of preference)
+    spec_paths = [
+        project_root + "/.tdk/.tdk-out/generated/spec.master",  # New unified output folder
+        project_root + "/spec.master",                       # Original location (legacy)
+    ]
+    
+    spec_path = None
+    for path in spec_paths:
+        check_cmd = "test -f '{}' && echo 'yes' || echo 'no'".format(path)
+        exists = str(local(check_cmd, quiet=True, echo_off=True)).strip() == 'yes'
+        if exists:
+            spec_path = path
+            break
+    
+    if not spec_path:
+        return None
+    
+    # Read and parse the spec.master file
+    # Since we can't dynamically load, we parse it manually
+    spec_content_raw = read_file(spec_path, default="")
+    if not spec_content_raw:
+        return None
+    
+    # Ensure content is a string (read_file may return bytes)
+    spec_content = str(spec_content_raw)
+    
+    # Extract PRE_ALPHA_RESOURCES from the file
+    # This is a simple parser for the expected format
+    defaults = {}
+    pre_alpha = []
+    alpha = []
+    beta = []
+    
+    # Parse PRE_ALPHA_RESOURCES
+    if "PRE_ALPHA_RESOURCES" in spec_content:
+        # Extract the dict content between { and }
+        start = spec_content.find("PRE_ALPHA_RESOURCES = {")
+        if start != -1:
+            start = spec_content.find("{", start)
+            end = spec_content.find("}", start)
+            if start != -1 and end != -1:
+                dict_content = spec_content[start+1:end]
+                # Parse "key": True/False entries
+                for line in dict_content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        # Extract key before colon
+                        if '"' in line or "'" in line:
+                            # Find quoted key
+                            quote_char = '"' if '"' in line else "'"
+                            key_start = line.find(quote_char)
+                            key_end = line.find(quote_char, key_start + 1)
+                            if key_start != -1 and key_end != -1:
+                                key = line[key_start+1:key_end]
+                                # Check if value is True
+                                if "True" in line:
+                                    defaults[key] = True
+                                    pre_alpha.append(key)
+    
+    # Parse ALPHA_RESOURCES
+    if "ALPHA_RESOURCES" in spec_content:
+        start = spec_content.find("ALPHA_RESOURCES = {")
+        if start != -1:
+            start = spec_content.find("{", start)
+            end = spec_content.find("}", start)
+            if start != -1 and end != -1:
+                dict_content = spec_content[start+1:end]
+                for line in dict_content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        if '"' in line or "'" in line:
+                            quote_char = '"' if '"' in line else "'"
+                            key_start = line.find(quote_char)
+                            key_end = line.find(quote_char, key_start + 1)
+                            if key_start != -1 and key_end != -1:
+                                key = line[key_start+1:key_end]
+                                if "True" in line:
+                                    defaults[key] = True
+                                    alpha.append(key)
+    
+    # Parse BETA_RESOURCES
+    if "BETA_RESOURCES" in spec_content:
+        start = spec_content.find("BETA_RESOURCES = {")
+        if start != -1:
+            start = spec_content.find("{", start)
+            end = spec_content.find("}", start)
+            if start != -1 and end != -1:
+                dict_content = spec_content[start+1:end]
+                for line in dict_content.split("\n"):
+                    line = line.strip()
+                    if line and not line.startswith("#"):
+                        if '"' in line or "'" in line:
+                            quote_char = '"' if '"' in line else "'"
+                            key_start = line.find(quote_char)
+                            key_end = line.find(quote_char, key_start + 1)
+                            if key_start != -1 and key_end != -1:
+                                key = line[key_start+1:key_end]
+                                if "True" in line:
+                                    defaults[key] = True
+                                    beta.append(key)
+    
+    return struct(
+        DEFAULTS = defaults,
+        FOCUS_PRE_ALPHA = pre_alpha,
+        FOCUS_ALPHA = pre_alpha + alpha,
+        FOCUS_BETA = pre_alpha + alpha + beta,
+    )
+
+# -----------------------------------------------------------------------------
+# 🎛️ RESOURCE DEFAULTS
+# -----------------------------------------------------------------------------
+# LOADED FROM: spec.master (searched in order):
+#   1. .tdk/.tdk-out/generated/spec.master (new unified output folder)
+#   2. spec.master (legacy location in project root)
+# PURPOSE: Single source of truth for resource enable/disable configuration
+# 
+# 📖 To modify which resources run, edit: spec.master
+# -----------------------------------------------------------------------------
+
+
+def get_global_config():
+    verdaccio_url_docker = os.environ.get(
+        "VERDACCIO_URL_DOCKER",
+        PlatformDockerConstants.VERDACCIO_URL_DOCKER,
+    )
+    return {
+        "verdaccio_url_local": PlatformDockerConstants.VERDACCIO_URL_LOCAL,
+        "verdaccio_url_docker": verdaccio_url_docker,
+        "npm_registry": PlatformDockerConstants.VERDACCIO_NPM_REGISTRY,
+        "internal_scope": "@" + PlatformDockerConstants.PROJECT_NAME + "/",
+        "library_roots": {
+            "platform": "shared-platform-engineering",
+            "product": "shared-product-engineering",
+            "ddd": "shared-ddd-layers",
+        },
+        # Relative to .tilt/topologies/tilt/discovery/
+        "resources_root": RESOURCES_ROOT,
+        "database": PlatformDockerConstants.DB_CONFIG,
+        "docker": {
+            "base_image": PlatformDockerConstants.BUN_IMAGE,
+            "nginx_image": "nginx:alpine",
+            "golden_l1_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l1:latest",
+            "golden_l2_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l2:latest",
+            "golden_l3_backend_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l3-backend:latest",
+            "golden_l3_frontend_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l3-frontend:latest",
+            "golden_l3_migrator_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l3-migrator:latest",
+            "golden_l4_backend_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l4-backend:latest",
+            "golden_l4_frontend_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l4-frontend:latest",
+            "golden_l4_migrator_image": PlatformDockerConstants.PROJECT_NAME_HYPHEN + "-l4-migrator:latest",
+            "network_prefix": PlatformDockerConstants.PROJECT_NAME + "_",
+        },
+        "ports": {
+            "backend": 3000,
+            "frontend": 80,
+            "postgres": 5432,
+            "nats": 4222,
+            "verdaccio": PlatformDockerConstants.VERDACCIO_PORT,
+        },
+    }
+
+
+GLOBAL_CONFIG = get_global_config()
+
+INFRA_RESOURCES = [
+    {"name": "database-management", "memory": 1152},
+    {"name": "proxy", "memory": 384},
+    {"name": "api-gateway", "memory": 512},
+    {"name": "verdaccio", "memory": 128},
+    {"name": "infisical", "memory": 192},
+    {"name": "monitoring", "memory": 1536},
+    {"name": "elk", "memory": 1024},
+    {"name": "debezium", "memory": 448},
+]
+
+CORE_INFRA = [
+    "init-networks",
+    "golden-layers-build",
+    "postgres",
+    "nats",
+    "traefik",
+]
+
+INFRA_STACK_MAP = {
+    "golden-layers-build": "golden-image",
+    "postgres": "database-management",
+    "nats": "database-management",
+    "redis": "database-management",
+    "traefik": "proxy",
+    "verdaccio": "verdaccio",
+    "infisical": "infisical",
+}
+
+OPTIONAL_INFRA = {
+    "monitoring": ["signoz-frontend", "signoz-otel-collector", "signoz-query-service", "clickhouse", "elasticsearch", "skywalking-oap", "skywalking-ui"],
+    "infisical": ["infisical", "infisical-db", "infisical-redis"],
+    "elk": ["elasticsearch", "logstash", "kibana"],
+    "debezium": ["kafka", "zookeeper", "nats-http-bridge", "debezium-connect", "enhanced-connector-setup"],
+}
+
+# -----------------------------------------------------------------------------
+# 📚 LIBRARY DEFINITIONS
+# -----------------------------------------------------------------------------
+# Libraries are auto-discovered from the filesystem in libraries.star.
+# Only non-auto-discoverable definitions (e.g. DDD layer paths) live here.
+# -----------------------------------------------------------------------------
+
+DDD_LIBS = [
+    {"name": "domain", "path": "shared-ddd-layers/domain"},
+    {"name": "infrastructure", "path": "shared-ddd-layers/infrastructure"},
+    {"name": "application", "path": "shared-ddd-layers/application"},
+    {"name": "presentation", "path": "shared-ddd-layers/presentation"},
+]
+
+# Export DEFAULTS and focus filters for use by other modules via Config struct
+# Note: These will be populated by load_project_defaults() in resource_registry.star
+Config = struct(
+    load_project_defaults = load_project_defaults,
+    DEFAULTS = DEFAULTS,
+    FOCUS_PRE_ALPHA = FOCUS_PRE_ALPHA,
+    FOCUS_ALPHA = FOCUS_ALPHA,
+    FOCUS_BETA = FOCUS_BETA,
+    GLOBAL = GLOBAL_CONFIG,
+    INFRA_RESOURCES = INFRA_RESOURCES,
+    CORE_INFRA = CORE_INFRA,
+    INFRA_STACK_MAP = INFRA_STACK_MAP,
+    OPTIONAL_INFRA = OPTIONAL_INFRA,
+    DDD_LIBS = DDD_LIBS,
+
+)

package/discovery/constants.star

@@ -0,0 +1,127 @@
+# =============================================================================
+# 📍 TOPOLOGIES - DISCOVERY CONSTANTS
+# =============================================================================
+
+# Import manifest filename from local manifest folder (self-contained)
+load("./manifest/constants.star", 
+    _manifest_file="MANIFEST_FILENAME", 
+    _manifest_file_yaml="MANIFEST_FILENAME_YAML"
+)
+
+# Re-export for consumers of this module
+MANIFEST_FILENAME = _manifest_file
+MANIFEST_FILENAME_YAML = _manifest_file_yaml
+
+# NOTE: These paths are resolved relative to the TILTFILE directory (project root)
+# NOT relative to this module's location
+RESOURCES_ROOT = "services/product"
+RESOURCES_ROOT_PREFIX = ""
+
+# Check for project-specific discovery patterns
+def _get_project_discovery_roots():
+    """Get discovery roots from project spec.master if available."""
+    project_root = os.environ.get('TDK_PROJECT_ROOT', '')
+    if not project_root:
+        return []
+    
+    # Try to read spec.master for custom discovery patterns
+    spec_paths = [
+        project_root + "/.tdk/.tdk-out/spec.master",
+        project_root + "/spec.master",
+    ]
+    
+    for spec_path in spec_paths:
+        check_cmd = "test -f '{}' && echo 'yes' || echo 'no'".format(spec_path)
+        exists = str(local(check_cmd, quiet=True, echo_off=True)).strip() == 'yes'
+        if exists:
+            # Read and parse spec.master for RESOURCE_PATTERNS
+            spec_content = str(read_file(spec_path, default=""))
+            if "RESOURCE_PATTERNS" in spec_content:
+                # Extract patterns from the array
+                start = spec_content.find("RESOURCE_PATTERNS = [")
+                if start != -1:
+                    start = spec_content.find("[", start)
+                    end = spec_content.find("]", start)
+                    if start != -1 and end != -1:
+                        array_content = spec_content[start+1:end]
+                        patterns = []
+                        for part in array_content.split(","):
+                            part = part.strip().strip("'").strip('"').strip()
+                            if part and not part.startswith("#"):
+                                patterns.append(part)
+                        if patterns:
+
+                            return patterns
+    return []
+
+# Get project-specific roots or fall back to defaults
+_PROJECT_DISCOVERY_ROOTS = _get_project_discovery_roots()
+
+# Phase 2: Universal Discovery - Scan roots configuration
+# Each root is scanned for service.json files
+# Paths are relative to PROJECT_ROOT (where Tiltfile is located)
+_DISCOVERY_ROOTS_RAW = _PROJECT_DISCOVERY_ROOTS if _PROJECT_DISCOVERY_ROOTS else [
+    RESOURCES_ROOT,                           # services/product/*/*/
+    "services/platform",                     # services/platform/*/*/
+    "shared-product-engineering",            # shared-product-engineering/*/
+    "cli-platform-engineering",              # cli-platform-engineering/*/
+]
+
+# SAFETY: Filter out tdk-cli's own directories to prevent self-modification
+# The CLI should never scan or write to its own installation directory
+def _filter_cli_dirs(roots):
+    """Remove tdk-cli internal directories from scan roots."""
+    filtered = []
+    for root in roots:
+        # Skip if path contains tdk-cli internal paths
+        if "tdk-cli" in root and "discovery" in root:
+            print("⚠️  Skipping tdk-cli internal directory: {}".format(root))
+            continue
+        if root.startswith("discovery/") or root == "discovery":
+            print("⚠️  Skipping discovery directory: {}".format(root))
+            continue
+        filtered.append(root)
+    return filtered
+
+DISCOVERY_SCAN_ROOTS = _filter_cli_dirs(_DISCOVERY_ROOTS_RAW)
+
+# Phase 2: Explicit stack extraction patterns for each root type
+# Documents HOW to extract stack from different directory structures
+SCAN_ROOT_PATTERNS = {
+    RESOURCES_ROOT: {
+        "stack_depth": -3,  # services/product/{STACK}/{service}/
+        "description": "Stack-grouped services (product)"
+    },
+    "services/platform": {
+        "stack_depth": -2,  # services/platform/{SERVICE}/ (flat)
+        "description": "Flat platform services"
+    },
+    "shared-product-engineering": {
+        "stack_depth": -2,  # shared-product-engineering/{SERVICE}/ (flat)
+        "description": "Shared libraries (flat)"
+    },
+    "cli-platform-engineering": {
+        "stack_depth": -2,  # cli-platform-engineering/{SERVICE}/ (flat)
+        "description": "CLI tools (flat)"
+    },
+}
+
+# Phase 2: Performance guardrails (Phase 3 optimization baseline)
+MAX_SCAN_TIMEOUT_MS = 5000    # 5 seconds per root (warn if exceeded)
+MAX_MANIFESTS_PER_ROOT = 50   # Warn if root has more manifests
+
+# Autogenerated files directory (relative to this file's location)
+PRODUCT_SNAPSHOT_DIR = "../../product_autogenerated_resource_manifests"
+
+PRODUCT_SNAPSHOT_FILES = {
+    "services": "services.autogenerated.star",
+    "dependencies": "resource_dependencies.autogenerated.star",
+    "aliases": "resource_aliases.autogenerated.star",
+    "paths": "resource_paths.autogenerated.star",
+    "index": "index.autogenerated.star",
+    "readme": "README.autogenerated.md",
+}
+
+PRODUCT_TOPOLOGY_DIR = "../../product_autogenerated_services_list"
+PRODUCT_STACKS_INDEX_FILE = "stacks.autogenerated.star"
+PRODUCT_STACK_RESOURCES_FILE = "resources.autogenerated.star"

package/discovery/deterministic.star

@@ -0,0 +1,211 @@
+# Deterministic Tilt Configuration for TDK Landscape
+# Inspired by Mira Murati's "Defeating Nondeterminism" philosophy
+# Goal: Same filesystem state + same environment = identical Tilt output
+
+# =============================================================================
+# DETERMINISTIC MODE SETTINGS
+# =============================================================================
+# Set TILT_DETERMINISTIC=true to enable strict deterministic mode
+# This ensures "same input, same output" guarantees
+# =============================================================================
+
+# Read deterministic mode flag
+_DETERMINISTIC_MODE = os.environ.get('TILT_DETERMINISTIC', 'false').lower() == 'true'
+
+if _DETERMINISTIC_MODE:
+    print("🔒 DETERMINISTIC MODE ENABLED")
+    print("  └─ Same filesystem state + same environment = identical output")
+    print("  └─ All operations are ordered, reproducible, and invariant")
+
+# =============================================================================
+# DETERMINISTIC FILE DISCOVERY
+# =============================================================================
+
+def deterministic_find(path, pattern):
+    """Deterministic file discovery - always returns sorted results.
+    
+    Args:
+        path: Directory to search
+        pattern: File pattern to match
+        
+    Returns:
+        Sorted list of files (alphabetical, cross-platform consistent)
+    """
+    # Always use | sort to ensure alphabetical ordering
+    # This makes output invariant to filesystem ordering (ext4 vs APFS vs NTFS)
+    cmd = "find {} -type f -name '{}' 2>/dev/null | sort".format(path, pattern)
+    result = str(local(cmd, quiet=True, echo_off=True)).strip()
+    if result:
+        return result.split('\n')
+    return []
+
+def deterministic_resource_discovery(scan_roots):
+    """Deterministically discover resources across all scan roots.
+    
+    Args:
+        scan_roots: List of directories to scan for service.json files
+        
+    Returns:
+        Sorted list of resource paths (guaranteed order across runs)
+    """
+    all_resources = []
+    
+    # Process scan roots in fixed order (not parallel)
+    for root in sorted(scan_roots):
+        resources = deterministic_find(root, 'service.json')
+        all_resources.extend(resources)
+    
+    # Final sort to ensure absolute ordering
+    return sorted(all_resources)
+
+# =============================================================================
+# DETERMINISTIC LOCAL EXECUTION
+# =============================================================================
+
+def deterministic_local(cmd, **kwargs):
+    """Execute local command in deterministic mode.
+    
+    In deterministic mode:
+    - Commands run sequentially (not parallel)
+    - Output is normalized (timestamps, PIDs, temp paths)
+    - Cache is used when available
+    
+    Args:
+        cmd: Command to execute
+        **kwargs: Additional arguments for local()
+        
+    Returns:
+        Command output (normalized in deterministic mode)
+    """
+    # Force quiet mode and no echo for determinism
+    kwargs['quiet'] = True
+    kwargs['echo_off'] = True
+    
+    if _DETERMINISTIC_MODE:
+        # In deterministic mode, we could add:
+        # - Output normalization
+        # - Result caching
+        # - Strict ordering
+        pass
+    
+    return local(cmd, **kwargs)
+
+# =============================================================================
+# DETERMINISTIC RESOURCE ORDERING
+# =============================================================================
+
+def sort_resources_by_name(resources):
+    """Sort resources alphabetically for deterministic ordering.
+    
+    Args:
+        resources: List of resource dictionaries or names
+        
+    Returns:
+        Sorted list (by name)
+    """
+    if resources and isinstance(resources[0], dict):
+        return sorted(resources, key=lambda x: x.get('name', ''))
+    return sorted(resources)
+
+# =============================================================================
+# DETERMINISTIC ENVIRONMENT
+# =============================================================================
+
+def get_deterministic_env():
+    """Get environment variables that affect determinism.
+    
+    Returns:
+        Dict of environment variables that must be constant for reproducibility
+    """
+    env_keys = [
+        'TILT_DETERMINISTIC',
+        'TILT_TRIGGER_MODE',
+        'MOCK_TIKKIE',
+        'INFISICAL_ENABLED',
+        'VERBOSE',
+        'AUTO_DISCOVER',
+        'TILT_MAX_PARALLEL_UPDATES',
+    ]
+    return {k: os.environ.get(k, '') for k in env_keys}
+
+def validate_deterministic_env():
+    """Validate that environment is suitable for deterministic runs.
+    
+    Returns:
+        True if environment is deterministic, False otherwise
+    """
+    if not _DETERMINISTIC_MODE:
+        return True
+    
+    # Check for sources of nondeterminism
+    warnings = []
+    
+    # Parallel updates can cause ordering differences
+    parallel = os.environ.get('TILT_MAX_PARALLEL_UPDATES', '')
+    if parallel and int(parallel) > 1:
+        warnings.append("TILT_MAX_PARALLEL_UPDATES > 1 may cause ordering differences")
+    
+    # Auto trigger mode can cause timing-dependent behavior
+    if os.environ.get('TILT_TRIGGER_MODE', 'manual') != 'manual':
+        warnings.append("TILT_TRIGGER_MODE=auto may cause timing-dependent behavior")
+    
+    if warnings:
+        print("⚠️  Deterministic mode warnings:")
+        for w in warnings:
+            print("    - {}".format(w))
+    
+    return len(warnings) == 0
+
+# =============================================================================
+# DETERMINISTIC MANIFEST GENERATION
+# =============================================================================
+
+def deterministic_yaml_generation(resource_paths):
+    """Generate YAML manifests deterministically from JSON manifests.
+    
+    Args:
+        resource_paths: List of service.json paths
+        
+    Returns:
+        List of generated/verified YAML paths
+    """
+    yaml_paths = []
+    
+    # Process in sorted order for determinism
+    for resource_json in sorted(resource_paths):
+        resource_dir = os.path.dirname(resource_json)
+        resource_yaml = os.path.join(resource_dir, 'service.yaml')
+        
+        # Check if regeneration needed
+        needs_regen = False
+        if not os.path.exists(resource_yaml):
+            needs_regen = True
+        else:
+            # Compare modification times
+            json_stat = os.stat(resource_json)
+            yaml_stat = os.path.exists(resource_yaml) and os.stat(resource_yaml)
+            if yaml_stat and json_stat.st_mtime > yaml_stat.st_mtime:
+                needs_regen = True
+        
+        yaml_paths.append(resource_yaml)
+    
+    return sorted(yaml_paths)
+
+# =============================================================================
+# EXPORTS
+# =============================================================================
+
+DETERMINISTIC_EXPORTS = {
+    'deterministic_find': deterministic_find,
+    'deterministic_resource_discovery': deterministic_resource_discovery,
+    'deterministic_local': deterministic_local,
+    'sort_resources_by_name': sort_resources_by_name,
+    'get_deterministic_env': get_deterministic_env,
+    'validate_deterministic_env': validate_deterministic_env,
+    'deterministic_yaml_generation': deterministic_yaml_generation,
+    'DETERMINISTIC_MODE': _DETERMINISTIC_MODE,
+}
+
+print("📋 Deterministic Tilt utilities loaded")
+if _DETERMINISTIC_MODE:
+    print("  └─ All operations will be ordered and reproducible")