cpflow 4.0.1 → 4.1.1

data/docs/terraform/example/.controlplane/controlplane.yml

@@ -0,0 +1,29 @@
+allow_org_override_by_env: true
+allow_app_override_by_env: true
+
+aliases:
+  common: &common
+    cpln_org: my-org-staging
+    default_location: aws-us-east-2
+    setup_app_templates:
+      - app
+      - postgres
+      - rails
+    one_off_workload: rails
+    app_workloads:
+      - rails
+    additional_workloads:
+      - postgres
+apps:
+  rails-app-staging:
+    <<: *common
+    hooks:
+      post_creation: bundle exec rake db:prepare
+      pre_deletion: bundle exec rake db:drop
+
+  rails-app-production:
+    <<: *common
+    allow_org_override_by_env: false
+    allow_app_override_by_env: false
+    cpln_org: my-org-production
+    upstream: rails-app-staging

data/docs/terraform/example/.controlplane/templates/app.yml

@@ -0,0 +1,38 @@
+kind: gvc
+name: {{APP_NAME}}
+description: Global Virtual Cloud for Rails Application
+spec:
+  env:
+    - name: DATABASE_URL
+      value: "postgres://user:password@postgres.{{APP_NAME}}.cpln.local:5432/{{APP_NAME}}"
+    - name: RAILS_ENV
+      value: production
+    - name: RAILS_SERVE_STATIC_FILES
+      value: "true"
+  staticPlacement:
+    locationLinks:
+      - {{APP_LOCATION_LINK}}
+  pullSecretLinks:
+    - "/org/org-name/secret/rails-app-secret"
+  loadBalancer:
+    dedicated: true
+    trustedProxies: 0
+
+---
+
+kind: identity
+name: rails-app-identity
+description: Identity for Rails Application
+tags:
+  environment: production
+
+---
+
+kind: secret
+name: rails-app-secret
+description: Secret for Rails Application
+type: aws
+data:
+  accessKey: 'AccessKeyExample'
+  secretKey: 'SecretKeyExample'
+  region: 'us-west-2'

data/docs/terraform/example/.controlplane/templates/postgres.yml

@@ -0,0 +1,30 @@
+kind: workload
+name: postgres
+spec:
+  type: standard
+  containers:
+    - name: postgres
+      cpu: 500m
+      env:
+        - name: POSTGRES_USER
+          value: "user"
+        - name: POSTGRES_PASSWORD
+          value: "password"
+        - name: POSTGRES_DB
+          value: "rails_app"
+      inheritEnv: true
+      image: "postgres:latest"
+      memory: 1Gi
+      ports:
+        - number: 5432
+          protocol: tcp
+  defaultOptions:
+    autoscaling:
+      maxScale: 1
+    capacityAI: false
+  firewallConfig:
+    external:
+      inboundAllowCIDR:
+        - 0.0.0.0/0
+      outboundAllowCIDR:
+        - 0.0.0.0/0

data/docs/terraform/example/.controlplane/templates/rails.yml

@@ -0,0 +1,26 @@
+kind: workload
+name: rails
+spec:
+  type: standard
+  containers:
+    - name: rails
+      cpu: 300m
+      env:
+        - name: LOG_LEVEL
+          value: debug
+      inheritEnv: true
+      image: {{APP_IMAGE_LINK}}
+      memory: 512Mi
+      ports:
+        - number: 3000
+          protocol: http
+  defaultOptions:
+    autoscaling:
+      maxScale: 1
+    capacityAI: false
+  firewallConfig:
+    external:
+      inboundAllowCIDR:
+        - 0.0.0.0/0
+      outboundAllowCIDR:
+        - 0.0.0.0/0

data/docs/terraform/overview.md

@@ -0,0 +1,105 @@
+# Terraform
+
+## Overview
+
+The Terraform feature in this project allows you to manage your Control Plane (CPLN) configurations using Terraform by:
+1. Generating Terraform configuration files from existing CPLN YAML configuration files
+2. Easily importing existing infrastructure into Terraform management
+
+You can continue working with CPLN configuration files in YAML format and start using Terraform at any time.
+
+## Benefits of Using Terraform Over YAML Configs
+
+1. **State Management**: Terraform maintains a state file that tracks the current state of your infrastructure, making it easier to manage changes and updates.
+2. **Dependency Management**: Terraform automatically handles dependencies between resources, ensuring that they are created or destroyed in the correct order.
+3. **Multi-Cloud Support**: With Terraform, you can manage resources across multiple cloud providers seamlessly, allowing for a more flexible architecture.
+4. **Plan and Apply**: Terraform provides a clear plan of what changes will be made before applying them, reducing the risk of unintended modifications.
+
+## Usage
+
+Let's take a look at how to deploy a [simple Rails application](https://github.com/shakacode/control-plane-flow/tree/main/docs/terraform/example/.controlplane/controlplane.yml) on CPLN using Terraform:
+
+```
+.controlplane/
+├── templates/
+│   ├── app.yml -- GVC config
+│   ├── postgres.yml -- Workload config for PostgreSQL
+│   └── rails.yml -- Workload config for Rails
+└── controlplane.yml -- Configs for overall application
+```
+
+### Generating Terraform configurations
+
+To generate Terraform configurations, run the following command from the project root:
+
+```sh
+cpflow terraform generate
+```
+
+Invoking this command will generate a new `terraform` folder with subfolders containing Terraform configurations for each application described in `controlplane.yml`:
+
+```
+terraform/
+├── rails-app-production/ -- Terraform configurations for production environment
+│   ├── gvc.tf -- GVC config in HCL
+│   ├── identities.tf -- Identities config in HCL
+│   ├── postgres.tf -- Postgres workload config in HCL
+│   ├── postgres_envs.tf -- ENV variables for Postgres workload in HCL
+│   ├── providers.tf -- Providers config in HCL
+│   ├── rails.tf -- Rails workload config in HCL
+│   ├── rails_envs.tf -- ENV variables for Rails workload in HCL
+│   ├── required_providers.tf -- Required providers config in HCL
+│   └── secrets.tf -- Secrets config in HCL
+├── rails-app-staging/ -- Terraform configurations for staging environment
+│   ├── gvc.tf -- GVC config in HCL
+│   ├── identities.tf -- Identities config in HCL
+│   ├── postgres.tf -- Postgres workload config in HCL
+│   ├── postgres_envs.tf -- ENV variables for Postgres workload in HCL
+│   ├── providers.tf -- Providers config in HCL
+│   ├── rails.tf -- Rails workload config in HCL
+│   ├── rails_envs.tf -- ENV variables for Rails workload in HCL
+│   ├── required_providers.tf -- Required providers config in HCL
+│   └── secrets.tf -- Secrets config in HCL
+├── workload/ -- Terraform configurations for workload module
+│   ├── main.tf -- Main config for workload resource in HCL
+│   ├── required_providers.tf -- Required providers for Terraform in HCL
+│   └── variables.tf -- Variables used to create config for workload resource in HCL
+```
+
+### Importing existing infrastructure
+
+Now we need to import existing infrastructure into Terraform management because some resources can already exist on CPLN and Terraform needs to know about this:
+
+```sh
+cpflow terraform import
+```
+
+This command will initialize Terraform and import resources defined in your `controlplane.yml` and `templates` folder into the Terraform state for each application.
+
+Please note that during the import process, you may encounter errors indicating that non-existing resources are being imported. This is expected behavior and can be safely ignored.
+
+### Application deployment using Terraform
+
+Preparations are complete, and now we can use Terraform commands directly to deploy our application.
+
+1. **Navigate to the Application Folder**:
+   ```sh
+   cd terraform/rails-app-staging
+   ```
+
+2. **Plan the Deployment**:
+   ```sh
+   terraform plan
+   ```
+
+3. **Apply the Configuration**:
+   ```sh
+   terraform apply
+   ```
+
+You can visit [Details](https://github.com/shakacode/control-plane-flow/tree/main/docs/terraform/details.md) to learn more about how CPLN templates in YAML format are transformed to Terraform configurations.
+
+## References
+
+- [Terraform Provider Plugin](https://shakadocs.controlplane.com/terraform/installation#terraform-provider-plugin)
+- [Terraform - Control Plane Examples](https://github.com/controlplane-com/examples/tree/main/terraform)

data/lib/command/base.rb

@@ -12,6 +12,8 @@ module Command
     VALIDATIONS_WITH_ADDITIONAL_OPTIONS = %w[templates].freeze
     ALL_VALIDATIONS = VALIDATIONS_WITHOUT_ADDITIONAL_OPTIONS + VALIDATIONS_WITH_ADDITIONAL_OPTIONS
 
+    # Used to call the command (`cpflow SUBCOMMAND_NAME NAME`)
+    SUBCOMMAND_NAME = nil
     # Used to call the command (`cpflow NAME`)
     # NAME = ""
     # Displayed when running `cpflow help` or `cpflow help NAME` (defaults to `NAME`)
@@ -43,11 +45,21 @@ module Command
       @config = config
     end
 
-    def self.all_commands
-      Dir["#{__dir__}/*.rb"].each_with_object({}) do |file, result|
-        filename = File.basename(file, ".rb")
-        classname = File.read(file).match(/^\s+class (\w+) < Base($| .*$)/)&.captures&.first
-        result[filename.to_sym] = Object.const_get("::Command::#{classname}") if classname
+    def self.all_commands # rubocop:disable Metrics/MethodLength
+      Dir["#{__dir__}/**/*.rb"].each_with_object({}) do |file, result|
+        content = File.read(file)
+
+        classname = content.match(/^\s+class (?!Base\b)(\w+) < (?:.*(?!Command::)Base)(?:$| .*$)/)&.captures&.first
+        next unless classname
+
+        namespaces = content.scan(/^\s+module (\w+)/).flatten
+        full_classname = [*namespaces, classname].join("::").prepend("::")
+
+        command_key = File.basename(file, ".rb")
+        prefix = namespaces[1..].map(&:downcase).join("_")
+        command_key.prepend(prefix.concat("_")) unless prefix.empty?
+
+        result[command_key.to_sym] = Object.const_get(full_classname)
       end
     end
 
@@ -442,6 +454,29 @@ module Command
         }
       }
     end
+
+    def self.docker_context_option
+      {
+        name: :docker_context,
+        params: {
+          desc: "Path to the docker build context directory",
+          type: :string,
+          required: false
+        }
+      }
+    end
+
+    def self.dir_option(required: false)
+      {
+        name: :dir,
+        params: {
+          banner: "DIR",
+          desc: "Output directory",
+          type: :string,
+          required: required
+        }
+      }
+    end
     # rubocop:enable Metrics/MethodLength
 
     def self.all_options

data/lib/command/base_sub_command.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Inspired by https://github.com/rails/thor/wiki/Subcommands
+class BaseSubCommand < Thor
+  def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
+    "#{basename} #{subcommand_prefix} #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    name
+      .gsub(/.*::/, "")
+      .gsub(/^[A-Z]/) { |match| match[0].downcase }
+      .gsub(/[A-Z]/) { |match| "-#{match[0].downcase}" }
+  end
+end

data/lib/command/build_image.rb

@@ -5,7 +5,8 @@ module Command
     NAME = "build-image"
     OPTIONS = [
       app_option(required: true),
-      commit_option
+      commit_option,
+      docker_context_option
     ].freeze
     ACCEPTS_EXTRA_OPTIONS = true
     DESCRIPTION = "Builds and pushes the image to Control Plane"
@@ -34,9 +35,12 @@ module Command
       build_args = []
       build_args.push("GIT_COMMIT=#{commit}") if commit
 
+      docker_context = config.options[:docker_context] || config.app_dir
+
       cp.image_build(image_url, dockerfile: dockerfile,
                                 docker_args: config.args,
-                                build_args: build_args)
+                                build_args: build_args,
+                                docker_context: docker_context)
 
       push_path = "/org/#{config.org}/image/#{image_name}"
 

data/lib/command/delete.rb

@@ -106,9 +106,9 @@ module Command
     def delete_volumesets
       @volumesets.each do |volumeset|
         step("Deleting volumeset '#{volumeset['name']}' from app '#{config.app}'") do
-          # If the volumeset is attached to a workload, we need to delete the workload first
-          workload = volumeset.dig("status", "usedByWorkload")&.split("/")&.last
-          cp.delete_workload(workload) if workload
+          # If the volumeset is attached to workloads, we need to delete the workloads first
+          workloads = volumeset.dig("status", "workloadLinks")&.map { |workload_link| workload_link.split("/").last }
+          workloads&.each { |workload| cp.delete_workload(workload) }
 
           cp.delete_volumeset(volumeset["name"])
         end

data/lib/command/deploy_image.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "resolv"
+
 module Command
   class DeployImage < Base
     NAME = "deploy-image"

data/lib/command/generate.rb

@@ -9,7 +9,7 @@ module Command
     end
 
     def self.source_root
-      File.expand_path("../", __dir__)
+      Cpflow.root_path.join("lib")
     end
   end
 

data/lib/command/ps.rb

@@ -34,7 +34,7 @@ module Command
         cp.fetch_workload!(workload)
 
         result = cp.fetch_workload_replicas(workload, location: location)
-        result["items"].each { |replica| puts replica }
+        result&.dig("items")&.each { |replica| puts replica }
       end
     end
   end

data/lib/command/ps_stop.rb

@@ -75,7 +75,8 @@ module Command
 
       step("Waiting for replica '#{replica}' to not be ready", retry_on_failure: true) do
         result = cp.fetch_workload_replicas(workload, location: config.location)
-        !result["items"].include?(replica)
+        items = result&.dig("items")
+        items && !items.include?(replica)
       end
     end
   end

data/lib/command/run.rb

@@ -274,7 +274,7 @@ module Command
     def wait_for_replica_for_job
       step("Waiting for replica to start, which runs job '#{job}'", retry_on_failure: true) do
         result = cp.fetch_workload_replicas(runner_workload, location: location)
-        @replica = result["items"].find { |item| item.include?(job) }
+        @replica = result&.dig("items")&.find { |item| item.include?(job) }
 
         replica || false
       end

data/lib/command/setup_app.rb

@@ -14,8 +14,8 @@ module Command
       - Creates an app and all its workloads
       - Specify the templates for the app and workloads through `setup_app_templates` in the `.controlplane/controlplane.yml` file
       - This should only be used for temporary apps like review apps, never for persistent apps like production or staging (to update workloads for those, use 'cpflow apply-template' instead)
-      - Configures app to have org-level secrets with default name "{APP_PREFIX}-secrets"
-        using org-level policy with default name "{APP_PREFIX}-secrets-policy" (names can be customized, see docs)
+      - Configures app to have org-level secrets with default name `"{APP_PREFIX}-secrets"`
+        using org-level policy with default name `"{APP_PREFIX}-secrets-policy"` (names can be customized, see docs)
       - Creates identity for secrets if it does not exist
       - Use `--skip-secrets-setup` to prevent the automatic setup of secrets,
         or set it through `skip_secrets_setup` in the `.controlplane/controlplane.yml` file

data/lib/command/terraform/base.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Base < Command::Base
+      private
+
+      def templates
+        parser = TemplateParser.new(self)
+        template_files = Dir["#{parser.template_dir}/*.yml"]
+
+        if template_files.empty?
+          Shell.warn("No templates found in #{parser.template_dir}")
+          return []
+        end
+
+        parser.parse(template_files)
+      rescue StandardError => e
+        Shell.warn("Error parsing templates: #{e.message}")
+        []
+      end
+
+      def terraform_dir
+        @terraform_dir ||= begin
+          full_path = config.options.fetch(:dir, Cpflow.root_path.join("terraform"))
+          Pathname.new(full_path).tap do |path|
+            FileUtils.mkdir_p(path)
+          rescue StandardError => e
+            Shell.abort("Invalid directory: #{e.message}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/command/terraform/generate.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Generate < Base
+      SUBCOMMAND_NAME = "terraform"
+      NAME = "generate"
+      OPTIONS = [
+        app_option,
+        dir_option
+      ].freeze
+      DESCRIPTION = "Generates terraform configuration files"
+      LONG_DESCRIPTION = <<~DESC
+        - Generates terraform configuration files based on `controlplane.yml` and `templates/` config
+      DESC
+      WITH_INFO_HEADER = false
+
+      def call
+        Array(config.app || config.apps.keys).each do |app|
+          config.instance_variable_set(:@app, app.to_s)
+          generate_app_config
+        end
+      end
+
+      private
+
+      def generate_app_config
+        copy_workload_module
+
+        terraform_app_dir = cleaned_terraform_app_dir
+        generate_provider_configs(terraform_app_dir)
+
+        templates.each do |template|
+          TerraformConfig::Generator.new(config: config, template: template).tf_configs.each do |filename, tf_config|
+            File.write(terraform_app_dir.join(filename), tf_config.to_tf, mode: "a+")
+          end
+        rescue TerraformConfig::Generator::InvalidTemplateError => e
+          Shell.warn(e.message)
+        end
+      end
+
+      def copy_workload_module
+        FileUtils.copy_entry(
+          Cpflow.root_path.join("lib/core/terraform_config/workload"),
+          terraform_dir.join("workload")
+        )
+      end
+
+      def generate_provider_configs(terraform_app_dir)
+        generate_required_providers(terraform_app_dir)
+        generate_providers(terraform_app_dir)
+      rescue StandardError => e
+        Shell.abort("Failed to generate provider config files: #{e.message}")
+      end
+
+      def generate_required_providers(terraform_app_dir)
+        File.write(terraform_app_dir.join("required_providers.tf"), required_cpln_provider.to_tf)
+      end
+
+      def generate_providers(terraform_app_dir)
+        cpln_provider = TerraformConfig::Provider.new(name: "cpln", org: config.org)
+        File.write(terraform_app_dir.join("providers.tf"), cpln_provider.to_tf)
+      end
+
+      def required_cpln_provider
+        TerraformConfig::RequiredProvider.new(
+          name: "cpln",
+          org: config.org,
+          source: "controlplane-com/cpln",
+          version: "~> 1.0"
+        )
+      end
+
+      def cleaned_terraform_app_dir
+        full_path = terraform_dir.join(config.app)
+
+        unless File.expand_path(full_path).include?(Cpflow.root_path.to_s)
+          Shell.abort("Directory to save terraform configuration files cannot be outside of current directory")
+        end
+
+        if Dir.exist?(full_path)
+          clean_terraform_app_dir(full_path)
+        else
+          FileUtils.mkdir_p(full_path)
+        end
+
+        full_path
+      end
+
+      def clean_terraform_app_dir(terraform_app_dir)
+        Dir.children(terraform_app_dir).each do |child|
+          next if child == ".terraform.lock.hcl"
+
+          FileUtils.rm_rf(terraform_app_dir.join(child))
+        end
+      end
+    end
+  end
+end

data/lib/command/terraform/import.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Import < Base
+      SUBCOMMAND_NAME = "terraform"
+      NAME = "import"
+      OPTIONS = [
+        app_option,
+        dir_option
+      ].freeze
+      DESCRIPTION = "Imports terraform resources"
+      LONG_DESCRIPTION = <<~DESC
+        - Imports terraform resources from the generated configuration files
+      DESC
+      WITH_INFO_HEADER = false
+
+      def call
+        Array(config.app || config.apps.keys).each do |app|
+          config.instance_variable_set(:@app, app.to_s)
+
+          Dir.chdir(terraform_app_dir) do
+            run_terraform_init
+
+            resources.each do |resource|
+              run_terraform_import(resource[:address], resource[:id])
+            end
+          end
+        end
+      end
+
+      private
+
+      def run_terraform_init
+        result = Shell.cmd("terraform", "init", capture_stderr: true)
+
+        if result[:success]
+          Shell.info(result[:output])
+        else
+          Shell.abort("Failed to initialize terraform - #{result[:output]}")
+        end
+      end
+
+      def run_terraform_import(address, id)
+        result = Shell.cmd("terraform", "import", address, id, capture_stderr: true)
+        Shell.info(result[:output])
+      end
+
+      def resources
+        tf_configs.filter_map do |tf_config|
+          next unless tf_config.importable?
+
+          { address: tf_config.reference, id: resource_id(tf_config) }
+        end
+      end
+
+      def tf_configs
+        templates.flat_map do |template|
+          TerraformConfig::Generator.new(config: config, template: template).tf_configs.values
+        end
+      end
+
+      def resource_id(tf_config)
+        case tf_config
+        when TerraformConfig::Gvc, TerraformConfig::Policy,
+             TerraformConfig::Secret, TerraformConfig::Agent,
+             TerraformConfig::AuditContext
+          tf_config.name
+        else
+          "#{config.app}:#{tf_config.name}"
+        end
+      end
+
+      def terraform_app_dir
+        terraform_dir.join(config.app)
+      end
+    end
+  end
+end

data/lib/core/controlplane.rb

@@ -90,7 +90,7 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     api.query_images(org: a_org, gvc: a_gvc, gvc_op_type: gvc_op)
   end
 
-  def image_build(image, dockerfile:, docker_args: [], build_args: [])
+  def image_build(image, dockerfile:, docker_context:, docker_args: [], build_args: [])
     # https://docs.controlplane.com/guides/push-image#step-2
     # Might need to use `docker buildx build` if compatiblitity issues arise
     cmd = "docker build --platform=linux/amd64 -t #{image} -f #{dockerfile}"
@@ -98,7 +98,7 @@ class Controlplane # rubocop:disable Metrics/ClassLength
 
     cmd += " #{docker_args.join(' ')}" if docker_args.any?
     build_args.each { |build_arg| cmd += " --build-arg #{build_arg}" }
-    cmd += " #{config.app_dir}"
+    cmd += " #{docker_context}"
 
     perform!(cmd)
   end
@@ -192,7 +192,7 @@ class Controlplane # rubocop:disable Metrics/ClassLength
   end
 
   def fetch_workload_replicas(workload, location:)
-    cmd = "cpln workload replica get #{workload} #{gvc_org} --location #{location} -o yaml"
+    cmd = "cpln workload replica get #{workload} #{gvc_org} --location #{location} -o yaml 2> /dev/null"
     perform_yaml(cmd)
   end
 
@@ -213,8 +213,8 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     end
   end
 
-  def workload_deployments_ready?(workload, location:, expected_status:)
-    deployed_replicas = fetch_workload_replicas(workload, location: location)["items"].length
+  def workload_deployments_ready?(workload, location:, expected_status:) # rubocop:disable Metrics/CyclomaticComplexity
+    deployed_replicas = fetch_workload_replicas(workload, location: location)&.dig("items")&.length || 0
     return deployed_replicas.zero? if expected_status == false
 
     deployments = fetch_workload_deployments(workload)["items"]