cpflow 4.1.0 → 4.2.0

data/lib/command/base_sub_command.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# Inspired by https://github.com/rails/thor/wiki/Subcommands
+class BaseSubCommand < Thor
+  def self.banner(command, _namespace = nil, _subcommand = false) # rubocop:disable Style/OptionalBooleanParameter
+    "#{basename} #{subcommand_prefix} #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    name
+      .gsub(/.*::/, "")
+      .gsub(/^[A-Z]/) { |match| match[0].downcase }
+      .gsub(/[A-Z]/) { |match| "-#{match[0].downcase}" }
+  end
+end

data/lib/command/generate.rb

@@ -9,7 +9,7 @@ module Command
     end
 
     def self.source_root
-      File.expand_path("../", __dir__)
+      Cpflow.root_path.join("lib")
     end
   end
 

data/lib/command/ps.rb

@@ -34,7 +34,7 @@ module Command
         cp.fetch_workload!(workload)
 
         result = cp.fetch_workload_replicas(workload, location: location)
-        result["items"].each { |replica| puts replica }
+        result&.dig("items")&.each { |replica| puts replica }
       end
     end
   end

data/lib/command/ps_stop.rb

@@ -75,7 +75,8 @@ module Command
 
       step("Waiting for replica '#{replica}' to not be ready", retry_on_failure: true) do
         result = cp.fetch_workload_replicas(workload, location: config.location)
-        !result["items"].include?(replica)
+        items = result&.dig("items")
+        items && !items.include?(replica)
       end
     end
   end

data/lib/command/ps_wait.rb

@@ -11,6 +11,7 @@ module Command
     DESCRIPTION = "Waits for workloads in app to be ready after re-deployment"
     LONG_DESCRIPTION = <<~DESC
       - Waits for workloads in app to be ready after re-deployment
+      - Use Unix timeout command to set a maximum wait time (e.g., `timeout 300 cpflow ps:wait ...`)
     DESC
     EXAMPLES = <<~EX
       ```sh
@@ -18,7 +19,10 @@ module Command
       cpflow ps:wait -a $APP_NAME
 
       # Waits for a specific workload in app.
-      cpflow ps:swait -a $APP_NAME -w $WORKLOAD_NAME
+      cpflow ps:wait -a $APP_NAME -w $WORKLOAD_NAME
+
+      # Waits for all workloads with a 5-minute timeout.
+      timeout 300 cpflow ps:wait -a $APP_NAME
       ```
     EX
 

data/lib/command/run.rb

@@ -97,7 +97,7 @@ module Command
 
     attr_reader :interactive, :detached, :location, :original_workload, :runner_workload,
                 :default_image, :default_cpu, :default_memory, :job_timeout, :job_history_limit,
-                :container, :expected_deployed_version, :job, :replica, :command
+                :container, :job, :replica, :command
 
     def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       @interactive = config.options[:interactive] || interactive_command?
@@ -126,10 +126,7 @@ module Command
       end
 
       create_runner_workload if cp.fetch_workload(runner_workload).nil?
-      wait_for_runner_workload_deploy
       update_runner_workload
-      wait_for_runner_workload_update if expected_deployed_version
-
       start_job
       wait_for_replica_for_job
 
@@ -191,7 +188,7 @@ module Command
         }
 
         # Create runner workload
-        cp.apply_hash("kind" => "workload", "name" => runner_workload, "spec" => spec)
+        cp.apply_hash({ "kind" => "workload", "name" => runner_workload, "spec" => spec }, wait: true)
       end
     end
 
@@ -242,21 +239,7 @@ module Command
       return unless should_update
 
       step("Updating runner workload '#{runner_workload}'") do
-        # Update runner workload
-        @expected_deployed_version = (cp.cron_workload_deployed_version(runner_workload) || 0) + 1
-        cp.apply_hash("kind" => "workload", "name" => runner_workload, "spec" => spec)
-      end
-    end
-
-    def wait_for_runner_workload_deploy
-      step("Waiting for runner workload '#{runner_workload}' to be deployed", retry_on_failure: true) do
-        !cp.cron_workload_deployed_version(runner_workload).nil?
-      end
-    end
-
-    def wait_for_runner_workload_update
-      step("Waiting for runner workload '#{runner_workload}' to be updated", retry_on_failure: true) do
-        (cp.cron_workload_deployed_version(runner_workload) || 0) >= expected_deployed_version
+        cp.apply_hash({ "kind" => "workload", "name" => runner_workload, "spec" => spec }, wait: true)
       end
     end
 
@@ -274,7 +257,7 @@ module Command
     def wait_for_replica_for_job
       step("Waiting for replica to start, which runs job '#{job}'", retry_on_failure: true) do
         result = cp.fetch_workload_replicas(runner_workload, location: location)
-        @replica = result["items"].find { |item| item.include?(job) }
+        @replica = result&.dig("items")&.find { |item| item.include?(job) }
 
         replica || false
       end

data/lib/command/terraform/base.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Base < Command::Base
+      private
+
+      def templates
+        parser = TemplateParser.new(self)
+        template_files = Dir["#{parser.template_dir}/*.yml"]
+
+        if template_files.empty?
+          Shell.warn("No templates found in #{parser.template_dir}")
+          return []
+        end
+
+        parser.parse(template_files)
+      rescue StandardError => e
+        Shell.warn("Error parsing templates: #{e.message}")
+        []
+      end
+
+      def terraform_dir
+        @terraform_dir ||= begin
+          full_path = config.options.fetch(:dir, Cpflow.root_path.join("terraform"))
+          Pathname.new(full_path).tap do |path|
+            FileUtils.mkdir_p(path)
+          rescue StandardError => e
+            Shell.abort("Invalid directory: #{e.message}")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/command/terraform/generate.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Generate < Base
+      SUBCOMMAND_NAME = "terraform"
+      NAME = "generate"
+      OPTIONS = [
+        app_option,
+        dir_option
+      ].freeze
+      DESCRIPTION = "Generates terraform configuration files"
+      LONG_DESCRIPTION = <<~DESC
+        - Generates terraform configuration files based on `controlplane.yml` and `templates/` config
+      DESC
+      WITH_INFO_HEADER = false
+
+      def call
+        Array(config.app || config.apps.keys).each do |app|
+          config.instance_variable_set(:@app, app.to_s)
+          generate_app_config
+        end
+      end
+
+      private
+
+      def generate_app_config
+        copy_workload_module
+
+        terraform_app_dir = cleaned_terraform_app_dir
+        generate_provider_configs(terraform_app_dir)
+
+        templates.each do |template|
+          TerraformConfig::Generator.new(config: config, template: template).tf_configs.each do |filename, tf_config|
+            File.write(terraform_app_dir.join(filename), tf_config.to_tf, mode: "a+")
+          end
+        rescue TerraformConfig::Generator::InvalidTemplateError => e
+          Shell.warn(e.message)
+        end
+      end
+
+      def copy_workload_module
+        FileUtils.copy_entry(
+          Cpflow.root_path.join("lib/core/terraform_config/workload"),
+          terraform_dir.join("workload")
+        )
+      end
+
+      def generate_provider_configs(terraform_app_dir)
+        generate_required_providers(terraform_app_dir)
+        generate_providers(terraform_app_dir)
+      rescue StandardError => e
+        Shell.abort("Failed to generate provider config files: #{e.message}")
+      end
+
+      def generate_required_providers(terraform_app_dir)
+        File.write(terraform_app_dir.join("required_providers.tf"), required_cpln_provider.to_tf)
+      end
+
+      def generate_providers(terraform_app_dir)
+        cpln_provider = TerraformConfig::Provider.new(name: "cpln", org: config.org)
+        File.write(terraform_app_dir.join("providers.tf"), cpln_provider.to_tf)
+      end
+
+      def required_cpln_provider
+        TerraformConfig::RequiredProvider.new(
+          name: "cpln",
+          org: config.org,
+          source: "controlplane-com/cpln",
+          version: "~> 1.0"
+        )
+      end
+
+      def cleaned_terraform_app_dir
+        full_path = terraform_dir.join(config.app)
+
+        unless File.expand_path(full_path).include?(Cpflow.root_path.to_s)
+          Shell.abort("Directory to save terraform configuration files cannot be outside of current directory")
+        end
+
+        if Dir.exist?(full_path)
+          clean_terraform_app_dir(full_path)
+        else
+          FileUtils.mkdir_p(full_path)
+        end
+
+        full_path
+      end
+
+      def clean_terraform_app_dir(terraform_app_dir)
+        Dir.children(terraform_app_dir).each do |child|
+          next if child == ".terraform.lock.hcl"
+
+          FileUtils.rm_rf(terraform_app_dir.join(child))
+        end
+      end
+    end
+  end
+end

data/lib/command/terraform/import.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Command
+  module Terraform
+    class Import < Base
+      SUBCOMMAND_NAME = "terraform"
+      NAME = "import"
+      OPTIONS = [
+        app_option,
+        dir_option
+      ].freeze
+      DESCRIPTION = "Imports terraform resources"
+      LONG_DESCRIPTION = <<~DESC
+        - Imports terraform resources from the generated configuration files
+      DESC
+      WITH_INFO_HEADER = false
+
+      def call
+        Array(config.app || config.apps.keys).each do |app|
+          config.instance_variable_set(:@app, app.to_s)
+
+          Dir.chdir(terraform_app_dir) do
+            run_terraform_init
+
+            resources.each do |resource|
+              run_terraform_import(resource[:address], resource[:id])
+            end
+          end
+        end
+      end
+
+      private
+
+      def run_terraform_init
+        result = Shell.cmd("terraform", "init", capture_stderr: true)
+
+        if result[:success]
+          Shell.info(result[:output])
+        else
+          Shell.abort("Failed to initialize terraform - #{result[:output]}")
+        end
+      end
+
+      def run_terraform_import(address, id)
+        result = Shell.cmd("terraform", "import", address, id, capture_stderr: true)
+        Shell.info(result[:output])
+      end
+
+      def resources
+        tf_configs.filter_map do |tf_config|
+          next unless tf_config.importable?
+
+          { address: tf_config.reference, id: resource_id(tf_config) }
+        end
+      end
+
+      def tf_configs
+        templates.flat_map do |template|
+          TerraformConfig::Generator.new(config: config, template: template).tf_configs.values
+        end
+      end
+
+      def resource_id(tf_config)
+        case tf_config
+        when TerraformConfig::Gvc, TerraformConfig::Policy,
+             TerraformConfig::Secret, TerraformConfig::Agent,
+             TerraformConfig::AuditContext
+          tf_config.name
+        else
+          "#{config.app}:#{tf_config.name}"
+        end
+      end
+
+      def terraform_app_dir
+        terraform_dir.join(config.app)
+      end
+    end
+  end
+end

data/lib/core/config.rb

@@ -25,7 +25,7 @@ class Config # rubocop:disable Metrics/ClassLength
     return unless trace_mode
 
     ControlplaneApiDirect.trace = trace_mode
-    Shell.warn("Trace mode is enabled, this will print sensitive information to the console.")
+    Shell.warn("Trace mode is enabled. Sensitive data is redacted, but please review output before sharing.")
   end
 
   def org

data/lib/core/controlplane.rb

@@ -192,7 +192,7 @@ class Controlplane # rubocop:disable Metrics/ClassLength
   end
 
   def fetch_workload_replicas(workload, location:)
-    cmd = "cpln workload replica get #{workload} #{gvc_org} --location #{location} -o yaml"
+    cmd = "cpln workload replica get #{workload} #{gvc_org} --location #{location} -o yaml 2> /dev/null"
     perform_yaml(cmd)
   end
 
@@ -213,8 +213,8 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     end
   end
 
-  def workload_deployments_ready?(workload, location:, expected_status:)
-    deployed_replicas = fetch_workload_replicas(workload, location: location)["items"].length
+  def workload_deployments_ready?(workload, location:, expected_status:) # rubocop:disable Metrics/CyclomaticComplexity
+    deployed_replicas = fetch_workload_replicas(workload, location: location)&.dig("items")&.length || 0
     return deployed_replicas.zero? if expected_status == false
 
     deployments = fetch_workload_deployments(workload)["items"]
@@ -404,11 +404,12 @@ class Controlplane # rubocop:disable Metrics/ClassLength
   end
 
   # apply
-  def apply_template(data) # rubocop:disable Metrics/MethodLength
+  def apply_template(data, wait: false) # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
     Tempfile.create do |f|
       f.write(data)
       f.rewind
       cmd = "cpln apply #{gvc_org} --file #{f.path}"
+      cmd += " --ready" if wait && ENV.fetch("DISABLE_APPLY_READY", nil).nil?
       if Shell.tmp_stderr
         cmd += " 2> #{Shell.tmp_stderr.path}" if Shell.should_hide_output?
 
@@ -429,8 +430,8 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     end
   end
 
-  def apply_hash(data)
-    apply_template(data.to_yaml)
+  def apply_hash(data, wait: false)
+    apply_template(data.to_yaml, wait: wait)
   end
 
   def parse_apply_result(result) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity

data/lib/core/controlplane_api_direct.rb

@@ -1,5 +1,27 @@
 # frozen_string_literal: true
 
+class RedactedDebugOutput
+  SAFE_HEADERS = %w[Content-Type Content-Length Accept Host Date Cache-Control Connection].freeze
+  HEADER_REGEX = /^([A-Za-z\-]+): (.+)$/.freeze
+
+  def <<(msg)
+    $stdout << redact(msg)
+  end
+
+  private
+
+  def redact(msg)
+    msg.lines.map { |line| redact_line(line) }.join
+  end
+
+  def redact_line(line)
+    match = line.match(HEADER_REGEX)
+    return line.gsub(/[\w\-._]{50,}/, "[REDACTED]") unless match
+
+    SAFE_HEADERS.any? { |h| h.casecmp(match[1]).zero? } ? line : "#{match[1]}: [REDACTED]\n"
+  end
+end
+
 class ControlplaneApiDirect
   API_METHODS = {
     get: Net::HTTP::Get,
@@ -37,7 +59,7 @@ class ControlplaneApiDirect
 
     http = Net::HTTP.new(uri.hostname, uri.port)
     http.use_ssl = uri.scheme == "https"
-    http.set_debug_output($stdout) if trace
+    http.set_debug_output(RedactedDebugOutput.new) if trace
 
     response = http.start { |ht| ht.request(request) }
 

data/lib/core/shell.rb

@@ -5,10 +5,6 @@ class Shell
     attr_reader :tmp_stderr, :verbose
   end
 
-  def self.shell
-    @shell ||= Thor::Shell::Color.new
-  end
-
   def self.use_tmp_stderr
     @tmp_stderr = Tempfile.create
 
@@ -35,6 +31,10 @@ class Shell
     shell.yes?("#{message} (y/N)")
   end
 
+  def self.info(message)
+    shell.say(message)
+  end
+
   def self.warn(message)
     Kernel.warn(color("WARNING: #{message}", :yellow))
   end
@@ -97,4 +97,9 @@ class Shell
       exit(ExitCode::INTERRUPT)
     end
   end
+
+  def self.shell
+    @shell ||= Thor::Shell::Color.new
+  end
+  private_class_method :shell
 end

data/lib/core/terraform_config/agent.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module TerraformConfig
+  class Agent < Base
+    attr_reader :name, :description, :tags
+
+    def initialize(name:, description: nil, tags: nil)
+      super()
+
+      @name = name
+      @description = description
+      @tags = tags
+    end
+
+    def importable?
+      true
+    end
+
+    def reference
+      "cpln_agent.#{name}"
+    end
+
+    def to_tf
+      block :resource, :cpln_agent, name do
+        argument :name, name
+        argument :description, description, optional: true
+        argument :tags, tags, optional: true
+      end
+    end
+  end
+end

data/lib/core/terraform_config/audit_context.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module TerraformConfig
+  class AuditContext < Base
+    attr_reader :name, :description, :tags
+
+    def initialize(name:, description: nil, tags: nil)
+      super()
+
+      @name = name
+      @description = description
+      @tags = tags
+    end
+
+    def importable?
+      true
+    end
+
+    def reference
+      "cpln_audit_context.#{name}"
+    end
+
+    def to_tf
+      block :resource, :cpln_audit_context, name do
+        argument :name, name
+        argument :description, description, optional: true
+        argument :tags, tags, optional: true
+      end
+    end
+  end
+end

data/lib/core/terraform_config/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require_relative "dsl"
+
+module TerraformConfig
+  class Base
+    include Dsl
+
+    def importable?
+      false
+    end
+
+    def reference
+      raise NotImplementedError if importable?
+    end
+
+    def to_tf
+      raise NotImplementedError
+    end
+
+    def locals
+      {}
+    end
+  end
+end

data/lib/core/terraform_config/dsl.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module TerraformConfig
+  module Dsl
+    extend Forwardable
+
+    EXPRESSION_PATTERN = /(var|local|cpln_\w+)\./.freeze
+
+    def_delegators :current_context, :put, :output
+
+    def block(name, *labels)
+      switch_context do
+        put("#{block_declaration(name, labels)} {\n")
+        yield
+        put("}\n")
+      end
+
+      # There is extra indent for whole output that needs to be removed
+      output.unindent
+    end
+
+    def argument(name, value, optional: false, raw: false)
+      return if value.nil? && optional
+
+      content =
+        if value.is_a?(Hash)
+          operator = raw ? ": " : " = "
+          "{\n#{value.map { |n, v| "#{n}#{operator}#{tf_value(v)}" }.join("\n").indent(2)}\n}\n"
+        else
+          "#{tf_value(value)}\n"
+        end
+
+      # remove quotes from expression values
+      content = content.gsub(/("#{EXPRESSION_PATTERN}.*")/) { ::Regexp.last_match(1)[1...-1] }
+
+      put("#{name} = #{content}", indent: 2)
+    end
+
+    private
+
+    def tf_value(value)
+      value = value.to_s if value.is_a?(Symbol)
+
+      case value
+      when String
+        tf_string_value(value)
+      when Hash
+        tf_hash_value(value)
+      else
+        value
+      end
+    end
+
+    def tf_string_value(value)
+      return value if expression?(value)
+      return "\"#{value}\"" unless value.include?("\n")
+
+      "EOF\n#{value.indent(2)}\nEOF"
+    end
+
+    def tf_hash_value(value)
+      JSON.pretty_generate(value.crush)
+          .gsub(/"(\w+)":/) { "#{::Regexp.last_match(1)}:" } # remove quotes from keys
+    end
+
+    def expression?(value)
+      value.match?(/^#{EXPRESSION_PATTERN}/)
+    end
+
+    def block_declaration(name, labels)
+      result = name.to_s
+      return result unless labels.any?
+
+      result + " #{labels.map { |label| tf_value(label) }.join(' ')}"
+    end
+
+    class Context
+      attr_accessor :output
+
+      def initialize
+        @output = ""
+      end
+
+      def put(content, indent: 0)
+        @output += content.to_s.indent(indent)
+      end
+    end
+
+    def switch_context
+      old_context = current_context
+      @current_context = Context.new
+      yield
+    ensure
+      old_context.put(current_context.output, indent: 2)
+      @current_context = old_context
+    end
+
+    def current_context
+      @current_context ||= Context.new
+    end
+  end
+end