cpl 2.0.2 → 2.2.0

data/lib/command/run.rb

@@ -36,12 +36,17 @@ module Command
       - - log async fetching for non-interactive mode
       - The Dockerfile entrypoint is used as the command by default, which assumes `exec "${@}"` to be present,
         and the args ["bash", "-c", cmd_to_run] are passed
-      - The entrypoint can be overriden through `--entrypoint`, which must be a single command or a script path that exists in the container,
+      - The entrypoint can be overridden through `--entrypoint`, which must be a single command or a script path that exists in the container,
         and the args ["bash", "-c", cmd_to_run] are passed,
         unless the entrypoint is `bash`, in which case the args ["-c", cmd_to_run] are passed
       - Providing `--entrypoint none` sets the entrypoint to `bash` by default
       - If `fix_terminal_size` is `true` in the `.controlplane/controlplane.yml` file,
-        the remote terminal size will be fixed to match the local terminal size (may also be overriden through `--terminal-size`)
+        the remote terminal size will be fixed to match the local terminal size (may also be overridden through `--terminal-size`)
+      - By default, all jobs use a CPU size of 1 (1 core) and a memory size of 2Gi (2 gibibytes)
+        (can be configured through `runner_job_default_cpu` and `runner_job_default_memory` in `controlplane.yml`,
+        and also overridden per job through `--cpu` and `--memory`)
+      - By default, the job is stopped if it takes longer than 6 hours to finish
+        (can be configured though `runner_job_timeout` in `controlplane.yml`)
     DESC
     EXAMPLES = <<~EX
       ```sh
@@ -84,12 +89,17 @@ module Command
       ```
     EX
 
+    DEFAULT_JOB_CPU = "1"
+    DEFAULT_JOB_MEMORY = "2Gi"
+    DEFAULT_JOB_TIMEOUT = 21_600 # 6 hours
+    DEFAULT_JOB_HISTORY_LIMIT = 10
     MAGIC_END = "---cpl run command finished---"
 
     attr_reader :interactive, :detached, :location, :original_workload, :runner_workload,
+                :default_image, :default_cpu, :default_memory, :job_timeout, :job_history_limit,
                 :container, :expected_deployed_version, :job, :replica, :command
 
-    def call # rubocop:disable Metrics/MethodLength
+    def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
       @interactive = config.options[:interactive] || interactive_command?
       @detached = config.options[:detached]
       @log_method = config.options[:log_method]
@@ -97,6 +107,11 @@ module Command
       @location = config.location
       @original_workload = config.options[:workload] || config[:one_off_workload]
       @runner_workload = "#{original_workload}-runner"
+      @default_image = "#{config.app}:#{Controlplane::NO_IMAGE_AVAILABLE}"
+      @default_cpu = config.current[:runner_job_default_cpu] || DEFAULT_JOB_CPU
+      @default_memory = config.current[:runner_job_default_memory] || DEFAULT_JOB_MEMORY
+      @job_timeout = config.current[:runner_job_timeout] || DEFAULT_JOB_TIMEOUT
+      @job_history_limit = DEFAULT_JOB_HISTORY_LIMIT
 
       unless interactive
         @internal_sigint = false
@@ -110,12 +125,10 @@ module Command
         end
       end
 
-      if cp.fetch_workload(runner_workload).nil?
-        create_runner_workload
-        wait_for_runner_workload_create
-      end
+      create_runner_workload if cp.fetch_workload(runner_workload).nil?
+      wait_for_runner_workload_deploy
       update_runner_workload
-      wait_for_runner_workload_update
+      wait_for_runner_workload_update if expected_deployed_version
 
       start_job
       wait_for_replica_for_job
@@ -154,6 +167,11 @@ module Command
         container_spec.delete("livenessProbe")
         container_spec.delete("readinessProbe")
 
+        # Set image, CPU, and memory to default values
+        container_spec["image"] = default_image
+        container_spec["cpu"] = default_cpu
+        container_spec["memory"] = default_memory
+
         # Ensure cron workload won't run per schedule
         spec["defaultOptions"]["suspend"] = true
 
@@ -163,9 +181,14 @@ module Command
 
         # Set cron job props
         spec["type"] = "cron"
+        spec["job"] = {
+          # Next job set to run on January 1st, 2029
+          "schedule" => "0 0 1 1 1",
 
-        # Next job set to run on January 1st, 2029
-        spec["job"] = { "schedule" => "0 0 1 1 1", "restartPolicy" => "Never" }
+          "restartPolicy" => "Never",
+          "activeDeadlineSeconds" => job_timeout,
+          "historyLimit" => job_history_limit
+        }
 
         # Create runner workload
         cp.apply_hash("kind" => "workload", "name" => runner_workload, "spec" => spec)
@@ -173,46 +196,60 @@ module Command
     end
 
     def update_runner_workload # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-      step("Updating runner workload '#{runner_workload}' based on '#{original_workload}'") do # rubocop:disable Metrics/BlockLength
-        @expected_deployed_version = cp.cron_workload_deployed_version(runner_workload)
-        should_update = false
+      should_update = false
+      spec = nil
 
+      step("Checking if runner workload '#{runner_workload}' needs to be updated") do # rubocop:disable Metrics/BlockLength
         _, original_container_spec = base_workload_specs(original_workload)
         spec, container_spec = base_workload_specs(runner_workload)
 
-        # Override image if specified
-        image = config.options[:image]
-        image_link = if image
-                       image = latest_image if image == "latest"
-                       "/org/#{config.org}/image/#{image}"
-                     else
-                       original_container_spec["image"]
-                     end
-        if container_spec["image"] != image_link
-          container_spec["image"] = image_link
+        # Keep ENV synced between original and runner workloads
+        original_env_str = original_container_spec["env"]&.sort_by { |env| env["name"] }.to_s
+        env_str = container_spec["env"]&.sort_by { |env| env["name"] }.to_s
+        if original_env_str != env_str
+          container_spec["env"] = original_container_spec["env"]
+          should_update = true
+        end
+
+        if container_spec["image"] != default_image
+          container_spec["image"] = default_image
           should_update = true
         end
 
-        # Container overrides
-        if config.options[:cpu] && container_spec["cpu"] != config.options[:cpu]
-          container_spec["cpu"] = config.options[:cpu]
+        if container_spec["cpu"] != default_cpu
+          container_spec["cpu"] = default_cpu
           should_update = true
         end
-        if config.options[:memory] && container_spec["memory"] != config.options[:memory]
-          container_spec["memory"] = config.options[:memory]
+
+        if container_spec["memory"] != default_memory
+          container_spec["memory"] = default_memory
           should_update = true
         end
 
-        next true unless should_update
+        if spec["job"]["activeDeadlineSeconds"] != job_timeout
+          spec["job"]["activeDeadlineSeconds"] = job_timeout
+          should_update = true
+        end
 
+        if spec["job"]["historyLimit"] != job_history_limit
+          spec["job"]["historyLimit"] = job_history_limit
+          should_update = true
+        end
+
+        true
+      end
+
+      return unless should_update
+
+      step("Updating runner workload '#{runner_workload}'") do
         # Update runner workload
-        @expected_deployed_version += 1
+        @expected_deployed_version = cp.cron_workload_deployed_version(runner_workload) + 1
         cp.apply_hash("kind" => "workload", "name" => runner_workload, "spec" => spec)
       end
     end
 
-    def wait_for_runner_workload_create
-      step("Waiting for runner workload '#{runner_workload}' to be created", retry_on_failure: true) do
+    def wait_for_runner_workload_deploy
+      step("Waiting for runner workload '#{runner_workload}' to be deployed", retry_on_failure: true) do
         !cp.cron_workload_deployed_version(runner_workload).nil?
       end
     end
@@ -302,12 +339,14 @@ module Command
     def base_workload_specs(workload)
       spec = cp.fetch_workload!(workload).fetch("spec")
       container_spec = spec["containers"].detect { _1["name"] == original_workload } || spec["containers"].first
-      @container = container_spec["name"]
 
       [spec, container_spec]
     end
 
     def build_job_start_yaml # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
+      _, original_container_spec = base_workload_specs(original_workload)
+      @container = original_container_spec["name"]
+
       job_start_hash = { "name" => container }
 
       if config.options[:use_local_token]
@@ -335,6 +374,18 @@ module Command
         job_start_hash["args"].push('eval "$CPL_RUNNER_SCRIPT"')
       end
 
+      image = config.options[:image]
+      image_link = if image
+                     image = cp.latest_image if image == "latest"
+                     "/org/#{config.org}/image/#{image}"
+                   else
+                     original_container_spec["image"]
+                   end
+
+      job_start_hash["image"] = image_link
+      job_start_hash["cpu"] = config.options[:cpu] if config.options[:cpu]
+      job_start_hash["memory"] = config.options[:memory] if config.options[:memory]
+
       job_start_hash.to_yaml
     end
 
@@ -434,6 +485,8 @@ module Command
     end
 
     def print_detached_commands
+      return unless replica
+
       app_workload_replica_config = app_workload_replica_args.join(" ")
       progress.puts(
         "\n\n" \
@@ -451,7 +504,7 @@ module Command
         Shell.debug("JOB STATUS", status)
 
         case status
-        when "active"
+        when "active", "pending"
           sleep 1
         when "successful"
           break ExitCode::SUCCESS

data/lib/command/setup_app.rb

@@ -5,18 +5,27 @@ module Command
     NAME = "setup-app"
     OPTIONS = [
       app_option(required: true),
-      skip_secret_access_binding_option
+      skip_secret_access_binding_option,
+      skip_secrets_setup_option,
+      skip_post_creation_hook_option
     ].freeze
     DESCRIPTION = "Creates an app and all its workloads"
     LONG_DESCRIPTION = <<~DESC
       - Creates an app and all its workloads
       - Specify the templates for the app and workloads through `setup_app_templates` in the `.controlplane/controlplane.yml` file
-      - This should only be used for temporary apps like review apps, never for persistent apps like production (to update workloads for those, use 'cpl apply-template' instead)
-      - Automatically binds the app to the secrets policy, as long as both the identity and the policy exist
-      - Use `--skip-secret-access-binding` to prevent the automatic bind
+      - This should only be used for temporary apps like review apps, never for persistent apps like production or staging (to update workloads for those, use 'cpl apply-template' instead)
+      - Configures app to have org-level secrets with default name "{APP_PREFIX}-secrets"
+        using org-level policy with default name "{APP_PREFIX}-secrets-policy" (names can be customized, see docs)
+      - Creates identity for secrets if it does not exist
+      - Use `--skip-secrets-setup` to prevent the automatic setup of secrets,
+        or set it through `skip_secrets_setup` in the `.controlplane/controlplane.yml` file
+      - Runs a post-creation hook after the app is created if `hooks.post_creation` is specified in the `.controlplane/controlplane.yml` file
+      - If the hook exits with a non-zero code, the command will stop executing and also exit with a non-zero code
+      - Use `--skip-post-creation-hook` to skip the hook if specified in `controlplane.yml`
     DESC
+    VALIDATIONS = %w[config templates].freeze
 
-    def call # rubocop:disable Metrics/MethodLength
+    def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength
       templates = config[:setup_app_templates]
 
       app = cp.fetch_gvc
@@ -26,24 +35,79 @@ module Command
               "or run 'cpl apply-template #{templates.join(' ')} -a #{config.app}'."
       end
 
-      Cpl::Cli.start(["apply-template", *templates, "-a", config.app])
+      skip_secrets_setup = config.options[:skip_secret_access_binding] ||
+                           config.options[:skip_secrets_setup] || config.current[:skip_secrets_setup]
 
-      return if config.options[:skip_secret_access_binding]
+      create_secret_and_policy_if_not_exist unless skip_secrets_setup
+
+      args = []
+      args.push("--add-app-identity") unless skip_secrets_setup
+      Cpl::Cli.start(["apply-template", *templates, "-a", config.app, *args])
+
+      bind_identity_to_policy unless skip_secrets_setup
+      run_post_creation_hook unless config.options[:skip_post_creation_hook]
+    end
+
+    private
+
+    def create_secret_and_policy_if_not_exist
+      create_secret_if_not_exists
+      create_policy_if_not_exists
 
       progress.puts
+    end
 
-      if cp.fetch_identity(app_identity).nil? || cp.fetch_policy(app_secrets_policy).nil?
-        raise "Can't bind identity to policy: identity '#{app_identity}' or " \
-              "policy '#{app_secrets_policy}' doesn't exist. " \
-              "Please create them or use `--skip-secret-access-binding` to ignore this message." \
-              "You can also set a custom secrets name with `secrets_name` " \
-              "and a custom secrets policy name with `secrets_policy_name` " \
-              "in the `.controlplane/controlplane.yml` file."
+    def create_secret_if_not_exists
+      if cp.fetch_secret(config.secrets)
+        progress.puts("Secret '#{config.secrets}' already exists. Skipping creation...")
+      else
+        step("Creating secret '#{config.secrets}'") do
+          cp.apply_hash(build_secret_hash)
+        end
       end
+    end
 
-      step("Binding identity to policy") do
-        cp.bind_identity_to_policy(app_identity_link, app_secrets_policy)
+    def create_policy_if_not_exists
+      if cp.fetch_policy(config.secrets_policy)
+        progress.puts("Policy '#{config.secrets_policy}' already exists. Skipping creation...")
+      else
+        step("Creating policy '#{config.secrets_policy}'") do
+          cp.apply_hash(build_policy_hash)
+        end
       end
     end
+
+    def build_secret_hash
+      {
+        "kind" => "secret",
+        "name" => config.secrets,
+        "type" => "dictionary",
+        "data" => {}
+      }
+    end
+
+    def build_policy_hash
+      {
+        "kind" => "policy",
+        "name" => config.secrets_policy,
+        "targetKind" => "secret",
+        "targetLinks" => ["//secret/#{config.secrets}"]
+      }
+    end
+
+    def bind_identity_to_policy
+      progress.puts
+
+      step("Binding identity '#{config.identity}' to policy '#{config.secrets_policy}'") do
+        cp.bind_identity_to_policy(config.identity_link, config.secrets_policy)
+      end
+    end
+
+    def run_post_creation_hook
+      post_creation_hook = config.current.dig(:hooks, :post_creation)
+      return unless post_creation_hook
+
+      run_command_in_latest_image(post_creation_hook, title: "post-creation hook")
+    end
   end
 end

data/lib/command/test.rb

@@ -11,6 +11,7 @@ module Command
       - For debugging purposes
     DESC
     HIDE = true
+    VALIDATIONS = [].freeze
 
     def call
       # Modify this method to trigger the code you want to test.

data/lib/command/version.rb

@@ -9,6 +9,7 @@ module Command
       - Can also be done with `cpl --version` or `cpl -v`
     DESC
     WITH_INFO_HEADER = false
+    VALIDATIONS = [].freeze
 
     def call
       puts Cpl::VERSION

data/lib/core/config.rb

@@ -18,6 +18,8 @@ class Config # rubocop:disable Metrics/ClassLength
 
     ensure_required_options!
 
+    warn_deprecated_options
+
     Shell.verbose_mode(options[:verbose])
     trace_mode = options[:trace]
     return unless trace_mode
@@ -38,10 +40,34 @@ class Config # rubocop:disable Metrics/ClassLength
     current&.fetch(:name)
   end
 
+  def identity
+    "#{app}-identity"
+  end
+
+  def identity_link
+    "/org/#{org}/gvc/#{app}/identity/#{identity}"
+  end
+
+  def secrets
+    current&.dig(:secrets_name) || "#{app_prefix}-secrets"
+  end
+
+  def secrets_policy
+    current&.dig(:secrets_policy_name) || "#{secrets}-policy"
+  end
+
   def location
     @location ||= load_location_from_options || load_location_from_env || load_location_from_file
   end
 
+  def location_link
+    "/org/#{org}/location/#{location}"
+  end
+
+  def image_link(image)
+    "/org/#{org}/image/#{image}"
+  end
+
   def domain
     @domain ||= load_domain_from_options || load_domain_from_file
   end
@@ -84,6 +110,8 @@ class Config # rubocop:disable Metrics/ClassLength
     @apps ||= config[:apps].to_h do |app_name, app_options|
       ensure_config_app!(app_name, app_options)
 
+      check_deprecated_options(app_options)
+
       app_options_with_new_keys = app_options.to_h do |key, value|
         new_key = new_option_keys[key]
         new_key ? [new_key, value] : [key, value]
@@ -96,14 +124,7 @@ class Config # rubocop:disable Metrics/ClassLength
   def current
     return unless app
 
-    @current ||= begin
-      app_config = find_app_config(app)
-      ensure_config_app!(app, app_config)
-
-      warn_deprecated_options(app_config)
-
-      app_config
-    end
+    @current ||= find_app_config(app)
   end
 
   def app_matches?(app_name1, app_name2, app_options)
@@ -275,11 +296,18 @@ class Config # rubocop:disable Metrics/ClassLength
     strip_str_and_validate(current.fetch(:default_domain))
   end
 
-  def warn_deprecated_options(app_options)
-    deprecated_option_keys = new_option_keys.select { |old_key| app_options.key?(old_key) }
-    return if deprecated_option_keys.empty?
+  def check_deprecated_options(app_options)
+    @deprecated_option_keys ||= {}
+
+    new_option_keys.each do |old_key, new_key|
+      @deprecated_option_keys[old_key] = new_key if app_options.key?(old_key)
+    end
+  end
+
+  def warn_deprecated_options
+    return if !@deprecated_option_keys || @deprecated_option_keys.empty?
 
-    deprecated_option_keys.each do |old_key, new_key|
+    @deprecated_option_keys.each do |old_key, new_key|
       Shell.warn_deprecated("Option '#{old_key}' is deprecated, " \
                             "please use '#{new_key}' instead (in 'controlplane.yml').")
     end

data/lib/core/controlplane.rb

@@ -3,6 +3,8 @@
 class Controlplane # rubocop:disable Metrics/ClassLength
   attr_reader :config, :api, :gvc, :org
 
+  NO_IMAGE_AVAILABLE = "NO_IMAGE_AVAILABLE"
+
   def initialize(config)
     @config = config
     @api = ControlplaneApi.new
@@ -37,6 +39,51 @@ class Controlplane # rubocop:disable Metrics/ClassLength
 
   # image
 
+  def latest_image(a_gvc = gvc, a_org = org, refresh: false)
+    @latest_image ||= {}
+    @latest_image[a_gvc] = nil if refresh
+    @latest_image[a_gvc] ||=
+      begin
+        items = query_images(a_gvc, a_org)["items"]
+        latest_image_from(items, app_name: a_gvc)
+      end
+  end
+
+  def latest_image_next(a_gvc = gvc, a_org = org, commit: nil)
+    commit ||= config.options[:commit]
+
+    @latest_image_next ||= {}
+    @latest_image_next[a_gvc] ||= begin
+      latest_image_name = latest_image(a_gvc, a_org)
+      image = latest_image_name.split(":").first
+      image += ":#{extract_image_number(latest_image_name) + 1}"
+      image += "_#{commit}" if commit
+      image
+    end
+  end
+
+  def latest_image_from(items, app_name: gvc, name_only: true)
+    matching_items = items.select { |item| item["name"].start_with?("#{app_name}:") }
+
+    # Or special string to indicate no image available
+    if matching_items.empty?
+      name_only ? "#{app_name}:#{NO_IMAGE_AVAILABLE}" : nil
+    else
+      latest_item = matching_items.max_by { |item| extract_image_number(item["name"]) }
+      name_only ? latest_item["name"] : latest_item
+    end
+  end
+
+  def extract_image_number(image_name)
+    return 0 if image_name.end_with?(NO_IMAGE_AVAILABLE)
+
+    image_name.match(/:(\d+)/)&.captures&.first.to_i
+  end
+
+  def extract_image_commit(image_name)
+    image_name.match(/_(\h+)$/)&.captures&.first
+  end
+
   def query_images(a_gvc = gvc, a_org = org, partial_gvc_match: nil)
     partial_gvc_match = config.should_app_start_with?(a_gvc) if partial_gvc_match.nil?
     gvc_op = partial_gvc_match ? "~" : "="
@@ -209,6 +256,11 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     api.update_workload(org: org, gvc: gvc, workload: workload, data: data)
   end
 
+  def workload_suspended?(workload)
+    details = fetch_workload!(workload)
+    details["spec"]["defaultOptions"]["suspend"]
+  end
+
   def workload_force_redeployment(workload)
     cmd = "cpln workload force-redeployment #{workload} #{gvc_org}"
     perform!(cmd)
@@ -324,6 +376,12 @@ class Controlplane # rubocop:disable Metrics/ClassLength
     api.log_get(org: org, gvc: gvc, workload: workload, replica: replica, from: from, to: to)
   end
 
+  # secrets
+
+  def fetch_secret(secret)
+    api.fetch_secret(org: org, secret: secret)
+  end
+
   # identities
 
   def fetch_identity(identity, a_gvc = gvc)

data/lib/core/controlplane_api.rb

@@ -52,7 +52,7 @@ class ControlplaneApi # rubocop:disable Metrics/ClassLength
     # params << "direction=forward"
     params = params.map { |k, v| %(#{k}=#{CGI.escape(v)}) }.join("&")
 
-    api_json_direct("/logs/org/#{org}/loki/api/v1/query_range?#{params}", method: :get, host: :logs)
+    api_json("/logs/org/#{org}/loki/api/v1/query_range?#{params}", method: :get, host: :logs)
   end
 
   def query_workloads(org:, gvc:, workload:, gvc_op_type:, workload_op_type:) # rubocop:disable Metrics/MethodLength
@@ -116,6 +116,14 @@ class ControlplaneApi # rubocop:disable Metrics/ClassLength
     api_json("/org/#{org}/domain/#{domain}", method: :patch, body: data)
   end
 
+  def fetch_secret(org:, secret:)
+    api_json("/org/#{org}/secret/#{secret}", method: :get)
+  end
+
+  def delete_secret(org:, secret:)
+    api_json("/org/#{org}/secret/#{secret}", method: :delete)
+  end
+
   def fetch_identity(org:, gvc:, identity:)
     api_json("/org/#{org}/gvc/#{gvc}/identity/#{identity}", method: :get)
   end
@@ -124,6 +132,10 @@ class ControlplaneApi # rubocop:disable Metrics/ClassLength
     api_json("/org/#{org}/policy/#{policy}", method: :get)
   end
 
+  def delete_policy(org:, policy:)
+    api_json("/org/#{org}/policy/#{policy}", method: :delete)
+  end
+
   private
 
   def fetch_query_pages(result)
@@ -152,13 +164,7 @@ class ControlplaneApi # rubocop:disable Metrics/ClassLength
     result
   end
 
-  # switch between cpln rest and api
   def api_json(...)
     ControlplaneApiDirect.new.call(...)
   end
-
-  # only for api (where not impelemented in cpln rest)
-  def api_json_direct(...)
-    ControlplaneApiDirect.new.call(...)
-  end
 end

data/lib/core/controlplane_api_direct.rb

@@ -98,7 +98,7 @@ class ControlplaneApiDirect
   end
 
   def refresh_api_token
-    @@api_token[:token] = `cpln profile token`.chomp
+    @@api_token[:token] = Shell.cmd("cpln", "profile", "token")[:output].chomp
   end
 
   def self.reset_api_token