cpflow 5.3.0 → 6.0.0

data/lib/command/delete.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../core/disposable_postgres_helpers"
+
 module Command
   class Delete < Base # rubocop:disable Metrics/ClassLength
     NAME = "delete"
@@ -14,6 +16,7 @@ module Command
       - Deletes the whole app (GVC with all workloads, all volumesets and all images) or a specific workload
       - Also unbinds the app from the secrets policy and any configured `shared_secret_grants` policies, as long as both the identity and each policy exist (and are bound)
       - For the app-specific secrets policy, removes every permission held by the app identity; for `shared_secret_grants`, removes only `reveal`
+      - Removes a marked per-app dictionary and exact-target unbound policy for generated review credentials, including after the opt-in is removed from a dynamically matched review-app entry
       - Will ask for explicit user confirmation
       - Runs a pre-deletion hook before the app is deleted if `hooks.pre_deletion` is specified in the `.controlplane/controlplane.yml` file
       - If the hook exits with a non-zero code, the command will stop executing and also exit with a non-zero code
@@ -50,8 +53,10 @@ module Command
     end
 
     def delete_whole_app
-      return progress.puts("App '#{config.app}' does not exist.") if cp.fetch_gvc.nil?
+      return handle_missing_app if cp.fetch_gvc.nil?
 
+      postgres_helpers.validate!
+      validate_disposable_review_secret_resources!
       check_volumesets
       check_images
       return unless confirm_delete(config.app)
@@ -61,11 +66,104 @@ module Command
       policy_unbinds = secret_policy_unbinds
       run_pre_deletion_hook unless config.options[:skip_pre_deletion_hook]
       unbind_identity_from_policy(policy_unbinds)
+      delete_app_resources
+    end
+
+    def handle_missing_app
+      progress.puts("App '#{config.app}' does not exist.")
+      cleanup_state = validate_disposable_review_secret_resources!
+      postgres_resources = postgres_helpers.validate!
+      return if cleanup_state == :none && postgres_resources.empty?
+      return unless confirm_delete("disposable secrets for app #{config.app}")
+
+      unbind_missing_app_disposable_policy! if cleanup_state == :own_binding
+      delete_generated_review_secret_resources
+      delete_postgres_helpers
+    end
+
+    def validate_disposable_review_secret_resources!
+      cleanup_state = disposable_review_secret_cleanup_state
+      if cleanup_state == :unsafe
+        raise "Review app secret resources have unexpected ownership, grants, or target; leaving them for inspection."
+      end
+
+      cleanup_state
+    end
+
+    def disposable_review_secret_cleanup_state
+      names = config.disposable_review_secret_resource_names
+      return :none unless names
+
+      secret_name, policy_name = names
+      policy = cp.fetch_policy(policy_name)
+      secret = cp.fetch_secret(secret_name)
+      return :none if policy.nil? && secret.nil?
+
+      classify_disposable_review_secret_resources(policy, secret, secret_name)
+    end
+
+    def classify_disposable_review_secret_resources(policy, secret, secret_name)
+      return generated_review_secret_owned_by_app?(secret, secret_name) ? :safe : :unsafe if policy.nil?
+      return :unsafe unless generated_review_secret_policy?(policy, secret_name)
+      return :unsafe unless generated_review_secret_or_absent?(secret, secret_name)
+      return :safe if unbound_review_policy?(policy)
+
+      only_app_identity_bindings?(policy) ? :own_binding : :unsafe
+    end
+
+    def generated_review_secret_or_absent?(secret, secret_name)
+      secret.nil? || generated_review_secret_owned_by_app?(secret, secret_name)
+    end
+
+    def only_app_identity_bindings?(policy)
+      generated_review_policy_binding_state(policy) == :bound
+    end
+
+    def unbound_review_policy?(policy)
+      generated_review_policy_binding_state(policy) == :unbound
+    end
+
+    def unbind_missing_app_disposable_policy!
+      secret_name, policy_name = config.disposable_review_secret_resource_names
+      policy = cp.fetch_policy(policy_name)
+      return if policy.nil?
+
+      unless generated_review_secret_policy?(policy, secret_name) &&
+             (unbound_review_policy?(policy) || only_app_identity_bindings?(policy))
+        refuse_unsafe_review_secret_resources!
+      end
+
+      remove_missing_app_policy_bindings(policy, policy_name)
+    end
+
+    def remove_missing_app_policy_bindings(policy, policy_name)
+      identity_policy_permissions(policy).each do |permission|
+        step("Unbinding deleted app identity from disposable review app policy") do
+          cp.unbind_identity_from_policy(config.identity_link, policy_name, permission: permission)
+        end
+      end
+    end
+
+    def delete_app_resources
       delete_volumesets
+      # Keep the GVC discoverable by stale-app cleanup if disposable resource deletion fails.
+      # The app identity has already been unbound from its secret policy.
+      delete_generated_review_secret_resources
+      delete_postgres_helpers
       delete_gvc
       delete_images
     end
 
+    def postgres_helpers
+      @postgres_helpers ||= DisposablePostgresHelpers.new(config, cp)
+    end
+
+    def delete_postgres_helpers
+      postgres_helpers.delete do |kind, name, &operation|
+        step("Deleting disposable PostgreSQL #{kind} '#{name}'", &operation)
+      end
+    end
+
     def check_volumesets
       @volumesets = cp.fetch_volumesets["items"]
       return progress.puts("No volumesets to delete from app '#{config.app}'.") unless @volumesets.any?
@@ -103,6 +201,53 @@ module Command
       end
     end
 
+    def delete_generated_review_secret_resources
+      names = config.disposable_review_secret_resource_names
+      return unless names
+
+      secret_name, policy_name = names
+
+      policy = cp.fetch_policy(policy_name)
+      secret = cp.fetch_secret(secret_name)
+      return delete_generated_review_secret_without_policy(secret, secret_name) if policy.nil?
+
+      delete_generated_review_secret_with_policy(policy, secret, secret_name, policy_name)
+    end
+
+    def delete_generated_review_secret_without_policy(secret, secret_name)
+      return if secret.nil?
+
+      refuse_unsafe_review_secret_resources! unless generated_review_secret_owned_by_app?(secret, secret_name)
+
+      step("Deleting orphaned disposable review app secret dictionary") { cp.delete_secret(secret_name) }
+    end
+
+    def delete_generated_review_secret_with_policy(policy, secret, secret_name, policy_name)
+      refuse_unsafe_review_secret_resources! unless disposable_review_secret_policy?(policy, secret_name)
+
+      if secret
+        refuse_unsafe_review_secret_resources! unless generated_review_secret_owned_by_app?(secret, secret_name)
+
+        step("Deleting disposable review app secret dictionary") { cp.delete_secret(secret_name) }
+      end
+      step("Deleting disposable review app secret policy") { cp.delete_policy(policy_name) }
+    end
+
+    def refuse_unsafe_review_secret_resources!
+      raise "Review app secret resources have unexpected ownership, grants, or target; " \
+            "leaving secret resources for inspection."
+    end
+
+    def disposable_review_secret_policy?(policy, secret_name)
+      generated_review_secret_policy?(policy, secret_name) && unbound_review_policy?(policy)
+    end
+
+    def generated_review_secret_policy?(policy, secret_name)
+      policy_targets_secret?(policy, secret_name) &&
+        generated_review_app_tag(policy) == config.app &&
+        %w[target targetQuery gvc].all? { |key| policy[key].nil? }
+    end
+
     def delete_workload(workload)
       step("Deleting workload '#{workload}' from app '#{config.app}'") do
         cp.delete_workload(workload)
@@ -136,10 +281,13 @@ module Command
     end
 
     def secret_policy_unbinds
-      return [] if cp.fetch_identity(config.identity).nil?
+      unless cp.fetch_identity(config.identity)
+        return [disposable_review_secret_policy_unbind(include_current: true)].compact
+      end
 
       [
         app_secret_policy_unbind,
+        disposable_review_secret_policy_unbind,
         *shared_secret_policy_unbinds
       ].compact
     end
@@ -151,6 +299,28 @@ module Command
       )
     end
 
+    def disposable_review_secret_policy_unbind(include_current: false)
+      names = config.disposable_review_secret_resource_names
+      return unless names && (include_current || names.last != config.secrets_policy)
+
+      secret_name, policy_name = names
+      policy = verified_disposable_review_secret_policy(secret_name, policy_name)
+      return unless policy
+
+      policy_unbind_for(policy_name, "Unbinding identity from disposable review app policy '#{policy_name}'", policy)
+    end
+
+    def verified_disposable_review_secret_policy(secret_name, policy_name)
+      secret = cp.fetch_secret(secret_name)
+      return if secret && !generated_review_secret_owned_by_app?(secret, secret_name)
+
+      policy = cp.fetch_policy(policy_name)
+      return unless policy && generated_review_secret_policy?(policy, secret_name) &&
+                    (unbound_review_policy?(policy) || only_app_identity_bindings?(policy))
+
+      policy
+    end
+
     def shared_secret_policy_unbinds
       config.shared_secret_grants.filter_map do |grant|
         shared_secret_policy_unbind(grant)
@@ -190,8 +360,8 @@ module Command
       )
     end
 
-    def policy_unbind_for(policy_name, message)
-      policy = cp.fetch_policy(policy_name)
+    def policy_unbind_for(policy_name, message, policy = nil)
+      policy ||= cp.fetch_policy(policy_name)
       return if policy.nil?
 
       permissions = identity_policy_permissions(policy)

data/lib/command/deploy_image.rb

@@ -4,6 +4,12 @@ require "resolv"
 
 module Command
   class DeployImage < Base # rubocop:disable Metrics/ClassLength
+    WORKLOAD_IMAGE_UPDATE_MAX_ATTEMPTS = 30
+    # Preserve the general failure cap; only optimistic-concurrency conflicts get the longer convergence window.
+    WORKLOAD_IMAGE_UPDATE_CONFLICT_MAX_ATTEMPTS = 120
+    WORKLOAD_IMAGE_UPDATE_CONFLICT_PATTERN =
+      /(?:\b409\b|(?:\A|\n)[ \t]*(?:error:[ \t]*)?conflict[ \t]*(?:\r?\n|\z))/i
+
     NAME = "deploy-image"
     OPTIONS = [
       app_option(required: true),
@@ -122,7 +128,8 @@ module Command
 
           container_name = container["name"]
           step("Deploying image '#{image}' for workload '#{workload}'") do
-            cp.workload_set_image_ref(workload, container: container_name, image: image)
+            update_workload_image_ref(workload, container_name, image)
+
             deployed_endpoints[workload] = endpoint_for_workload(workload_data)
             # A missing public endpoint is valid; the image update still completed successfully.
             true
@@ -136,6 +143,42 @@ module Command
       deployed_endpoints
     end
 
+    def update_workload_image_ref(workload, container, image)
+      attempts = 0
+      mark_deployment_started
+
+      loop do
+        attempts += 1
+        result = cp.workload_set_image_ref(workload, container: container, image: image)
+        return true if result.fetch(:success)
+
+        output = result.fetch(:output).to_s
+        raise workload_image_update_error(output) if attempts >= workload_image_update_limit(output)
+
+        wait_before_workload_image_update_retry
+      end
+    end
+
+    def mark_deployment_started
+      marker_path = ENV.fetch("CPFLOW_DEPLOYMENT_STARTED_FILE", "")
+      File.write(marker_path, "started\n") unless marker_path.empty?
+    end
+
+    def workload_image_update_limit(output)
+      return WORKLOAD_IMAGE_UPDATE_CONFLICT_MAX_ATTEMPTS if output.match?(WORKLOAD_IMAGE_UPDATE_CONFLICT_PATTERN)
+
+      WORKLOAD_IMAGE_UPDATE_MAX_ATTEMPTS
+    end
+
+    def workload_image_update_error(output)
+      output.strip.empty? ? "Command exited with non-zero status." : output
+    end
+
+    def wait_before_workload_image_update_retry
+      progress.print(".")
+      Kernel.sleep(1)
+    end
+
     def print_deployed_endpoints(deployed_endpoints)
       progress.puts("\nDeployed endpoints:")
       deployed_endpoints.each do |workload, endpoint|

data/lib/command/generate.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "yaml"
+require "pathname"
+require "shellwords"
 
 require_relative "generator_helpers"
 require_relative "../core/repo_introspection"
@@ -34,10 +36,29 @@ module Command
     # `.tool-versions`, or the `Gemfile`. Keep this on a supported release line
     # (https://www.ruby-lang.org/en/downloads/branches/).
     DEFAULT_RUBY_VERSION = "3.3"
+    SQLITE_DATABASE_PREPARE_FUNCTION = <<~SH
+      prepare_sqlite_database() {
+        source_path="$1"
+        persistent_path="$2"
+        legacy_path="${3:-}"
+        if [ -n "${legacy_path}" ] && [ -e "${legacy_path}" ] && [ ! -e "${persistent_path}" ]; then
+          persistent_path="${legacy_path}"
+        fi
+        mkdir -p "$(dirname "${source_path}")" "$(dirname "${persistent_path}")"
+        if [ -e "${source_path}" ] && [ ! -L "${source_path}" ] && [ ! -e "${persistent_path}" ]; then
+          mv -n "${source_path}" "${persistent_path}"
+        fi
+        rm -f "${source_path}"
+        ln -s "${persistent_path}" "${source_path}"
+      }
+    SH
 
     def copy_files
+      validate_sqlite_database_paths!
       generated_paths = copy_template_files("generator_templates", base_template_files)
       generated_paths += copy_template_files("generator_templates_sqlite", SQLITE_TEMPLATE_FILES) if sqlite_project?
+      copy_dockerignore unless File.exist?(".dockerignore")
+      append_dockerignore_entries
       substitute_template_variables(generated_paths)
       make_shell_scripts_executable(generated_paths)
     end
@@ -64,6 +85,78 @@ module Command
       destination_path
     end
 
+    def copy_dockerignore
+      copy_file(
+        File.join("generator_templates", ".dockerignore"),
+        ".dockerignore",
+        verbose: ENV.fetch("HIDE_COMMAND_OUTPUT", nil) != "true"
+      )
+    end
+
+    def append_dockerignore_entries
+      contents = File.read(".dockerignore")
+      existing_lines = contents.lines(chomp: true)
+      additions = dockerignore_additions(existing_lines)
+      return if additions.empty?
+
+      File.open(".dockerignore", "a") do |file|
+        file.write("\n") unless contents.empty? || contents.end_with?("\n")
+        file.puts(additions)
+      end
+    end
+
+    def dockerignore_additions(existing_lines)
+      entries = File.readlines(File.join(self.class.source_root, "generator_templates", ".dockerignore"), chomp: true)
+      sqlite_entries = sqlite_project? ? sqlite_database_ignore_entries : []
+      entries += sqlite_entries
+      env_entries = [".env*", "!.env.example"]
+      mandatory_exclusions = ["config/master.key", "config/credentials/*.key", *sqlite_entries]
+      ordinary_entries = entries - env_entries - mandatory_exclusions
+      additions = ordinary_entries - existing_lines
+      additions += mandatory_exclusions.reject { |entry| dockerignore_exclusion_effective?(existing_lines, entry) }
+      additions += env_entries unless dockerignore_env_exception_preserved?(existing_lines)
+      additions
+    end
+
+    def dockerignore_exclusion_effective?(lines, exclusion)
+      exclusion_index = lines.rindex(exclusion)
+      exclusion_index && lines[(exclusion_index + 1)..].none? { |line| line.start_with?("!") }
+    end
+
+    def dockerignore_env_exception_preserved?(lines)
+      exclusion_index = lines.rindex(".env*")
+      exception_index = lines.rindex("!.env.example")
+      return false unless exclusion_index && exception_index && exception_index > exclusion_index
+
+      lines[(exception_index + 1)..].none? { |line| line.start_with?("!") }
+    end
+
+    def sqlite_database_ignore_entries
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).flat_map do |database_path|
+        relative = docker_context_database_path(database_path)
+        next [] unless relative
+
+        base = "/#{escape_dockerignore_path(relative)}"
+        [base, "#{base}-wal", "#{base}-shm", "#{base}-journal"]
+      end.uniq
+    end
+
+    def escape_dockerignore_path(path)
+      path.gsub(/[\\*?\[\]]/) { |character| "\\#{character}" }
+    end
+
+    def docker_context_database_path(database_path)
+      path = Pathname.new(database_path).cleanpath
+      if path.absolute?
+        return unless path.to_s.start_with?("/app/")
+
+        return path.to_s.delete_prefix("/app/")
+      end
+
+      normalized = path.to_s
+      normalized unless normalized == "." || normalized.start_with?("../")
+    end
+
     def base_template_files
       COMMON_TEMPLATE_FILES + (sqlite_project? ? [] : POSTGRES_TEMPLATE_FILES)
     end
@@ -72,7 +165,8 @@ module Command
       {
         "__APP_PREFIX__" => inferred_app_prefix,
         "__RUBY_VERSION__" => inferred_ruby_version,
-        "__ASSET_PRECOMPILE_HOOK_RUN__" => asset_precompile_hook_run
+        "__ASSET_PRECOMPILE_HOOK_RUN__" => asset_precompile_hook_run,
+        "__SQLITE_DATABASE_SETUP__" => sqlite_database_setup
       }
     end
 
@@ -99,7 +193,7 @@ module Command
       return "" if stripped.empty?
       return "" unless single_line_asset_precompile_hook?(stripped)
 
-      "RUN #{stripped}\n\n"
+      "RUN export SECRET_KEY_BASE=NOT_USED_NON_BLANK && #{stripped}\n\n"
     end
 
     def single_line_asset_precompile_hook?(command)
@@ -113,6 +207,156 @@ module Command
       RepoIntrospection.sqlite_database_in_production?(Dir.pwd)
     end
 
+    def validate_sqlite_database_paths!
+      return unless sqlite_project?
+
+      validate_sqlite_database_locations!
+      validate_sqlite_persistence_targets!
+      return unless RepoIntrospection.unresolved_sqlite_database_paths_in_production?(Dir.pwd)
+
+      raise Cpflow::Error,
+            "Production SQLite database paths must be literal file paths in config/database.yml; " \
+            "runtime ERB paths cannot be persisted safely by the generated scaffold."
+    end
+
+    def validate_sqlite_database_locations!
+      database_paths = RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd)
+      validate_sqlite_paths_without_control_characters!(database_paths)
+      validate_sqlite_paths_under_app!(database_paths)
+      validate_sqlite_paths_below_mount_roots!(database_paths)
+    end
+
+    def validate_sqlite_paths_without_control_characters!(database_paths)
+      unsafe_path = database_paths.find { |database_path| database_path.match?(/[[:cntrl:]]/) }
+      return unless unsafe_path
+
+      raise Cpflow::Error,
+            "Production SQLite database paths cannot contain control characters; update config/database.yml " \
+            "before generating the scaffold."
+    end
+
+    def validate_sqlite_paths_under_app!(database_paths)
+      unsupported_path = database_paths.find do |database_path|
+        path = absolute_app_database_path(database_path)
+        path != "/app" && !path.start_with?("/app/")
+      end
+      return unless unsupported_path
+
+      raise Cpflow::Error,
+            "Production SQLite database path #{unsupported_path.inspect} must resolve under /app so the generated " \
+            "scaffold can persist it safely."
+    end
+
+    def validate_sqlite_paths_below_mount_roots!(database_paths)
+      mount_root_path = database_paths.find do |database_path|
+        ["/app/data", "/app/storage"].include?(absolute_app_database_path(database_path))
+      end
+      return unless mount_root_path
+
+      raise Cpflow::Error,
+            "Production SQLite database path #{mount_root_path.inspect} resolves to a volume mount directory; " \
+            "use a file path beneath data/ or storage/."
+    end
+
+    def sqlite_database_setup
+      redirects = sqlite_database_redirects
+      return "" if redirects.empty?
+
+      setup_calls = redirects.map do |source, target, legacy|
+        arguments = [source, target, legacy].compact.map { |path| Shellwords.shellescape(path) }
+        "prepare_sqlite_database #{arguments.join(' ')}"
+      end
+      "#{SQLITE_DATABASE_PREPARE_FUNCTION}\n#{setup_calls.join("\n")}\n"
+    end
+
+    def sqlite_database_redirects
+      return [] unless sqlite_project?
+
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).filter_map do |database_path|
+        source = absolute_app_database_path(database_path)
+        next if persistent_sqlite_path?(source)
+
+        target = persistent_sqlite_target(source)
+        [source, target, legacy_sqlite_database_path(source)]
+      end
+    end
+
+    def validate_sqlite_persistence_targets!
+      paths_by_target = sqlite_database_paths_by_persistence_target
+      validate_exact_sqlite_target_collisions!(paths_by_target)
+      validate_ancestor_sqlite_target_collisions!(paths_by_target.keys)
+    end
+
+    def validate_exact_sqlite_target_collisions!(paths_by_target)
+      collision = paths_by_target.find { |_target, paths| paths.uniq.size > 1 }
+      return unless collision
+
+      target = collision.first
+      paths = collision.last.uniq
+      raise Cpflow::Error,
+            "Production SQLite database paths #{paths.map(&:inspect).join(' and ')} resolve to the same persistent " \
+            "target #{target.inspect}; use distinct paths before generating the scaffold."
+    end
+
+    def validate_ancestor_sqlite_target_collisions!(targets)
+      collision = targets.combination(2).find do |first, second|
+        sqlite_path_ancestor?(first, second) || sqlite_path_ancestor?(second, first)
+      end
+      return unless collision
+
+      first, second = collision
+      raise Cpflow::Error,
+            "Production SQLite persistence targets #{first.inspect} and #{second.inspect} overlap; " \
+            "a database file cannot contain another database path."
+    end
+
+    def sqlite_path_ancestor?(ancestor, descendant)
+      descendant.start_with?("#{ancestor}/")
+    end
+
+    def sqlite_database_paths_by_persistence_target
+      paths_by_target = Hash.new { |hash, key| hash[key] = [] }
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).each do |database_path|
+        source = absolute_app_database_path(database_path)
+        target = persistent_sqlite_path?(source) ? source : persistent_sqlite_target(source)
+        [target, legacy_sqlite_database_path(source)].compact.each do |claimed_path|
+          claim_sqlite_database_paths(paths_by_target, claimed_path, database_path)
+        end
+      end
+      paths_by_target
+    end
+
+    def claim_sqlite_database_paths(paths_by_target, claimed_path, database_path)
+      sqlite_database_owned_paths(claimed_path).each do |owned_path|
+        paths_by_target[owned_path] << database_path
+      end
+    end
+
+    def sqlite_database_owned_paths(database_path)
+      [database_path, "#{database_path}-wal", "#{database_path}-shm", "#{database_path}-journal"]
+    end
+
+    def persistent_sqlite_target(source)
+      relative = source.delete_prefix("/")
+      File.join("/app/data", relative.delete_prefix("app/"))
+    end
+
+    def legacy_sqlite_database_path(source)
+      return unless source.start_with?("/app/db/")
+
+      File.join("/app/data", source.delete_prefix("/app/db/"))
+    end
+
+    def absolute_app_database_path(database_path)
+      path = Pathname.new(database_path)
+      (path.absolute? ? path : Pathname.new("/app").join(path)).cleanpath.to_s
+    end
+
+    def persistent_sqlite_path?(path)
+      path == "/app/data" || path.start_with?("/app/data/") ||
+        path == "/app/storage" || path.start_with?("/app/storage/")
+    end
+
     def normalized_asset_precompile_hook_command
       command = shakapacker_precompile_hook || react_on_rails_auto_bundle_hook
       return unless command
@@ -169,7 +413,7 @@ module Command
       - infers the app prefix from the current directory and wires staging, review, and production entries
       - infers the Docker base Ruby version from `.ruby-version`, `.tool-versions`, or the app's `Gemfile`
       - preserves repo-defined asset precompile hooks, including React on Rails auto bundle generation
-      - detects SQLite in `config/database.yml` and generates persistent `db` and `storage` volume templates instead of the default Postgres workload
+      - detects SQLite in `config/database.yml` and generates persistent `/app/data` and `/app/storage` volume templates without hiding image migrations under `/app/db`
     DESC
     EXAMPLES = <<~EX
       ```sh