cpflow 6.0.0.rc.0 → 6.0.1

data/lib/command/generate.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require "yaml"
+require "pathname"
+require "shellwords"
 
 require_relative "generator_helpers"
 require_relative "../core/repo_introspection"
@@ -34,10 +36,29 @@ module Command
     # `.tool-versions`, or the `Gemfile`. Keep this on a supported release line
     # (https://www.ruby-lang.org/en/downloads/branches/).
     DEFAULT_RUBY_VERSION = "3.3"
+    SQLITE_DATABASE_PREPARE_FUNCTION = <<~SH
+      prepare_sqlite_database() {
+        source_path="$1"
+        persistent_path="$2"
+        legacy_path="${3:-}"
+        if [ -n "${legacy_path}" ] && [ -e "${legacy_path}" ] && [ ! -e "${persistent_path}" ]; then
+          persistent_path="${legacy_path}"
+        fi
+        mkdir -p "$(dirname "${source_path}")" "$(dirname "${persistent_path}")"
+        if [ -e "${source_path}" ] && [ ! -L "${source_path}" ] && [ ! -e "${persistent_path}" ]; then
+          mv -n "${source_path}" "${persistent_path}"
+        fi
+        rm -f "${source_path}"
+        ln -s "${persistent_path}" "${source_path}"
+      }
+    SH
 
     def copy_files
+      validate_sqlite_database_paths!
       generated_paths = copy_template_files("generator_templates", base_template_files)
       generated_paths += copy_template_files("generator_templates_sqlite", SQLITE_TEMPLATE_FILES) if sqlite_project?
+      copy_dockerignore unless File.exist?(".dockerignore")
+      append_dockerignore_entries
       substitute_template_variables(generated_paths)
       make_shell_scripts_executable(generated_paths)
     end
@@ -64,6 +85,78 @@ module Command
       destination_path
     end
 
+    def copy_dockerignore
+      copy_file(
+        File.join("generator_templates", ".dockerignore"),
+        ".dockerignore",
+        verbose: ENV.fetch("HIDE_COMMAND_OUTPUT", nil) != "true"
+      )
+    end
+
+    def append_dockerignore_entries
+      contents = File.read(".dockerignore")
+      existing_lines = contents.lines(chomp: true)
+      additions = dockerignore_additions(existing_lines)
+      return if additions.empty?
+
+      File.open(".dockerignore", "a") do |file|
+        file.write("\n") unless contents.empty? || contents.end_with?("\n")
+        file.puts(additions)
+      end
+    end
+
+    def dockerignore_additions(existing_lines)
+      entries = File.readlines(File.join(self.class.source_root, "generator_templates", ".dockerignore"), chomp: true)
+      sqlite_entries = sqlite_project? ? sqlite_database_ignore_entries : []
+      entries += sqlite_entries
+      env_entries = [".env*", "!.env.example"]
+      mandatory_exclusions = ["config/master.key", "config/credentials/*.key", *sqlite_entries]
+      ordinary_entries = entries - env_entries - mandatory_exclusions
+      additions = ordinary_entries - existing_lines
+      additions += mandatory_exclusions.reject { |entry| dockerignore_exclusion_effective?(existing_lines, entry) }
+      additions += env_entries unless dockerignore_env_exception_preserved?(existing_lines)
+      additions
+    end
+
+    def dockerignore_exclusion_effective?(lines, exclusion)
+      exclusion_index = lines.rindex(exclusion)
+      exclusion_index && lines[(exclusion_index + 1)..].none? { |line| line.start_with?("!") }
+    end
+
+    def dockerignore_env_exception_preserved?(lines)
+      exclusion_index = lines.rindex(".env*")
+      exception_index = lines.rindex("!.env.example")
+      return false unless exclusion_index && exception_index && exception_index > exclusion_index
+
+      lines[(exception_index + 1)..].none? { |line| line.start_with?("!") }
+    end
+
+    def sqlite_database_ignore_entries
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).flat_map do |database_path|
+        relative = docker_context_database_path(database_path)
+        next [] unless relative
+
+        base = "/#{escape_dockerignore_path(relative)}"
+        [base, "#{base}-wal", "#{base}-shm", "#{base}-journal"]
+      end.uniq
+    end
+
+    def escape_dockerignore_path(path)
+      path.gsub(/[\\*?\[\]]/) { |character| "\\#{character}" }
+    end
+
+    def docker_context_database_path(database_path)
+      path = Pathname.new(database_path).cleanpath
+      if path.absolute?
+        return unless path.to_s.start_with?("/app/")
+
+        return path.to_s.delete_prefix("/app/")
+      end
+
+      normalized = path.to_s
+      normalized unless normalized == "." || normalized.start_with?("../")
+    end
+
     def base_template_files
       COMMON_TEMPLATE_FILES + (sqlite_project? ? [] : POSTGRES_TEMPLATE_FILES)
     end
@@ -72,7 +165,8 @@ module Command
       {
         "__APP_PREFIX__" => inferred_app_prefix,
         "__RUBY_VERSION__" => inferred_ruby_version,
-        "__ASSET_PRECOMPILE_HOOK_RUN__" => asset_precompile_hook_run
+        "__ASSET_PRECOMPILE_HOOK_RUN__" => asset_precompile_hook_run,
+        "__SQLITE_DATABASE_SETUP__" => sqlite_database_setup
       }
     end
 
@@ -99,7 +193,7 @@ module Command
       return "" if stripped.empty?
       return "" unless single_line_asset_precompile_hook?(stripped)
 
-      "RUN #{stripped}\n\n"
+      "RUN export SECRET_KEY_BASE=NOT_USED_NON_BLANK && #{stripped}\n\n"
     end
 
     def single_line_asset_precompile_hook?(command)
@@ -113,6 +207,156 @@ module Command
       RepoIntrospection.sqlite_database_in_production?(Dir.pwd)
     end
 
+    def validate_sqlite_database_paths!
+      return unless sqlite_project?
+
+      validate_sqlite_database_locations!
+      validate_sqlite_persistence_targets!
+      return unless RepoIntrospection.unresolved_sqlite_database_paths_in_production?(Dir.pwd)
+
+      raise Cpflow::Error,
+            "Production SQLite database paths must be literal file paths in config/database.yml; " \
+            "runtime ERB paths cannot be persisted safely by the generated scaffold."
+    end
+
+    def validate_sqlite_database_locations!
+      database_paths = RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd)
+      validate_sqlite_paths_without_control_characters!(database_paths)
+      validate_sqlite_paths_under_app!(database_paths)
+      validate_sqlite_paths_below_mount_roots!(database_paths)
+    end
+
+    def validate_sqlite_paths_without_control_characters!(database_paths)
+      unsafe_path = database_paths.find { |database_path| database_path.match?(/[[:cntrl:]]/) }
+      return unless unsafe_path
+
+      raise Cpflow::Error,
+            "Production SQLite database paths cannot contain control characters; update config/database.yml " \
+            "before generating the scaffold."
+    end
+
+    def validate_sqlite_paths_under_app!(database_paths)
+      unsupported_path = database_paths.find do |database_path|
+        path = absolute_app_database_path(database_path)
+        path != "/app" && !path.start_with?("/app/")
+      end
+      return unless unsupported_path
+
+      raise Cpflow::Error,
+            "Production SQLite database path #{unsupported_path.inspect} must resolve under /app so the generated " \
+            "scaffold can persist it safely."
+    end
+
+    def validate_sqlite_paths_below_mount_roots!(database_paths)
+      mount_root_path = database_paths.find do |database_path|
+        ["/app/data", "/app/storage"].include?(absolute_app_database_path(database_path))
+      end
+      return unless mount_root_path
+
+      raise Cpflow::Error,
+            "Production SQLite database path #{mount_root_path.inspect} resolves to a volume mount directory; " \
+            "use a file path beneath data/ or storage/."
+    end
+
+    def sqlite_database_setup
+      redirects = sqlite_database_redirects
+      return "" if redirects.empty?
+
+      setup_calls = redirects.map do |source, target, legacy|
+        arguments = [source, target, legacy].compact.map { |path| Shellwords.shellescape(path) }
+        "prepare_sqlite_database #{arguments.join(' ')}"
+      end
+      "#{SQLITE_DATABASE_PREPARE_FUNCTION}\n#{setup_calls.join("\n")}\n"
+    end
+
+    def sqlite_database_redirects
+      return [] unless sqlite_project?
+
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).filter_map do |database_path|
+        source = absolute_app_database_path(database_path)
+        next if persistent_sqlite_path?(source)
+
+        target = persistent_sqlite_target(source)
+        [source, target, legacy_sqlite_database_path(source)]
+      end
+    end
+
+    def validate_sqlite_persistence_targets!
+      paths_by_target = sqlite_database_paths_by_persistence_target
+      validate_exact_sqlite_target_collisions!(paths_by_target)
+      validate_ancestor_sqlite_target_collisions!(paths_by_target.keys)
+    end
+
+    def validate_exact_sqlite_target_collisions!(paths_by_target)
+      collision = paths_by_target.find { |_target, paths| paths.uniq.size > 1 }
+      return unless collision
+
+      target = collision.first
+      paths = collision.last.uniq
+      raise Cpflow::Error,
+            "Production SQLite database paths #{paths.map(&:inspect).join(' and ')} resolve to the same persistent " \
+            "target #{target.inspect}; use distinct paths before generating the scaffold."
+    end
+
+    def validate_ancestor_sqlite_target_collisions!(targets)
+      collision = targets.combination(2).find do |first, second|
+        sqlite_path_ancestor?(first, second) || sqlite_path_ancestor?(second, first)
+      end
+      return unless collision
+
+      first, second = collision
+      raise Cpflow::Error,
+            "Production SQLite persistence targets #{first.inspect} and #{second.inspect} overlap; " \
+            "a database file cannot contain another database path."
+    end
+
+    def sqlite_path_ancestor?(ancestor, descendant)
+      descendant.start_with?("#{ancestor}/")
+    end
+
+    def sqlite_database_paths_by_persistence_target
+      paths_by_target = Hash.new { |hash, key| hash[key] = [] }
+      RepoIntrospection.sqlite_database_paths_in_production(Dir.pwd).each do |database_path|
+        source = absolute_app_database_path(database_path)
+        target = persistent_sqlite_path?(source) ? source : persistent_sqlite_target(source)
+        [target, legacy_sqlite_database_path(source)].compact.each do |claimed_path|
+          claim_sqlite_database_paths(paths_by_target, claimed_path, database_path)
+        end
+      end
+      paths_by_target
+    end
+
+    def claim_sqlite_database_paths(paths_by_target, claimed_path, database_path)
+      sqlite_database_owned_paths(claimed_path).each do |owned_path|
+        paths_by_target[owned_path] << database_path
+      end
+    end
+
+    def sqlite_database_owned_paths(database_path)
+      [database_path, "#{database_path}-wal", "#{database_path}-shm", "#{database_path}-journal"]
+    end
+
+    def persistent_sqlite_target(source)
+      relative = source.delete_prefix("/")
+      File.join("/app/data", relative.delete_prefix("app/"))
+    end
+
+    def legacy_sqlite_database_path(source)
+      return unless source.start_with?("/app/db/")
+
+      File.join("/app/data", source.delete_prefix("/app/db/"))
+    end
+
+    def absolute_app_database_path(database_path)
+      path = Pathname.new(database_path)
+      (path.absolute? ? path : Pathname.new("/app").join(path)).cleanpath.to_s
+    end
+
+    def persistent_sqlite_path?(path)
+      path == "/app/data" || path.start_with?("/app/data/") ||
+        path == "/app/storage" || path.start_with?("/app/storage/")
+    end
+
     def normalized_asset_precompile_hook_command
       command = shakapacker_precompile_hook || react_on_rails_auto_bundle_hook
       return unless command
@@ -169,7 +413,7 @@ module Command
       - infers the app prefix from the current directory and wires staging, review, and production entries
       - infers the Docker base Ruby version from `.ruby-version`, `.tool-versions`, or the app's `Gemfile`
       - preserves repo-defined asset precompile hooks, including React on Rails auto bundle generation
-      - detects SQLite in `config/database.yml` and generates persistent `db` and `storage` volume templates instead of the default Postgres workload
+      - detects SQLite in `config/database.yml` and generates persistent `/app/data` and `/app/storage` volume templates without hiding image migrations under `/app/db`
     DESC
     EXAMPLES = <<~EX
       ```sh

data/lib/command/generate_github_actions.rb

@@ -12,6 +12,7 @@ module Command
     include GeneratorHelpers
 
     argument :staging_branch, type: :string, required: false
+    class_option :workflows, type: :array
 
     def copy_files
       relative_paths = generated_files
@@ -51,7 +52,12 @@ module Command
     def generated_files
       # Keep file discovery centralized on the command class so existence checks and
       # Thor's template copy list cannot drift.
-      GenerateGithubActions.generated_files
+      paths = GenerateGithubActions.generated_files
+      return paths unless options.key?("workflows")
+
+      paths.reject do |path|
+        path.start_with?(".github/workflows/") && !options["workflows"].include?(File.basename(path))
+      end
     end
 
     def staging_branch_filter
@@ -99,8 +105,8 @@ module Command
       - manual promotion from staging to production
       - nightly cleanup and PR help workflows
 
-      It also copies cpflow's composite actions into `.github/actions/cpflow-*`
-      so every local `uses:` target is checked in and can be audited directly.
+      The workflows load cpflow's composite actions from this gem's repository at
+      the ref each workflow pins, so no `.github/actions/cpflow-*` copies are generated.
 
       Pass `--staging-branch BRANCH` when staging should auto-deploy from a branch
       other than `main` or `master`; the generator will bake that branch into the
@@ -111,7 +117,7 @@ module Command
     DESC
     EXAMPLES = <<~EX
       ```sh
-      # Creates workflow wrappers, local composite actions, and validation helpers
+      # Creates workflow wrappers and validation helpers
       cpflow generate-github-actions
 
       # Creates the flow with staging deploys triggered from develop
@@ -125,23 +131,16 @@ module Command
     VALIDATIONS = [].freeze
     REQUIRES_STARTUP_CHECKS = false
 
-    # Resolve source roots from __dir__ rather than Cpflow.root_path because this file is
+    # Resolve the source root from __dir__ rather than Cpflow.root_path because this file is
     # loaded before `module Cpflow` finishes defining its class methods.
-    REPOSITORY_ROOT = Pathname.new(File.expand_path("../..", __dir__))
     TEMPLATE_ROOT = Pathname.new(File.expand_path("../github_flow_templates", __dir__))
-    ACTIONS_ROOT = REPOSITORY_ROOT.join(".github/actions")
 
     def self.generated_file_sources
       ensure_template_root!
 
-      template_sources = files_beneath(TEMPLATE_ROOT).to_h do |source|
+      files_beneath(TEMPLATE_ROOT).to_h do |source|
         [source.relative_path_from(TEMPLATE_ROOT).to_s, source]
-      end
-      action_sources = files_beneath(ACTIONS_ROOT, "cpflow-*/**/*").to_h do |source|
-        [source.relative_path_from(REPOSITORY_ROOT).to_s, source]
-      end
-
-      template_sources.merge(action_sources).sort.to_h.freeze
+      end.sort.to_h.freeze
     end
 
     def self.generated_files
@@ -154,11 +153,10 @@ module Command
 
     def self.ensure_template_root!
       raise "cpflow template directory not found: #{TEMPLATE_ROOT}" unless TEMPLATE_ROOT.directory?
-      raise "cpflow action directory not found: #{ACTIONS_ROOT}" unless ACTIONS_ROOT.directory?
     end
 
-    def self.files_beneath(root, pattern = "**/*")
-      Dir.glob(root.join(pattern).to_s, File::FNM_DOTMATCH)
+    def self.files_beneath(root)
+      Dir.glob(root.join("**/*").to_s, File::FNM_DOTMATCH)
          .select { |path| File.file?(path) }
          .map { |path| Pathname.new(path) }
          .sort

data/lib/command/setup_app.rb

@@ -18,13 +18,14 @@ module Command
       - Configures app to have org-level secrets with default name `"{APP_PREFIX}-secrets"`
         using org-level policy with default name `"{APP_PREFIX}-secrets-policy"` (names can be customized, see docs)
       - Creates identity for secrets if it does not exist
+      - For dynamically named review apps with `generated_review_secret_keys`, checks an existing policy before writing credentials, creates a per-app dictionary, skips its secret template during initial setup, and fills missing disposable keys without printing or rotating values
       - Binds the app identity to any configured `shared_secret_grants` policies as part of the secrets setup flow; skipped when `--skip-secrets-setup` or `--skip-secret-access-binding` is provided, or `skip_secrets_setup` is set
       - Use `--skip-secrets-setup` to prevent the automatic setup of secrets,
         or set it through `skip_secrets_setup` in the `.controlplane/controlplane.yml` file
       - Runs a post-creation hook after the app is created if `hooks.post_creation` is specified in the `.controlplane/controlplane.yml` file
       - If the hook exits with a non-zero code, the command will stop executing and also exit with a non-zero code
       - Use `--skip-post-creation-hook` to skip the hook if specified in `controlplane.yml`
-      - Use `--refresh-templates` to apply configured templates noninteractively to an existing app while preserving each workload's configured app image even when workloads are unready or use mixed image versions, skipping existing secret resources entirely, repairing secrets access bindings, and skipping the post-creation hook
+      - Use `--refresh-templates` to apply configured templates noninteractively to an existing app while preserving each workload's configured app image even when workloads are unready or use mixed image versions, skipping existing secret templates (but filling missing opt-in generated review keys), repairing secrets access bindings, and skipping the post-creation hook
     DESC
     VALIDATIONS = %w[config templates].freeze
 
@@ -49,8 +50,13 @@ module Command
 
       args = []
       args.push("--add-app-identity") unless skip_secrets_setup
-      args.push("--yes", "--preserve-existing-runtime") if refresh_templates
-      run_cpflow_command("apply-template", *templates, "-a", config.app, *args)
+      args.push("--yes") if refresh_templates
+      args.push("--preserve-existing-runtime") if refresh_templates
+      if !skip_secrets_setup && config.generated_review_secret_keys.any?
+        args.push("--exclude-policy-template", config.secrets_policy)
+        args.push("--exclude-secret-template", config.secrets) unless refresh_templates
+      end
+      run_cpflow_command("apply-template", "-a", config.app, *args, *templates)
 
       bind_identity_to_policy unless skip_secrets_setup
       bind_shared_secret_policy_grants(shared_secret_policy_grant_pairs) unless skip_secrets_setup
@@ -65,24 +71,81 @@ module Command
     end
 
     def create_secret_and_policy_if_not_exist
+      validate_existing_generated_review_policy_before_secret!
       create_secret_if_not_exists
       create_policy_if_not_exists
 
       progress.puts
     end
 
+    def validate_existing_generated_review_policy_before_secret!
+      return if config.generated_review_secret_keys.empty?
+
+      policy = cp.fetch_policy(config.secrets_policy)
+      verify_generated_review_policy!(policy) if policy
+    end
+
     def create_secret_if_not_exists
-      if cp.fetch_secret(config.secrets)
-        progress.puts("Secret '#{config.secrets}' already exists. Skipping creation...")
+      secret = cp.fetch_secret(config.secrets)
+      return existing_secret_if_any(secret) if secret
+
+      step("Creating secret '#{config.secrets}'") { create_new_secret }
+    end
+
+    def existing_secret_if_any(secret)
+      if config.generated_review_secret_keys.any? && !generated_review_secret_owned_by_app?(secret)
+        raise "Existing review app secret dictionary is not owned by this app."
+      end
+
+      progress.puts("Secret '#{config.secrets}' already exists. Skipping creation...")
+      fill_missing_generated_review_secrets
+    end
+
+    def create_new_secret
+      if config.generated_review_secret_keys.any?
+        cp.create_sensitive_secret(config.secrets, generated_review_secret_data)
       else
-        step("Creating secret '#{config.secrets}'") do
-          cp.apply_hash(build_secret_hash)
-        end
+        cp.apply_hash(build_secret_hash)
+      end
+    end
+
+    def fill_missing_generated_review_secrets
+      keys = config.generated_review_secret_keys
+      return if keys.empty?
+
+      data = revealed_review_secret_data
+
+      missing_keys = keys.reject { |key| data[key].is_a?(String) && !data[key].empty? }
+      return if missing_keys.empty?
+
+      step("Adding missing generated review app secret fields") do
+        cp.patch_sensitive_secret_data(config.secrets, generated_review_secret_data(missing_keys))
+      end
+    end
+
+    def revealed_review_secret_data
+      revealed = begin
+        cp.reveal_secret(config.secrets, required: true)
+      rescue StandardError
+        raise "Cannot safely inspect existing review app secret dictionary.", cause: nil
       end
+      raise "Cannot safely inspect existing review app secret dictionary." unless revealed.is_a?(Hash)
+      raise "Existing review app secret is not a dictionary." unless revealed["type"] == "dictionary"
+
+      data = revealed["data"]
+      raise "Cannot safely inspect existing review app secret dictionary." unless data.is_a?(Hash)
+
+      data
+    end
+
+    def generated_review_secret_data(keys = config.generated_review_secret_keys)
+      keys.to_h { |key| [key, SecureRandom.hex(32)] }
     end
 
     def create_policy_if_not_exists
-      if cp.fetch_policy(config.secrets_policy)
+      policy = cp.fetch_policy(config.secrets_policy)
+      if policy
+        verify_generated_review_policy!(policy) if config.generated_review_secret_keys.any?
         progress.puts("Policy '#{config.secrets_policy}' already exists. Skipping creation...")
       else
         step("Creating policy '#{config.secrets_policy}'") do
@@ -91,6 +154,16 @@ module Command
       end
     end
 
+    def verify_generated_review_policy!(policy)
+      expected_target = policy_targets_secret?(policy, config.secrets)
+      owned_policy = generated_review_app_tag(policy) == config.app
+      no_extra_selectors = %w[target targetQuery gvc].all? { |key| policy[key].nil? }
+      own_bindings = generated_review_policy_binding_state(policy) != :unsafe
+      return if expected_target && owned_policy && no_extra_selectors && own_bindings
+
+      raise "Existing review app secret policy has an unexpected target or binding."
+    end
+
     def build_secret_hash
       {
         "kind" => "secret",
@@ -101,17 +174,26 @@ module Command
     end
 
     def build_policy_hash
-      {
+      policy = {
         "kind" => "policy",
         "name" => config.secrets_policy,
         "targetKind" => "secret",
         "targetLinks" => ["//secret/#{config.secrets}"]
       }
+      policy["tags"] = { ::Config::GENERATED_REVIEW_APP_TAG => config.app } if config.generated_review_secret_keys.any?
+      policy
     end
 
     def bind_identity_to_policy
       progress.puts
 
+      if config.generated_review_secret_keys.any?
+        policy = cp.fetch_policy(config.secrets_policy)
+        raise "Cannot safely inspect review app secret policy before binding." unless policy.is_a?(Hash)
+
+        verify_generated_review_policy!(policy)
+      end
+
       step("Binding identity '#{config.identity}' to policy '#{config.secrets_policy}'") do
         cp.bind_identity_to_policy(config.identity_link, config.secrets_policy)
       end