cpflow 5.3.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.agents/agent-workflow.yml +16 -25
- data/.agents/bin/README.md +19 -9
- data/.agents/bin/docs +1 -1
- data/.agents/bin/lint +1 -1
- data/.agents/bin/setup +1 -1
- data/.agents/bin/test +1 -1
- data/.agents/bin/validate +2 -1
- data/.agents/legacy-workflow-policy.yml +11 -0
- data/.agents/shaka.md +20 -0
- data/.agents/trusted-actions.yml +12 -0
- data/.coderabbit.yaml +14 -0
- data/.github/actions/cpflow-build-docker-image/action.yml +153 -22
- data/.github/actions/cpflow-delete-control-plane-app/action.yml +2 -1
- data/.github/actions/cpflow-delete-control-plane-app/delete-app.sh +3 -0
- data/.github/actions/cpflow-preview-promotion/action.yml +82 -0
- data/.github/actions/cpflow-preview-promotion/preview-promotion.sh +90 -0
- data/.github/actions/cpflow-setup-environment/action.yml +8 -6
- data/.github/dependabot.yml +10 -0
- data/.github/workflows/check_cpln_links.yml +6 -1
- data/.github/workflows/claude-code-review.yml +5 -2
- data/.github/workflows/claude.yml +6 -4
- data/.github/workflows/coderabbit-lifecycle-review.yml +29 -0
- data/.github/workflows/command_docs.yml +7 -2
- data/.github/workflows/cpflow-cleanup-stale-review-apps.yml +12 -5
- data/.github/workflows/cpflow-delete-review-app.yml +18 -11
- data/.github/workflows/cpflow-deploy-review-app.yml +38 -17
- data/.github/workflows/cpflow-deploy-staging.yml +28 -11
- data/.github/workflows/cpflow-help-command.yml +3 -3
- data/.github/workflows/cpflow-promote-staging-to-production.yml +135 -16
- data/.github/workflows/cpflow-review-app-help.yml +1 -1
- data/.github/workflows/rspec-shared.yml +34 -26
- data/.github/workflows/rspec-specific.yml +10 -2
- data/.github/workflows/rspec.yml +95 -4
- data/.github/workflows/rubocop.yml +9 -2
- data/.github/workflows/trigger-docs-site.yml +2 -0
- data/.rubocop.yml +6 -1
- data/AGENTS.md +18 -3
- data/CHANGELOG.md +39 -1
- data/CONTRIBUTING.md +39 -2
- data/Gemfile.lock +1 -1
- data/README.md +10 -2
- data/cpflow.gemspec +3 -15
- data/docs/ai-github-flow-prompt.md +3 -3
- data/docs/ci-automation.md +284 -68
- data/docs/commands.md +38 -15
- data/docs/postgres-review-apps.md +90 -0
- data/docs/postgres.md +233 -375
- data/docs/rds-private-networking.md +12 -12
- data/docs/releasing.md +15 -4
- data/docs/secrets-and-env-values.md +42 -0
- data/docs/sidebars.ts +1 -0
- data/examples/controlplane.yml +5 -0
- data/lib/command/ai_github_flow_prompt.rb +2 -2
- data/lib/command/apply_template.rb +36 -4
- data/lib/command/base.rb +56 -3
- data/lib/command/cleanup_stale_apps.rb +28 -4
- data/lib/command/copy_image_from_upstream.rb +3 -0
- data/lib/command/delete.rb +174 -4
- data/lib/command/deploy_image.rb +44 -1
- data/lib/command/generate.rb +247 -3
- data/lib/command/generate_github_actions.rb +43 -13
- data/lib/command/run.rb +224 -34
- data/lib/command/setup_app.rb +92 -10
- data/lib/command/update_github_actions.rb +82 -43
- data/lib/core/config.rb +73 -0
- data/lib/core/controlplane.rb +167 -80
- data/lib/core/controlplane_api.rb +28 -2
- data/lib/core/controlplane_api_direct.rb +17 -0
- data/lib/core/disposable_postgres_helpers.rb +167 -0
- data/lib/core/doctor_service.rb +7 -0
- data/lib/core/github_flow_readiness/checks.rb +2 -1
- data/lib/core/github_flow_readiness_service.rb +4 -0
- data/lib/core/repo_introspection.rb +112 -2
- data/lib/core/shell.rb +35 -10
- data/lib/core/timed_command.rb +111 -0
- data/lib/cpflow/version.rb +1 -1
- data/lib/cpflow.rb +1 -1
- data/lib/generator_templates/.dockerignore +16 -0
- data/lib/generator_templates/Dockerfile +9 -5
- data/lib/generator_templates/entrypoint.sh +2 -0
- data/lib/generator_templates/templates/postgres.yml +9 -0
- data/lib/generator_templates_sqlite/controlplane.yml +2 -1
- data/lib/generator_templates_sqlite/release_script.sh +1 -1
- data/lib/generator_templates_sqlite/templates/rails.yml +2 -1
- data/lib/github_flow_templates/.github/cpflow-help.md +39 -12
- data/lib/github_flow_templates/.github/workflows/cpflow-deploy-review-app.yml +1 -0
- data/lib/github_flow_templates/.github/workflows/cpflow-deploy-staging.yml +1 -0
- data/lib/github_flow_templates/.github/workflows/cpflow-promote-staging-to-production.yml +135 -16
- data/lib/github_flow_templates/bin/pin-cpflow-github-ref +37 -13
- data/lib/github_flow_templates/bin/test-cpflow-github-flow +125 -46
- data/lib/patches/hash.rb +2 -2
- data/rakelib/create_release.rake +14 -14
- data/script/check_shell_scripts +66 -0
- metadata +18 -17
|
@@ -12,11 +12,12 @@ module Command
|
|
|
12
12
|
include GeneratorHelpers
|
|
13
13
|
|
|
14
14
|
argument :staging_branch, type: :string, required: false
|
|
15
|
+
class_option :workflows, type: :array
|
|
15
16
|
|
|
16
17
|
def copy_files
|
|
17
18
|
relative_paths = generated_files
|
|
18
19
|
replacements = template_variables
|
|
19
|
-
|
|
20
|
+
copy_generated_files(relative_paths)
|
|
20
21
|
substitute_template_variables(relative_paths, replacements)
|
|
21
22
|
make_shell_scripts_executable(relative_paths)
|
|
22
23
|
end
|
|
@@ -27,11 +28,11 @@ module Command
|
|
|
27
28
|
|
|
28
29
|
private
|
|
29
30
|
|
|
30
|
-
def
|
|
31
|
+
def copy_generated_files(relative_paths)
|
|
31
32
|
relative_paths.each do |relative_path|
|
|
32
33
|
empty_directory(File.dirname(relative_path), verbose: false)
|
|
33
34
|
copy_file(
|
|
34
|
-
|
|
35
|
+
GenerateGithubActions.source_file(relative_path),
|
|
35
36
|
relative_path,
|
|
36
37
|
force: true,
|
|
37
38
|
verbose: ENV.fetch("HIDE_COMMAND_OUTPUT", nil) != "true"
|
|
@@ -51,7 +52,12 @@ module Command
|
|
|
51
52
|
def generated_files
|
|
52
53
|
# Keep file discovery centralized on the command class so existence checks and
|
|
53
54
|
# Thor's template copy list cannot drift.
|
|
54
|
-
GenerateGithubActions.generated_files
|
|
55
|
+
paths = GenerateGithubActions.generated_files
|
|
56
|
+
return paths unless options.key?("workflows")
|
|
57
|
+
|
|
58
|
+
paths.reject do |path|
|
|
59
|
+
path.start_with?(".github/workflows/") && !options["workflows"].include?(File.basename(path))
|
|
60
|
+
end
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
def staging_branch_filter
|
|
@@ -99,6 +105,9 @@ module Command
|
|
|
99
105
|
- manual promotion from staging to production
|
|
100
106
|
- nightly cleanup and PR help workflows
|
|
101
107
|
|
|
108
|
+
It also copies cpflow's composite actions into `.github/actions/cpflow-*`
|
|
109
|
+
so every local `uses:` target is checked in and can be audited directly.
|
|
110
|
+
|
|
102
111
|
Pass `--staging-branch BRANCH` when staging should auto-deploy from a branch
|
|
103
112
|
other than `main` or `master`; the generator will bake that branch into the
|
|
104
113
|
GitHub Actions push trigger and use it as the default STAGING_APP_BRANCH.
|
|
@@ -108,13 +117,13 @@ module Command
|
|
|
108
117
|
DESC
|
|
109
118
|
EXAMPLES = <<~EX
|
|
110
119
|
```sh
|
|
111
|
-
# Creates
|
|
120
|
+
# Creates workflow wrappers, local composite actions, and validation helpers
|
|
112
121
|
cpflow generate-github-actions
|
|
113
122
|
|
|
114
123
|
# Creates the flow with staging deploys triggered from develop
|
|
115
124
|
cpflow generate-github-actions --staging-branch develop
|
|
116
125
|
|
|
117
|
-
# Overwrites existing generated
|
|
126
|
+
# Overwrites existing generated GitHub Actions files from the installed cpflow gem
|
|
118
127
|
cpflow generate-github-actions --force
|
|
119
128
|
```
|
|
120
129
|
EX
|
|
@@ -122,22 +131,43 @@ module Command
|
|
|
122
131
|
VALIDATIONS = [].freeze
|
|
123
132
|
REQUIRES_STARTUP_CHECKS = false
|
|
124
133
|
|
|
125
|
-
# Resolve
|
|
134
|
+
# Resolve source roots from __dir__ rather than Cpflow.root_path because this file is
|
|
126
135
|
# loaded before `module Cpflow` finishes defining its class methods.
|
|
136
|
+
REPOSITORY_ROOT = Pathname.new(File.expand_path("../..", __dir__))
|
|
127
137
|
TEMPLATE_ROOT = Pathname.new(File.expand_path("../github_flow_templates", __dir__))
|
|
138
|
+
ACTIONS_ROOT = REPOSITORY_ROOT.join(".github/actions")
|
|
128
139
|
|
|
129
|
-
def self.
|
|
140
|
+
def self.generated_file_sources
|
|
130
141
|
ensure_template_root!
|
|
131
142
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
143
|
+
template_sources = files_beneath(TEMPLATE_ROOT).to_h do |source|
|
|
144
|
+
[source.relative_path_from(TEMPLATE_ROOT).to_s, source]
|
|
145
|
+
end
|
|
146
|
+
action_sources = files_beneath(ACTIONS_ROOT, "cpflow-*/**/*").to_h do |source|
|
|
147
|
+
[source.relative_path_from(REPOSITORY_ROOT).to_s, source]
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
template_sources.merge(action_sources).sort.to_h.freeze
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def self.generated_files
|
|
154
|
+
generated_file_sources.keys.freeze
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def self.source_file(relative_path)
|
|
158
|
+
generated_file_sources.fetch(relative_path).to_s
|
|
137
159
|
end
|
|
138
160
|
|
|
139
161
|
def self.ensure_template_root!
|
|
140
162
|
raise "cpflow template directory not found: #{TEMPLATE_ROOT}" unless TEMPLATE_ROOT.directory?
|
|
163
|
+
raise "cpflow action directory not found: #{ACTIONS_ROOT}" unless ACTIONS_ROOT.directory?
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def self.files_beneath(root, pattern = "**/*")
|
|
167
|
+
Dir.glob(root.join(pattern).to_s, File::FNM_DOTMATCH)
|
|
168
|
+
.select { |path| File.file?(path) }
|
|
169
|
+
.map { |path| Pathname.new(path) }
|
|
170
|
+
.sort
|
|
141
171
|
end
|
|
142
172
|
|
|
143
173
|
def call
|
data/lib/command/run.rb
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "timeout"
|
|
4
|
+
|
|
3
5
|
module Command
|
|
4
6
|
class Run < Base # rubocop:disable Metrics/ClassLength
|
|
5
7
|
INTERACTIVE_COMMANDS = [
|
|
@@ -49,6 +51,12 @@ module Command
|
|
|
49
51
|
(can be configured though `runner_job_timeout` in `controlplane.yml`)
|
|
50
52
|
- Waiting for a runner replica is limited to the smaller of `runner_job_timeout` and 1000 seconds.
|
|
51
53
|
A terminal cron status fails immediately, and reaching the observation deadline reports the last safe status
|
|
54
|
+
- With non-interactive log methods 2 and 3, after a command prints its completion marker, Control Plane
|
|
55
|
+
has up to 20 minutes to reconcile the cron job to a terminal status. This can be configured through
|
|
56
|
+
`runner_job_status_reconciliation_timeout` in `controlplane.yml`; timing out exits nonzero and reports
|
|
57
|
+
the job, replica, and last observed status
|
|
58
|
+
- Log method 1 does not emit a completion marker, so its job-status polling is not covered by the
|
|
59
|
+
post-command reconciliation timeout
|
|
52
60
|
- Non-interactive jobs return the Control Plane cron job status even when the job finishes before
|
|
53
61
|
Control Plane exposes a runner replica to attach logs to
|
|
54
62
|
- Injects `CPFLOW_GVC_ID` and `CPFLOW_GVC_CREATED` into the job, exposing the app's immutable GVC
|
|
@@ -108,15 +116,24 @@ module Command
|
|
|
108
116
|
DEFAULT_JOB_CPU = "1"
|
|
109
117
|
DEFAULT_JOB_MEMORY = "2Gi"
|
|
110
118
|
DEFAULT_JOB_TIMEOUT = 21_600 # 6 hours
|
|
119
|
+
DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT = 1_200 # 20 minutes
|
|
111
120
|
DEFAULT_JOB_HISTORY_LIMIT = 10
|
|
112
121
|
MAX_REPLICA_OBSERVATION_SECONDS = 1_000
|
|
113
122
|
REPLICA_OBSERVATION_POLL_INTERVAL_SECONDS = 1
|
|
123
|
+
LOG_QUERY_LOOKBACK_SECONDS = 60
|
|
124
|
+
POST_TERMINAL_LOG_DRAIN_SECONDS = 120
|
|
125
|
+
POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS = 1
|
|
126
|
+
LOG_REQUEST_TIMEOUT_SECONDS = 30
|
|
127
|
+
JOB_STATUS_UNAVAILABLE_RETRY_LIMIT = 5
|
|
128
|
+
JOB_STATUS_POLL_INTERVAL_SECONDS = 1
|
|
129
|
+
JOB_STATUS_REQUEST_TIMEOUT_SECONDS = 30
|
|
114
130
|
NORMALIZED_JOB_STATUS_PATTERN = /\A[a-z][a-z0-9_-]{0,31}\z/
|
|
115
131
|
MAGIC_END = "---cpflow run command finished---"
|
|
116
132
|
|
|
117
133
|
attr_reader :interactive, :detached, :location, :original_workload, :runner_workload,
|
|
118
134
|
:default_image, :default_cpu, :default_memory, :job_timeout, :job_history_limit,
|
|
119
|
-
:container, :job, :replica, :command,
|
|
135
|
+
:job_status_reconciliation_timeout, :container, :job, :replica, :command,
|
|
136
|
+
:job_completed_before_replica_exit_status
|
|
120
137
|
|
|
121
138
|
def call # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
122
139
|
@interactive = config.options[:interactive] || interactive_command?
|
|
@@ -130,6 +147,8 @@ module Command
|
|
|
130
147
|
@default_cpu = config.current[:runner_job_default_cpu] || DEFAULT_JOB_CPU
|
|
131
148
|
@default_memory = config.current[:runner_job_default_memory] || DEFAULT_JOB_MEMORY
|
|
132
149
|
@job_timeout = config.current[:runner_job_timeout] || DEFAULT_JOB_TIMEOUT
|
|
150
|
+
@job_status_reconciliation_timeout =
|
|
151
|
+
config.current[:runner_job_status_reconciliation_timeout] || DEFAULT_JOB_STATUS_RECONCILIATION_TIMEOUT
|
|
133
152
|
@job_history_limit = DEFAULT_JOB_HISTORY_LIMIT
|
|
134
153
|
|
|
135
154
|
unless interactive
|
|
@@ -596,10 +615,12 @@ module Command
|
|
|
596
615
|
if @log_method == 1 || @interactive
|
|
597
616
|
args_join(config.args)
|
|
598
617
|
else
|
|
618
|
+
# MAGIC_END must be a same-stream barrier after all payload output. Merge stderr into stdout before
|
|
619
|
+
# capturing the payload exit status and emitting the marker.
|
|
599
620
|
<<~SCRIPT
|
|
600
|
-
( #{args_join(config.args)} )
|
|
621
|
+
( #{args_join(config.args)} ) 2>&1
|
|
601
622
|
CPFLOW_EXIT_CODE=$?
|
|
602
|
-
|
|
623
|
+
printf '\\n%s\\n' '#{MAGIC_END}'
|
|
603
624
|
exit $CPFLOW_EXIT_CODE
|
|
604
625
|
SCRIPT
|
|
605
626
|
end
|
|
@@ -614,6 +635,7 @@ module Command
|
|
|
614
635
|
|
|
615
636
|
def wait_for_job_status_and_log(logs_pipe) # rubocop:disable Metrics/MethodLength
|
|
616
637
|
no_logs_counter = 0
|
|
638
|
+
command_finished = false
|
|
617
639
|
|
|
618
640
|
loop do
|
|
619
641
|
no_logs_counter += 1
|
|
@@ -623,12 +645,16 @@ module Command
|
|
|
623
645
|
|
|
624
646
|
no_logs_counter = 0
|
|
625
647
|
line = logs_pipe.gets
|
|
626
|
-
|
|
648
|
+
if line.chomp == MAGIC_END
|
|
649
|
+
command_finished = true
|
|
650
|
+
break
|
|
651
|
+
end
|
|
627
652
|
|
|
628
653
|
puts(line)
|
|
629
654
|
end
|
|
630
655
|
|
|
631
|
-
|
|
656
|
+
status_deadline = job_status_reconciliation_deadline if command_finished
|
|
657
|
+
resolve_job_status(status_deadline: status_deadline)
|
|
632
658
|
end
|
|
633
659
|
|
|
634
660
|
def print_detached_commands
|
|
@@ -642,51 +668,198 @@ module Command
|
|
|
642
668
|
)
|
|
643
669
|
end
|
|
644
670
|
|
|
645
|
-
|
|
671
|
+
# rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
672
|
+
def resolve_job_status(status_deadline: nil)
|
|
673
|
+
last_status = @last_job_status
|
|
646
674
|
loop do
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
case status
|
|
652
|
-
when "active", "pending"
|
|
653
|
-
sleep 1
|
|
654
|
-
when "successful"
|
|
655
|
-
break ExitCode::SUCCESS
|
|
656
|
-
else
|
|
675
|
+
remaining_seconds = status_deadline && (status_deadline - monotonic_time)
|
|
676
|
+
if remaining_seconds && remaining_seconds <= 0
|
|
677
|
+
progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_status), :red))
|
|
657
678
|
break ExitCode::ERROR_DEFAULT
|
|
658
679
|
end
|
|
680
|
+
|
|
681
|
+
request_timeout = if remaining_seconds
|
|
682
|
+
[JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining_seconds].min
|
|
683
|
+
else
|
|
684
|
+
JOB_STATUS_REQUEST_TIMEOUT_SECONDS
|
|
685
|
+
end
|
|
686
|
+
begin
|
|
687
|
+
status = current_job_status(timeout_seconds: request_timeout)
|
|
688
|
+
rescue Shell::CommandTimeout
|
|
689
|
+
next
|
|
690
|
+
end
|
|
691
|
+
last_status = status
|
|
692
|
+
@last_job_status = status
|
|
693
|
+
|
|
694
|
+
exit_status = job_exit_status(status, unavailable_is_pending: false)
|
|
695
|
+
break exit_status if exit_status
|
|
696
|
+
|
|
697
|
+
sleep_seconds = JOB_STATUS_POLL_INTERVAL_SECONDS
|
|
698
|
+
sleep_seconds = [sleep_seconds, remaining_seconds].min if remaining_seconds
|
|
699
|
+
Kernel.sleep(sleep_seconds) if sleep_seconds.positive?
|
|
659
700
|
end
|
|
660
701
|
end
|
|
661
702
|
|
|
662
|
-
def
|
|
663
|
-
|
|
703
|
+
def reconciled_job_status(unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
|
|
704
|
+
request_timeout:)
|
|
705
|
+
status = begin
|
|
706
|
+
current_job_status(timeout_seconds: request_timeout)
|
|
707
|
+
rescue Shell::CommandTimeout
|
|
708
|
+
nil
|
|
709
|
+
end
|
|
710
|
+
next_unavailable_status_streak = status.nil? ? unavailable_status_streak + 1 : 0
|
|
711
|
+
exit_status = job_exit_status(status, unavailable_is_pending: status.nil?)
|
|
712
|
+
|
|
713
|
+
if next_unavailable_status_streak > JOB_STATUS_UNAVAILABLE_RETRY_LIMIT && reconciliation_deadline.nil?
|
|
714
|
+
reconciliation_deadline = start_reconciliation_deadline(
|
|
715
|
+
reconciliation_deadline,
|
|
716
|
+
duration: POST_TERMINAL_LOG_DRAIN_SECONDS
|
|
717
|
+
)
|
|
718
|
+
reconciliation_deadline_origin = :status_outage
|
|
719
|
+
elsif reconciliation_deadline_origin == :status_outage && %w[active pending].include?(status)
|
|
720
|
+
reconciliation_deadline = reconciliation_deadline_origin = @post_terminal_log_from = nil
|
|
721
|
+
end
|
|
722
|
+
|
|
723
|
+
[exit_status, next_unavailable_status_streak, status, reconciliation_deadline, reconciliation_deadline_origin]
|
|
724
|
+
end
|
|
725
|
+
|
|
726
|
+
def job_exit_status(status, unavailable_is_pending:)
|
|
727
|
+
Shell.debug("JOB STATUS", normalized_job_status(status))
|
|
728
|
+
|
|
729
|
+
case status
|
|
730
|
+
when nil then unavailable_is_pending ? nil : ExitCode::ERROR_DEFAULT
|
|
731
|
+
when "active", "pending" then nil
|
|
732
|
+
when "successful" then ExitCode::SUCCESS
|
|
733
|
+
else ExitCode::ERROR_DEFAULT
|
|
734
|
+
end
|
|
735
|
+
end
|
|
736
|
+
# rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
737
|
+
|
|
738
|
+
def current_job_status(timeout_seconds: nil)
|
|
739
|
+
result = if timeout_seconds
|
|
740
|
+
cp.fetch_cron_workload(runner_workload, location: location, timeout_seconds: timeout_seconds)
|
|
741
|
+
else
|
|
742
|
+
cp.fetch_cron_workload(runner_workload, location: location)
|
|
743
|
+
end
|
|
664
744
|
job_details = result&.dig("items")&.find { |item| item["id"] == job }
|
|
665
745
|
job_details&.dig("status")
|
|
666
746
|
end
|
|
667
747
|
|
|
748
|
+
def job_status_reconciliation_deadline
|
|
749
|
+
monotonic_time + job_status_reconciliation_timeout
|
|
750
|
+
end
|
|
751
|
+
|
|
752
|
+
def job_status_reconciliation_timeout_message(status)
|
|
753
|
+
replica_args = app_workload_replica_args.join(" ")
|
|
754
|
+
|
|
755
|
+
<<~MESSAGE.chomp
|
|
756
|
+
ERROR: Control Plane job status did not reconcile within #{job_status_reconciliation_timeout} seconds after the command finished.
|
|
757
|
+
job: #{job}
|
|
758
|
+
replica: #{replica}
|
|
759
|
+
status: #{status || 'unknown'}
|
|
760
|
+
Inspect logs: cpflow logs #{replica_args}
|
|
761
|
+
Stop the stale runner: cpflow ps:stop #{replica_args}
|
|
762
|
+
MESSAGE
|
|
763
|
+
end
|
|
764
|
+
|
|
668
765
|
###########################################
|
|
669
766
|
### temporary extaction from run:detached
|
|
670
767
|
###########################################
|
|
671
|
-
|
|
768
|
+
# Tracks log completion, authoritative job status, provisional status outages, and bounded deadlines.
|
|
769
|
+
def show_logs_waiting # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
672
770
|
retries = 0
|
|
771
|
+
exit_status = nil
|
|
772
|
+
reconciliation_deadline = nil
|
|
773
|
+
reconciliation_deadline_origin = nil
|
|
774
|
+
unavailable_status_streak = 0
|
|
775
|
+
finish_marker_seen = false
|
|
776
|
+
last_job_status = nil
|
|
777
|
+
|
|
673
778
|
begin
|
|
674
|
-
|
|
779
|
+
# rubocop:disable Metrics/BlockLength
|
|
675
780
|
loop do
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
781
|
+
remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
|
|
782
|
+
break if remaining && !remaining.positive?
|
|
783
|
+
|
|
784
|
+
unless finish_marker_seen
|
|
785
|
+
log_request_timeout = remaining ? [LOG_REQUEST_TIMEOUT_SECONDS, remaining].min : LOG_REQUEST_TIMEOUT_SECONDS
|
|
786
|
+
log_status = begin
|
|
787
|
+
print_uniq_logs(timeout_seconds: log_request_timeout)
|
|
788
|
+
rescue Shell::CommandTimeout
|
|
789
|
+
:unchanged
|
|
790
|
+
end
|
|
791
|
+
finish_marker_seen = log_status == :finished
|
|
792
|
+
if finish_marker_seen && reconciliation_deadline.nil?
|
|
793
|
+
reconciliation_deadline = start_reconciliation_deadline(
|
|
794
|
+
reconciliation_deadline,
|
|
795
|
+
duration: job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS
|
|
796
|
+
)
|
|
797
|
+
reconciliation_deadline_origin = :finish_marker
|
|
798
|
+
elsif finish_marker_seen && reconciliation_deadline_origin == :status_outage
|
|
799
|
+
reconciliation_deadline = monotonic_time +
|
|
800
|
+
(job_status_reconciliation_timeout || POST_TERMINAL_LOG_DRAIN_SECONDS)
|
|
801
|
+
reconciliation_deadline_origin = :finish_marker
|
|
802
|
+
end
|
|
686
803
|
end
|
|
687
|
-
|
|
804
|
+
remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
|
|
805
|
+
break if remaining && !remaining.positive?
|
|
806
|
+
|
|
807
|
+
request_timeout = if remaining
|
|
808
|
+
[JOB_STATUS_REQUEST_TIMEOUT_SECONDS, remaining].min
|
|
809
|
+
else
|
|
810
|
+
JOB_STATUS_REQUEST_TIMEOUT_SECONDS
|
|
811
|
+
end
|
|
812
|
+
|
|
813
|
+
exit_status ||= begin
|
|
814
|
+
job_status_state = reconciled_job_status(
|
|
815
|
+
unavailable_status_streak, reconciliation_deadline, reconciliation_deadline_origin,
|
|
816
|
+
request_timeout: request_timeout
|
|
817
|
+
)
|
|
818
|
+
current_exit_status, unavailable_status_streak, observed_status,
|
|
819
|
+
reconciliation_deadline, reconciliation_deadline_origin = job_status_state
|
|
820
|
+
last_job_status = observed_status unless observed_status.nil?
|
|
821
|
+
current_exit_status
|
|
822
|
+
end
|
|
823
|
+
if exit_status && reconciliation_deadline_origin == :status_outage
|
|
824
|
+
reconciliation_deadline = monotonic_time + POST_TERMINAL_LOG_DRAIN_SECONDS
|
|
825
|
+
reconciliation_deadline_origin = :terminal
|
|
826
|
+
elsif exit_status && reconciliation_deadline.nil?
|
|
827
|
+
reconciliation_deadline = start_reconciliation_deadline(
|
|
828
|
+
reconciliation_deadline,
|
|
829
|
+
duration: POST_TERMINAL_LOG_DRAIN_SECONDS
|
|
830
|
+
)
|
|
831
|
+
reconciliation_deadline_origin = :terminal
|
|
832
|
+
end
|
|
833
|
+
break if finish_marker_seen && exit_status
|
|
834
|
+
|
|
835
|
+
remaining = reconciliation_deadline && (reconciliation_deadline - monotonic_time)
|
|
836
|
+
break if remaining && !remaining.positive?
|
|
688
837
|
|
|
689
|
-
|
|
838
|
+
poll_interval = POST_TERMINAL_LOG_POLL_INTERVAL_SECONDS
|
|
839
|
+
poll_interval = [poll_interval, remaining].min if remaining
|
|
840
|
+
Kernel.sleep(poll_interval)
|
|
841
|
+
end
|
|
842
|
+
# rubocop:enable Metrics/BlockLength
|
|
843
|
+
|
|
844
|
+
if exit_status && finish_marker_seen
|
|
845
|
+
exit_status
|
|
846
|
+
elsif exit_status
|
|
847
|
+
Shell.warn(
|
|
848
|
+
"Runner job reached terminal status #{normalized_job_status(last_job_status)} but the command " \
|
|
849
|
+
"completion marker was unavailable after #{POST_TERMINAL_LOG_DRAIN_SECONDS} seconds; " \
|
|
850
|
+
"returning authoritative job exit status #{exit_status} with incomplete output."
|
|
851
|
+
)
|
|
852
|
+
exit_status
|
|
853
|
+
elsif reconciliation_deadline_origin == :finish_marker && job_status_reconciliation_timeout
|
|
854
|
+
progress.puts(Shell.color(job_status_reconciliation_timeout_message(last_job_status), :red))
|
|
855
|
+
ExitCode::ERROR_DEFAULT
|
|
856
|
+
else
|
|
857
|
+
Shell.warn(
|
|
858
|
+
"Runner job status reconciliation reached the #{POST_TERMINAL_LOG_DRAIN_SECONDS}-second deadline " \
|
|
859
|
+
"(last_status: #{normalized_job_status(last_job_status)}); returning exit status #{ExitCode::ERROR_DEFAULT}."
|
|
860
|
+
)
|
|
861
|
+
ExitCode::ERROR_DEFAULT
|
|
862
|
+
end
|
|
690
863
|
rescue RuntimeError => e
|
|
691
864
|
raise "#{e} Exiting..." unless retries < 10 # MAX_RETRIES
|
|
692
865
|
|
|
@@ -696,12 +869,20 @@ module Command
|
|
|
696
869
|
end
|
|
697
870
|
end
|
|
698
871
|
|
|
699
|
-
def
|
|
872
|
+
def start_reconciliation_deadline(current_deadline, duration:)
|
|
873
|
+
return current_deadline if current_deadline
|
|
874
|
+
|
|
875
|
+
# Freeze the pre-terminal overlap so late-ingested entries do not age out of the query window.
|
|
876
|
+
@post_terminal_log_from = Time.now.to_i - LOG_QUERY_LOOKBACK_SECONDS
|
|
877
|
+
monotonic_time + duration
|
|
878
|
+
end
|
|
879
|
+
|
|
880
|
+
def print_uniq_logs(timeout_seconds: nil)
|
|
700
881
|
status = nil
|
|
701
882
|
|
|
702
883
|
@printed_log_entries ||= []
|
|
703
884
|
ts = Time.now.to_i
|
|
704
|
-
entries =
|
|
885
|
+
entries = fetch_log_entries(ts, timeout_seconds)
|
|
705
886
|
|
|
706
887
|
(entries - @printed_log_entries).sort.each do |(_ts, val)|
|
|
707
888
|
status ||= :changed
|
|
@@ -713,6 +894,15 @@ module Command
|
|
|
713
894
|
status || :unchanged
|
|
714
895
|
end
|
|
715
896
|
|
|
897
|
+
def fetch_log_entries(to, timeout_seconds)
|
|
898
|
+
from = @post_terminal_log_from || (to - LOG_QUERY_LOOKBACK_SECONDS)
|
|
899
|
+
return normalized_log_entries(from: from, to: to) unless timeout_seconds
|
|
900
|
+
|
|
901
|
+
Timeout.timeout(timeout_seconds, Shell::CommandTimeout) do
|
|
902
|
+
normalized_log_entries(from: from, to: to)
|
|
903
|
+
end
|
|
904
|
+
end
|
|
905
|
+
|
|
716
906
|
def normalized_log_entries(from:, to:)
|
|
717
907
|
log = cp.log_get(workload: runner_workload, from: from, to: to, replica: replica)
|
|
718
908
|
|
data/lib/command/setup_app.rb
CHANGED
|
@@ -18,13 +18,14 @@ module Command
|
|
|
18
18
|
- Configures app to have org-level secrets with default name `"{APP_PREFIX}-secrets"`
|
|
19
19
|
using org-level policy with default name `"{APP_PREFIX}-secrets-policy"` (names can be customized, see docs)
|
|
20
20
|
- Creates identity for secrets if it does not exist
|
|
21
|
+
- For dynamically named review apps with `generated_review_secret_keys`, checks an existing policy before writing credentials, creates a per-app dictionary, skips its secret template during initial setup, and fills missing disposable keys without printing or rotating values
|
|
21
22
|
- Binds the app identity to any configured `shared_secret_grants` policies as part of the secrets setup flow; skipped when `--skip-secrets-setup` or `--skip-secret-access-binding` is provided, or `skip_secrets_setup` is set
|
|
22
23
|
- Use `--skip-secrets-setup` to prevent the automatic setup of secrets,
|
|
23
24
|
or set it through `skip_secrets_setup` in the `.controlplane/controlplane.yml` file
|
|
24
25
|
- Runs a post-creation hook after the app is created if `hooks.post_creation` is specified in the `.controlplane/controlplane.yml` file
|
|
25
26
|
- If the hook exits with a non-zero code, the command will stop executing and also exit with a non-zero code
|
|
26
27
|
- Use `--skip-post-creation-hook` to skip the hook if specified in `controlplane.yml`
|
|
27
|
-
- Use `--refresh-templates` to apply configured templates noninteractively to an existing app while preserving each workload's configured app image even when workloads are unready or use mixed image versions, skipping existing secret
|
|
28
|
+
- Use `--refresh-templates` to apply configured templates noninteractively to an existing app while preserving each workload's configured app image even when workloads are unready or use mixed image versions, skipping existing secret templates (but filling missing opt-in generated review keys), repairing secrets access bindings, and skipping the post-creation hook
|
|
28
29
|
DESC
|
|
29
30
|
VALIDATIONS = %w[config templates].freeze
|
|
30
31
|
|
|
@@ -49,8 +50,13 @@ module Command
|
|
|
49
50
|
|
|
50
51
|
args = []
|
|
51
52
|
args.push("--add-app-identity") unless skip_secrets_setup
|
|
52
|
-
args.push("--yes"
|
|
53
|
-
|
|
53
|
+
args.push("--yes") if refresh_templates
|
|
54
|
+
args.push("--preserve-existing-runtime") if refresh_templates
|
|
55
|
+
if !skip_secrets_setup && config.generated_review_secret_keys.any?
|
|
56
|
+
args.push("--exclude-policy-template", config.secrets_policy)
|
|
57
|
+
args.push("--exclude-secret-template", config.secrets) unless refresh_templates
|
|
58
|
+
end
|
|
59
|
+
run_cpflow_command("apply-template", "-a", config.app, *args, *templates)
|
|
54
60
|
|
|
55
61
|
bind_identity_to_policy unless skip_secrets_setup
|
|
56
62
|
bind_shared_secret_policy_grants(shared_secret_policy_grant_pairs) unless skip_secrets_setup
|
|
@@ -65,24 +71,81 @@ module Command
|
|
|
65
71
|
end
|
|
66
72
|
|
|
67
73
|
def create_secret_and_policy_if_not_exist
|
|
74
|
+
validate_existing_generated_review_policy_before_secret!
|
|
68
75
|
create_secret_if_not_exists
|
|
69
76
|
create_policy_if_not_exists
|
|
70
77
|
|
|
71
78
|
progress.puts
|
|
72
79
|
end
|
|
73
80
|
|
|
81
|
+
def validate_existing_generated_review_policy_before_secret!
|
|
82
|
+
return if config.generated_review_secret_keys.empty?
|
|
83
|
+
|
|
84
|
+
policy = cp.fetch_policy(config.secrets_policy)
|
|
85
|
+
verify_generated_review_policy!(policy) if policy
|
|
86
|
+
end
|
|
87
|
+
|
|
74
88
|
def create_secret_if_not_exists
|
|
75
|
-
|
|
76
|
-
|
|
89
|
+
secret = cp.fetch_secret(config.secrets)
|
|
90
|
+
return existing_secret_if_any(secret) if secret
|
|
91
|
+
|
|
92
|
+
step("Creating secret '#{config.secrets}'") { create_new_secret }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def existing_secret_if_any(secret)
|
|
96
|
+
if config.generated_review_secret_keys.any? && !generated_review_secret_owned_by_app?(secret)
|
|
97
|
+
raise "Existing review app secret dictionary is not owned by this app."
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
progress.puts("Secret '#{config.secrets}' already exists. Skipping creation...")
|
|
101
|
+
fill_missing_generated_review_secrets
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def create_new_secret
|
|
105
|
+
if config.generated_review_secret_keys.any?
|
|
106
|
+
cp.create_sensitive_secret(config.secrets, generated_review_secret_data)
|
|
77
107
|
else
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
108
|
+
cp.apply_hash(build_secret_hash)
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def fill_missing_generated_review_secrets
|
|
113
|
+
keys = config.generated_review_secret_keys
|
|
114
|
+
return if keys.empty?
|
|
115
|
+
|
|
116
|
+
data = revealed_review_secret_data
|
|
117
|
+
|
|
118
|
+
missing_keys = keys.reject { |key| data[key].is_a?(String) && !data[key].empty? }
|
|
119
|
+
return if missing_keys.empty?
|
|
120
|
+
|
|
121
|
+
step("Adding missing generated review app secret fields") do
|
|
122
|
+
cp.patch_sensitive_secret_data(config.secrets, generated_review_secret_data(missing_keys))
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def revealed_review_secret_data
|
|
127
|
+
revealed = begin
|
|
128
|
+
cp.reveal_secret(config.secrets, required: true)
|
|
129
|
+
rescue StandardError
|
|
130
|
+
raise "Cannot safely inspect existing review app secret dictionary.", cause: nil
|
|
81
131
|
end
|
|
132
|
+
raise "Cannot safely inspect existing review app secret dictionary." unless revealed.is_a?(Hash)
|
|
133
|
+
raise "Existing review app secret is not a dictionary." unless revealed["type"] == "dictionary"
|
|
134
|
+
|
|
135
|
+
data = revealed["data"]
|
|
136
|
+
raise "Cannot safely inspect existing review app secret dictionary." unless data.is_a?(Hash)
|
|
137
|
+
|
|
138
|
+
data
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def generated_review_secret_data(keys = config.generated_review_secret_keys)
|
|
142
|
+
keys.to_h { |key| [key, SecureRandom.hex(32)] }
|
|
82
143
|
end
|
|
83
144
|
|
|
84
145
|
def create_policy_if_not_exists
|
|
85
|
-
|
|
146
|
+
policy = cp.fetch_policy(config.secrets_policy)
|
|
147
|
+
if policy
|
|
148
|
+
verify_generated_review_policy!(policy) if config.generated_review_secret_keys.any?
|
|
86
149
|
progress.puts("Policy '#{config.secrets_policy}' already exists. Skipping creation...")
|
|
87
150
|
else
|
|
88
151
|
step("Creating policy '#{config.secrets_policy}'") do
|
|
@@ -91,6 +154,16 @@ module Command
|
|
|
91
154
|
end
|
|
92
155
|
end
|
|
93
156
|
|
|
157
|
+
def verify_generated_review_policy!(policy)
|
|
158
|
+
expected_target = policy_targets_secret?(policy, config.secrets)
|
|
159
|
+
owned_policy = generated_review_app_tag(policy) == config.app
|
|
160
|
+
no_extra_selectors = %w[target targetQuery gvc].all? { |key| policy[key].nil? }
|
|
161
|
+
own_bindings = generated_review_policy_binding_state(policy) != :unsafe
|
|
162
|
+
return if expected_target && owned_policy && no_extra_selectors && own_bindings
|
|
163
|
+
|
|
164
|
+
raise "Existing review app secret policy has an unexpected target or binding."
|
|
165
|
+
end
|
|
166
|
+
|
|
94
167
|
def build_secret_hash
|
|
95
168
|
{
|
|
96
169
|
"kind" => "secret",
|
|
@@ -101,17 +174,26 @@ module Command
|
|
|
101
174
|
end
|
|
102
175
|
|
|
103
176
|
def build_policy_hash
|
|
104
|
-
{
|
|
177
|
+
policy = {
|
|
105
178
|
"kind" => "policy",
|
|
106
179
|
"name" => config.secrets_policy,
|
|
107
180
|
"targetKind" => "secret",
|
|
108
181
|
"targetLinks" => ["//secret/#{config.secrets}"]
|
|
109
182
|
}
|
|
183
|
+
policy["tags"] = { ::Config::GENERATED_REVIEW_APP_TAG => config.app } if config.generated_review_secret_keys.any?
|
|
184
|
+
policy
|
|
110
185
|
end
|
|
111
186
|
|
|
112
187
|
def bind_identity_to_policy
|
|
113
188
|
progress.puts
|
|
114
189
|
|
|
190
|
+
if config.generated_review_secret_keys.any?
|
|
191
|
+
policy = cp.fetch_policy(config.secrets_policy)
|
|
192
|
+
raise "Cannot safely inspect review app secret policy before binding." unless policy.is_a?(Hash)
|
|
193
|
+
|
|
194
|
+
verify_generated_review_policy!(policy)
|
|
195
|
+
end
|
|
196
|
+
|
|
115
197
|
step("Binding identity '#{config.identity}' to policy '#{config.secrets_policy}'") do
|
|
116
198
|
cp.bind_identity_to_policy(config.identity_link, config.secrets_policy)
|
|
117
199
|
end
|