putpaws 0.1.5 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,142 @@
1
+ require 'json'
2
+ require 'putpaws/launch_config'
3
+
4
+ # A launch after resolution (see Putpaws::LaunchResolver): `target` and
5
+ # `network` are looked up in infra.json and every override is applied.
6
+ # This is the single place that knows how a launch maps onto the ECS RunTask
7
+ # API, so that a manual run and a schedule start exactly the same thing.
8
+ class Putpaws::ResolvedLaunch
9
+ SHELL = ['/bin/sh', '-c']
10
+ # snake_case (aws-sdk params) -> camelCase (TaskOverride JSON for EventBridge Scheduler)
11
+ JSON_KEYS = {
12
+ container_overrides: 'containerOverrides',
13
+ task_role_arn: 'taskRoleArn',
14
+ ephemeral_storage: 'ephemeralStorage',
15
+ size_in_gi_b: 'sizeInGiB',
16
+ }
17
+
18
+ attr_reader :application, :attributes, :target, :network
19
+
20
+ def initialize(application:, attributes:, target:, network:)
21
+ @application = application
22
+ @attributes = attributes
23
+ @target = target
24
+ @network = network
25
+ end
26
+
27
+ Putpaws::LaunchConfig::FIELDS.each do |field|
28
+ next if %i[target network].include?(field)
29
+ define_method(field){ attributes[field] }
30
+ end
31
+
32
+ # The launch as it can be written to launch.json.
33
+ def to_h
34
+ attributes.dup
35
+ end
36
+
37
+ # Another launch based on this one with further overrides applied.
38
+ def with(overrides)
39
+ Putpaws::LaunchResolver.resolve(application: application, overrides: [attributes, overrides])
40
+ end
41
+
42
+ def region
43
+ application.ecs_region || application.region
44
+ end
45
+
46
+ def cluster
47
+ target.cluster
48
+ end
49
+
50
+ def cluster_name
51
+ target.cluster_name
52
+ end
53
+
54
+ def task_definition
55
+ target.task_definition
56
+ end
57
+
58
+ def container_name
59
+ container || target.container_name || 'app'
60
+ end
61
+
62
+ # A string is run through the shell; an array is passed as it is.
63
+ def command_array
64
+ return nil if command.nil?
65
+ command.is_a?(Array) ? command : SHELL + [command]
66
+ end
67
+
68
+ def network_configuration
69
+ {
70
+ awsvpc_configuration: {
71
+ subnets: network.subnets,
72
+ security_groups: network.security_groups,
73
+ assign_public_ip: network.assign_public_ip,
74
+ }
75
+ }
76
+ end
77
+
78
+ # launch_type and capacity_provider_strategy are mutually exclusive on RunTask.
79
+ def placement
80
+ if capacity_provider && capacity_provider != 'FARGATE'
81
+ {capacity_provider_strategy: [{capacity_provider: capacity_provider, weight: 1}]}
82
+ else
83
+ {launch_type: 'FARGATE'}
84
+ end
85
+ end
86
+
87
+ # `overrides` of ECS RunTask. Empty when the task definition runs as it is.
88
+ def task_overrides
89
+ overrides = {}
90
+ container_override = {name: container_name}
91
+ container_override[:command] = command_array if command
92
+ if env && !env.empty?
93
+ container_override[:environment] = env.map{|name, value| {name: name, value: value}}
94
+ end
95
+ overrides[:container_overrides] = [container_override] if container_override.size > 1
96
+ overrides[:cpu] = cpu.to_s if cpu
97
+ overrides[:memory] = memory.to_s if memory
98
+ overrides[:task_role_arn] = task_role if task_role
99
+ overrides[:ephemeral_storage] = {size_in_gi_b: ephemeral_storage} if ephemeral_storage
100
+ overrides
101
+ end
102
+
103
+ # The same overrides in the JSON shape of the ECS API (TaskOverride),
104
+ # which EventBridge Scheduler takes as the `input` of an ECS target.
105
+ def task_overrides_json
106
+ camelize(task_overrides)
107
+ end
108
+
109
+ # Everything that decides what gets started, for `dry=true` and `info`.
110
+ def describe
111
+ {
112
+ environment: application.name,
113
+ region: region,
114
+ launch: to_h,
115
+ target: {
116
+ name: target.name,
117
+ cluster: target.cluster,
118
+ task_definition: target.task_definition,
119
+ container_name: container_name,
120
+ },
121
+ network: {
122
+ name: network.name,
123
+ subnets: network.subnets,
124
+ security_groups: network.security_groups,
125
+ assign_public_ip: network.assign_public_ip,
126
+ },
127
+ }
128
+ end
129
+
130
+ private
131
+
132
+ def camelize(obj)
133
+ case obj
134
+ when Hash
135
+ obj.map{|k, v| [JSON_KEYS[k] || k.to_s, camelize(v)]}.to_h
136
+ when Array
137
+ obj.map{|x| camelize(x)}
138
+ else
139
+ obj
140
+ end
141
+ end
142
+ end
@@ -1,54 +1,88 @@
1
- require 'json'
2
- require 'pathname'
3
- require 'putpaws/infra_network_config'
4
- require 'putpaws/infra_target_config'
5
-
6
- class Putpaws::ScheduleConfig < Struct.new(
7
- :name, :region, :network, :target, :command, :args,
8
- keyword_init: true)
9
- def self.load(path_prefix: '.putpaws')
10
- @schedule_data ||= begin
11
- path = Pathname.new(path_prefix).join("schedule.json").to_s
12
- data = File.exist?(path) ?
13
- JSON.parse(File.read(path), symbolize_names: true).to_h :
14
- {}
15
- end
1
+ require 'putpaws/config_file'
2
+ require 'putpaws/launch_config'
3
+ require 'putpaws/launch_resolver'
4
+
5
+ # A schedule (.putpaws/<environment>/schedule.json): a launch of the same
6
+ # environment plus when to run it.
7
+ #
8
+ # "nightly-seed": {
9
+ # "launch": "seed",
10
+ # "schedule_expression": "cron(0 18 * * ? *)",
11
+ # "schedule_expression_timezone": "Asia/Tokyo"
12
+ # }
13
+ #
14
+ # Launch fields written here (command / env / cpu ...) override the launch
15
+ # for this schedule only. Everything else is passed to EventBridge Scheduler
16
+ # (create_schedule / update_schedule) as it is.
17
+ class Putpaws::ScheduleConfig
18
+ FILENAME = 'schedule.json'
19
+ OVERRIDE_FIELDS = Putpaws::LaunchConfig::FIELDS - %i[target network]
20
+ # What a schedule held in 0.1.x. They come from the launch / the environment now.
21
+ REMOVED_KEYS = %i[target network region]
22
+
23
+ attr_reader :application, :name, :launch, :overrides, :args
24
+
25
+ def self.path(application)
26
+ application.dir.join(FILENAME)
16
27
  end
17
28
 
18
- def self.build_args(name, config_args)
19
- region = config_args.delete(:region)
20
- network = config_args.delete(:network)
21
- target = config_args.delete(:target)
22
- command = config_args.delete(:command)
23
- {
24
- name: name.to_s,
25
- region: region,
26
- network: network,
27
- target: target,
28
- command: command,
29
- }.merge(args: config_args)
29
+ def self.load(application)
30
+ Putpaws::ConfigFile.read(path(application))
30
31
  end
31
32
 
32
- def self.all
33
- load.map{|k,v|
34
- new(build_args(k, v))
35
- }
33
+ def self.all(application)
34
+ load(application).map{|name, data| new(application: application, name: name, data: data)}
36
35
  end
37
36
 
38
- def self.find(name)
39
- schedule_data = load
40
- data = schedule_data[name.to_sym]
37
+ def self.find(application, name)
38
+ data = load(application)[name.to_sym]
41
39
  return nil unless data
42
- new(build_args(name, data))
40
+ new(application: application, name: name, data: data)
41
+ end
42
+
43
+ def self.find!(application, name)
44
+ find(application, name) or raise Putpaws::ConfigError,
45
+ "Schedule not found: #{name} (#{path(application)})\n" +
46
+ "Available: #{load(application).keys.join(', ')}"
47
+ end
48
+
49
+ def initialize(application:, name:, data:)
50
+ @application = application
51
+ @name = name.to_s
52
+ context = "schedule `#{@name}` (#{self.class.path(application)})"
53
+ raise Putpaws::ConfigError, "#{context} must be a JSON object" unless data.is_a?(Hash)
54
+
55
+ data = data.dup
56
+ removed = data.keys & REMOVED_KEYS
57
+ unless removed.empty?
58
+ raise Putpaws::ConfigError,
59
+ "#{context} has #{removed.map{|k| "`#{k}`"}.join(', ')}, which a schedule no longer holds.\n" +
60
+ "Name a launch of this environment instead: \"launch\": \"<name in launch.json>\""
61
+ end
62
+ @launch = data.delete(:launch)
63
+ if Putpaws::LaunchConfig.blank?(@launch)
64
+ raise Putpaws::ConfigError, "#{context} needs `launch` (a name in launch.json of the same environment)"
65
+ end
66
+ @overrides = Putpaws::LaunchConfig.normalize(
67
+ data.slice(*OVERRIDE_FIELDS), context: context
68
+ )
69
+ @args = data.reject{|k, _| OVERRIDE_FIELDS.include?(k)}
70
+ end
71
+
72
+ # Unique within the account and region: <environment>-<schedule>.
73
+ def aws_name
74
+ "#{application.name}-#{name}"
43
75
  end
44
76
 
45
- def network
46
- return nil unless self[:network]
47
- @network ||= Putpaws::InfraNetworkConfig.find(self[:network])
77
+ def launch_config
78
+ Putpaws::LaunchConfig.find!(application, launch)
48
79
  end
49
80
 
50
- def target
51
- return nil unless self[:target]
52
- @target ||= Putpaws::InfraTargetConfig.find(self[:target])
81
+ def resolve
82
+ Putpaws::LaunchResolver.resolve(
83
+ application: application,
84
+ launch: launch_config,
85
+ overrides: [overrides],
86
+ )
53
87
  end
54
88
  end
@@ -1,61 +1,77 @@
1
1
  require 'aws-sdk-scheduler'
2
2
 
3
3
  module Putpaws::Scheduler
4
+ # Deploys a schedule (Putpaws::ScheduleConfig) to EventBridge Scheduler.
5
+ # What gets started comes from the same resolved launch that `ecs:run` uses.
4
6
  class ScheduleCommand
5
- def self.config(config)
6
- new(**config.log_command_params)
7
- end
7
+ attr_reader :config, :launch
8
8
 
9
- attr_reader :client
10
- attr_reader :config, :network, :target
11
-
12
- def initialize(schedule_config)
9
+ def initialize(schedule_config, client: nil)
13
10
  @config = schedule_config
14
- @network = config.network
15
- @target = config.target
16
- @client = Aws::Scheduler::Client.new({region: config.region})
11
+ @launch = schedule_config.resolve
12
+ @client = client
13
+ end
14
+
15
+ def client
16
+ @client ||= Aws::Scheduler::Client.new({region: launch.region})
17
17
  end
18
18
 
19
19
  def generated_params
20
- {
21
- name: config.name,
20
+ target = launch.target
21
+ unless target.cluster_arn?
22
+ raise Putpaws::ConfigError,
23
+ "target `#{target.name}` needs the cluster ARN as `cluster` to be scheduled (.putpaws/infra.json)"
24
+ end
25
+ if target.scheduler_role.to_s.strip.empty?
26
+ raise Putpaws::ConfigError,
27
+ "target `#{target.name}` needs `scheduler_role` to be scheduled (.putpaws/infra.json)"
28
+ end
29
+
30
+ params = {
31
+ name: config.aws_name,
22
32
  flexible_time_window: {mode: "OFF"},
23
33
  target: {
24
34
  arn: target.cluster,
25
35
  role_arn: target.scheduler_role,
26
36
  retry_policy: {maximum_retry_attempts: 0},
27
- input: {
28
- containerOverrides: [
29
- {
30
- name: target.container_name,
31
- command: config.command,
32
- }
33
- ]
34
- }.to_json,
35
37
  ecs_parameters: {
36
- launch_type: "FARGATE",
37
- network_configuration: {
38
- awsvpc_configuration: {
39
- subnets: network.subnets,
40
- security_groups: network.security_groups,
41
- assign_public_ip: network.assign_public_ip,
42
- }
43
- },
38
+ network_configuration: launch.network_configuration,
44
39
  task_definition_arn: target.task_definition,
45
- task_count: 1, }
40
+ task_count: 1,
41
+ }.merge(launch.placement),
46
42
  }
47
43
  }
44
+ # EventBridge Scheduler hands `input` to RunTask as its `overrides`.
45
+ overrides = launch.task_overrides_json
46
+ params[:target][:input] = overrides.to_json unless overrides.empty?
47
+ params
48
48
  end
49
49
 
50
- def deploy!
51
- # TODO: Use deep merge here
52
- args = generated_params.merge(config.args)
50
+ # Whatever else is written in schedule.json (schedule_expression, state ...)
51
+ # is passed to create_schedule / update_schedule as it is.
52
+ def params
53
+ deep_merge(generated_params, config.args)
54
+ end
53
55
 
56
+ def deploy!
57
+ args = params
58
+ lookup = {name: args[:name]}
59
+ lookup[:group_name] = args[:group_name] if args[:group_name]
54
60
  begin
55
- res = client.get_schedule(name: config.name)
61
+ client.get_schedule(lookup)
56
62
  client.update_schedule(args)
57
- rescue Aws::Scheduler::Errors::ResourceNotFoundException => e
63
+ :updated
64
+ rescue Aws::Scheduler::Errors::ResourceNotFoundException
58
65
  client.create_schedule(args)
66
+ :created
67
+ end
68
+ end
69
+
70
+ private
71
+
72
+ def deep_merge(base, other)
73
+ base.merge(other) do |_, a, b|
74
+ a.is_a?(Hash) && b.is_a?(Hash) ? deep_merge(a, b) : b
59
75
  end
60
76
  end
61
77
  end
@@ -1,29 +1,49 @@
1
+ require "json"
1
2
  require "putpaws/prompt"
2
3
  require "putpaws/schedule_config"
3
4
  require "putpaws/scheduler/schedule_command"
4
5
 
5
6
  namespace :scheduler do
6
- desc "Select Schedule to Manage."
7
+ desc "Select Schedule to Manage. (Ex) schedule=nightly-seed"
7
8
  task :set_schedule_config do
8
9
  app = fetch(:app)
9
- schedules = (app.schedules && !app.schedules.empty?) ? app.schedules : []
10
- if schedules.empty?
11
- schedule_options = Putpaws::ScheduleConfig.all.map(&:name)
12
- raise "Please set `schedules` at .putpaws/schedule.json\nFollowing schedules available:\n#{schedule_options.to_json}\n"
10
+ schedule_config = if ENV['schedule'] && !ENV['schedule'].strip.empty?
11
+ Putpaws::ScheduleConfig.find!(app, ENV['schedule'].strip)
12
+ else
13
+ schedules = app.schedules
14
+ if schedules.empty?
15
+ raise "No schedule is defined in #{Putpaws::ScheduleConfig.path(app)}"
16
+ end
17
+ prompt = Putpaws::Prompt.safe
18
+ selected = prompt.select("Choose a schedule you're going to operate", schedules.map(&:name))
19
+ schedules.find{|x| x.name == selected}
13
20
  end
14
21
 
15
- prompt = Putpaws::Prompt.safe
16
- selected = prompt.select("Choose a task you're going to operate", schedules.map(&:name))
17
- schedule_config = schedules.find{|x| x.name == selected}
18
-
19
22
  set :schedule_config, schedule_config
20
23
  end
21
24
 
22
- desc "Deploy schedule."
25
+ desc "List the schedules of the environment (.putpaws/<environment>/schedule.json). No AWS access."
26
+ task :list do
27
+ app = fetch(:app)
28
+ schedules = app.schedules
29
+ if schedules.empty?
30
+ puts "No schedule is defined in #{Putpaws::ScheduleConfig.path(app)}"
31
+ next
32
+ end
33
+ schedules.each do |schedule|
34
+ puts format("%-24s launch: %-16s %s", schedule.name, schedule.launch, schedule.args[:schedule_expression])
35
+ end
36
+ end
37
+
38
+ desc "Deploy a schedule to EventBridge Scheduler as <environment>-<schedule>. (Ex) schedule=nightly-seed dry=true"
23
39
  task deploy: :set_schedule_config do
24
40
  schedule_config = fetch(:schedule_config)
25
- # pp schedule_config
26
41
  aws = Putpaws::Scheduler::ScheduleCommand.new(schedule_config)
27
- aws.deploy!
42
+ if ENV['dry'] && !ENV['dry'].strip.empty?
43
+ puts JSON.pretty_generate(aws.params)
44
+ next
45
+ end
46
+ result = aws.deploy!
47
+ puts "Schedule #{result}: #{schedule_config.aws_name}"
28
48
  end
29
49
  end
data/lib/putpaws/setup.rb CHANGED
@@ -1,10 +1,22 @@
1
1
  require "putpaws/dsl"
2
2
  include Putpaws::DSL
3
3
 
4
- apps = Putpaws::ApplicationConfig.all
4
+ # The first argument of a command is always the environment:
5
+ # putpaws <environment> <task> key=value ...
6
+ # Every environment folder (.putpaws/<environment>/application.json) becomes
7
+ # a task which selects that environment for the tasks that follow.
8
+ environments = Putpaws::ApplicationConfig.names
5
9
 
6
- apps.each do |app|
7
- Rake::Task.define_task(app.name) do
8
- set(:app, app)
10
+ environments.each do |name|
11
+ Rake::Task.define_task(name) do
12
+ set(:app, Putpaws::ApplicationConfig.find(name))
9
13
  end
10
- end
14
+ end
15
+
16
+ # Names of the 0.1.x layout (top-level keys of .putpaws/application.json)
17
+ # answer with a migration hint instead of "Don't know how to build task".
18
+ (Putpaws::ConfigFile.legacy_application_names - environments).each do |name|
19
+ Rake::Task.define_task(name) do
20
+ Putpaws::ConfigFile.ensure_no_legacy!
21
+ end
22
+ end
@@ -1,9 +1,9 @@
1
- desc "Show resolved settings for the service. No AWS access."
1
+ desc "Show the settings of the environment with its launches and schedules. No AWS access."
2
2
  task :info do
3
3
  app = fetch(:app)
4
4
  row = ->(key, value){ puts format("%-26s %s", "#{key}:", value) unless value.nil? }
5
5
 
6
- row.call('name', app.name)
6
+ row.call('environment', app.name)
7
7
  row.call('region', app.region)
8
8
  row.call('cluster', app.cluster)
9
9
  row.call('service', app.service)
@@ -15,21 +15,19 @@ task :info do
15
15
  row.call('build_log_group_prefix', app.build_log_group_prefix)
16
16
  row.call('build_region', app.build_region)
17
17
 
18
- if (network = app.network)
19
- row.call('network', network.name)
20
- row.call(' subnets', network.subnets && network.subnets.join(', '))
21
- row.call(' security_groups', network.security_groups && network.security_groups.join(', '))
22
- row.call(' assign_public_ip', network.assign_public_ip)
23
- end
24
-
25
- if (target = app.target)
26
- row.call('target', target.name)
27
- row.call(' cluster', target.cluster)
28
- row.call(' task_definition', target.task_definition)
29
- row.call(' container_name', target.container_name)
30
- row.call(' scheduler_role', target.scheduler_role)
18
+ launches = app.launches
19
+ unless launches.empty?
20
+ puts "launches:"
21
+ launches.each do |launch|
22
+ row.call(" #{launch.name}", "target=#{launch.target} network=#{launch.network}")
23
+ end
31
24
  end
32
25
 
33
26
  schedules = app.schedules
34
- row.call('schedules', schedules.map(&:name).join(', ')) unless schedules.empty?
27
+ unless schedules.empty?
28
+ puts "schedules:"
29
+ schedules.each do |schedule|
30
+ row.call(" #{schedule.name}", "launch=#{schedule.launch} #{schedule.args[:schedule_expression]}".strip)
31
+ end
32
+ end
35
33
  end
@@ -1,3 +1,3 @@
1
1
  module Putpaws
2
- VERSION = "0.1.5"
2
+ VERSION = "1.0.0"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: putpaws
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.5
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - metheglin
@@ -159,8 +159,10 @@ files:
159
159
  - lib/putpaws/code_build/build_task.rb
160
160
  - lib/putpaws/code_build/project_command.rb
161
161
  - lib/putpaws/code_build/tasks/build_task.rake
162
+ - lib/putpaws/config_file.rb
162
163
  - lib/putpaws/dsl.rb
163
164
  - lib/putpaws/ecs/ecs_task.rb
165
+ - lib/putpaws/ecs/launch_request.rb
164
166
  - lib/putpaws/ecs/run_command.rb
165
167
  - lib/putpaws/ecs/task_command.rb
166
168
  - lib/putpaws/ecs/tasks/ecs_task.rake
@@ -173,6 +175,8 @@ files:
173
175
  - lib/putpaws/info.rb
174
176
  - lib/putpaws/infra_network_config.rb
175
177
  - lib/putpaws/infra_target_config.rb
178
+ - lib/putpaws/launch_config.rb
179
+ - lib/putpaws/launch_resolver.rb
176
180
  - lib/putpaws/prompt.rb
177
181
  - lib/putpaws/provision/all.rb
178
182
  - lib/putpaws/provision/aws_clients.rb
@@ -182,6 +186,8 @@ files:
182
186
  - lib/putpaws/provision/presets/rails-nginx/preset.json
183
187
  - lib/putpaws/provision/presets/rails-nginx/templates/taskdef-app.json.erb
184
188
  - lib/putpaws/provision/presets/rails-nginx/templates/taskdef-web.json.erb
189
+ - lib/putpaws/provision/presets/rails/preset.json
190
+ - lib/putpaws/provision/presets/rails/templates/taskdef-app.json.erb
185
191
  - lib/putpaws/provision/provision_config.rb
186
192
  - lib/putpaws/provision/provision_task.rb
187
193
  - lib/putpaws/provision/resources/base.rb
@@ -196,6 +202,7 @@ files:
196
202
  - lib/putpaws/provision/state.rb
197
203
  - lib/putpaws/provision/tasks/provision.rake
198
204
  - lib/putpaws/provision/util.rb
205
+ - lib/putpaws/resolved_launch.rb
199
206
  - lib/putpaws/schedule_config.rb
200
207
  - lib/putpaws/scheduler/schedule_command.rb
201
208
  - lib/putpaws/scheduler/scheduler_task.rb