shards 2.0.1 → 2.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +14 -2
- data/README.md +21 -0
- data/lib/settings/format_methods.yaml +2 -3
- data/lib/settings/workflows/base.yaml +26 -0
- data/lib/settings/workflows/fast_terminal.yaml +3 -0
- data/lib/settings/workflows/fingerprint_terminal.yaml +23 -0
- data/lib/shards/base.rb +3 -0
- data/lib/shards/base_yaml_object.rb +51 -1
- data/lib/shards/client.rb +40 -0
- data/lib/shards/config.rb +3 -1
- data/lib/shards/db.rb +0 -5
- data/lib/shards/dns.rb +1 -1
- data/lib/shards/ip_search.rb +107 -0
- data/lib/shards/meta/meta.rb +14 -0
- data/lib/shards/stage.rb +44 -3
- data/lib/shards/version.rb +1 -1
- data/lib/shards/workflow/base.rb +41 -0
- data/lib/shards/workflow/fast_terminal.rb +1 -1
- data/lib/shards/workflow/fingerprint_terminal.rb +42 -0
- data/lib/shards/workflow/terminal.rb +1 -1
- data/lib/shards/workflow/workflow.rb +1 -0
- data/shards.gemspec +3 -0
- data/test.env +14 -1
- metadata +47 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 84a189cde1dd20ee0c703592580ddccbca10bdc5
|
4
|
+
data.tar.gz: bc439f40ae05dcdc97111c061933aa01da8e257a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7dcbd6cfdb02e578542507c18187aeb76f16292e48bae745311960fb79be9124d427ff4ce1943590bafb68ea67141064c746efbe96b37ba05973a744afb38cf
|
7
|
+
data.tar.gz: 34676aa6982b0d2b532e7087815ab74d141f60d6ccab91f5b41aef3a158fe8a336ff1c228b022267e894bee95dbb18c6bfec424cbef11bdf11ca3392885665c5
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
shards (2.
|
4
|
+
shards (2.1.6)
|
5
|
+
aws-sdk-ec2 (~> 1.48)
|
6
|
+
aws-sdk-elasticloadbalancing (~> 1.5)
|
7
|
+
aws-sdk-elasticloadbalancingv2 (~> 1.14)
|
5
8
|
aws-sdk-route53 (~> 1.9)
|
6
9
|
dotenv (~> 2.4)
|
7
10
|
git (~> 1.4)
|
@@ -21,12 +24,21 @@ GEM
|
|
21
24
|
rspec-expectations (>= 2.99)
|
22
25
|
thor (~> 0.19)
|
23
26
|
aws-eventstream (1.0.1)
|
24
|
-
aws-partitions (1.
|
27
|
+
aws-partitions (1.104.0)
|
25
28
|
aws-sdk-core (3.27.0)
|
26
29
|
aws-eventstream (~> 1.0)
|
27
30
|
aws-partitions (~> 1.0)
|
28
31
|
aws-sigv4 (~> 1.0)
|
29
32
|
jmespath (~> 1.0)
|
33
|
+
aws-sdk-ec2 (1.48.0)
|
34
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
35
|
+
aws-sigv4 (~> 1.0)
|
36
|
+
aws-sdk-elasticloadbalancing (1.5.0)
|
37
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
38
|
+
aws-sigv4 (~> 1.0)
|
39
|
+
aws-sdk-elasticloadbalancingv2 (1.14.0)
|
40
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
41
|
+
aws-sigv4 (~> 1.0)
|
30
42
|
aws-sdk-route53 (1.12.0)
|
31
43
|
aws-sdk-core (~> 3, >= 3.26.0)
|
32
44
|
aws-sigv4 (~> 1.0)
|
data/README.md
CHANGED
@@ -30,26 +30,47 @@ Add a settings.env file in your root application directory with the variables:
|
|
30
30
|
```bash
|
31
31
|
|
32
32
|
ENGINEERING_ROOT_PATH=/your/root/path/to/engineering/repo/project
|
33
|
+
|
33
34
|
CONFIG_FILE=relative/path/to/config.yml
|
35
|
+
|
34
36
|
PROXY_SUBDOMAIN=proxy.example.com
|
37
|
+
|
35
38
|
DOMAIN=example.com
|
39
|
+
|
36
40
|
CONFIG_FILES_ROOT_PATH=puppet/environments/common/bbug_config/files
|
41
|
+
|
42
|
+
PATH_TO_ENGINEERING_YAML=puppet/environments/common/bbug_engineering_config/files
|
43
|
+
|
37
44
|
SHARDS_FILE_NAME=shards.yml
|
45
|
+
|
38
46
|
SITES_FILE_NAME=sites.yml
|
47
|
+
|
39
48
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
49
|
+
|
40
50
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
51
|
+
|
41
52
|
DNS_ZONE_CODE=DNS Zone Code
|
53
|
+
|
42
54
|
PROXY_USER=proxy_username
|
43
55
|
|
56
|
+
REMOTE_SERVER_CURRENT_PATH=/rails/project/current_directory
|
57
|
+
|
44
58
|
# optional:
|
59
|
+
|
45
60
|
TEST=true #only for test.env delete this var in settings.env
|
61
|
+
|
46
62
|
CONFIG_REPLACE={ 'prod' => 'web' }
|
63
|
+
|
47
64
|
LIVE_BRANCH=shards_test # set this var to use in test.env or to change the default value 'v5'
|
65
|
+
|
48
66
|
WORKFLOW_YAML_PATH=path/to/your/workflows/ # you could overwrite yaml files using the same name in the defined path.
|
49
67
|
|
68
|
+
|
50
69
|
# You could overwrite methods using this metaprograming way ading your on your_file.yaml
|
51
70
|
# Then add this var:
|
71
|
+
|
52
72
|
# FORMAT_METHODS_FILE=your/path/to/your_file.yaml
|
73
|
+
|
53
74
|
# Finally to overwrite the default behavior write your code in your_file.yaml. The yaml will be merged so you need define just the methods thay you need overwrite.
|
54
75
|
# The default definitions are in spec/support/format_methods.yaml
|
55
76
|
|
@@ -61,8 +61,6 @@ Shards::Db:
|
|
61
61
|
net_server: blank_db_connection['host']
|
62
62
|
user: shard.username_by_server(server)
|
63
63
|
password: shard.password_by_server(server)
|
64
|
-
proxy: stage.get 'proxy'
|
65
|
-
proxy_user: ENV['PROXY_USER']
|
66
64
|
file: |
|
67
65
|
'/tmp/createdb' + dns.name + '.sh'
|
68
66
|
init_script: add "#!/bin/bash"
|
@@ -148,7 +146,8 @@ Shards::Shard:
|
|
148
146
|
[stage.downcase_location , domain , stage.name].join('_').downcase
|
149
147
|
|
150
148
|
Shards::Stage:
|
151
|
-
route53:
|
149
|
+
route53: |
|
150
|
+
[config_name + '01', downcase_location , ENV['DOMAIN']].join('.')
|
152
151
|
config_dir: |
|
153
152
|
[downcase_location, downcase_location + '_' + config_name, ENV['CONFIG_DIR_SUFFIX'] ].join('/')
|
154
153
|
blank_db: |
|
@@ -42,9 +42,35 @@ shared_steps:
|
|
42
42
|
commit_changes:
|
43
43
|
message: The changes has been committed and pushed to the remote repository.
|
44
44
|
wrong_message: We had an issue to commit and push the changes.
|
45
|
+
correct: check_fingerprints
|
46
|
+
wrong: finish
|
47
|
+
|
48
|
+
check_fingerprints:
|
49
|
+
message: The yaml files have been updated correctly in server(s).
|
50
|
+
wrong_message: Fingerprints are not matching %s
|
51
|
+
correct: application_reload_shards_task
|
52
|
+
wrong: loop_step
|
53
|
+
|
54
|
+
application_reload_shards_task:
|
55
|
+
message: Reload task have been executed successfully.
|
56
|
+
wrong_message: Errors found in the reload task %s.
|
45
57
|
correct: dns_upsert
|
46
58
|
wrong: finish
|
47
59
|
|
60
|
+
puppet_agent_update:
|
61
|
+
message: Puppet has been executed successfully.
|
62
|
+
correct: check_fingerprints
|
63
|
+
wrong_message: Errors executing the puppet update % s
|
64
|
+
wrong: check_fingerprints
|
65
|
+
|
66
|
+
loop_step:
|
67
|
+
correct: puppet_agent_update
|
68
|
+
message: The step %s will be executed after wait %d seconds.
|
69
|
+
wrong: finish
|
70
|
+
wrong_message: This is the %dth loop. The maximun is (%d).
|
71
|
+
total_loops: 3
|
72
|
+
seconds_between_loop: 180
|
73
|
+
|
48
74
|
dns_upsert:
|
49
75
|
message: The dns record with the host %s has been succesfully created.
|
50
76
|
wrong_message: Error. Could not register the host %s as a dns record.
|
@@ -0,0 +1,23 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
fingerprint_terminal:
|
5
|
+
|
6
|
+
validate_stage:
|
7
|
+
correct: check_fingerprints
|
8
|
+
wrong_message: Stage %s does not exist.
|
9
|
+
wrong: clean_repo_and_exit
|
10
|
+
|
11
|
+
correct_steps_summary:
|
12
|
+
- validate_client
|
13
|
+
- validate_domain
|
14
|
+
- validate_location
|
15
|
+
- validate_stage
|
16
|
+
- check_fingerprints
|
17
|
+
- loop_step
|
18
|
+
# It runs a puppet agent -t 3 times when the above check fails.
|
19
|
+
- application_reload_shards_task
|
20
|
+
- dns_upsert
|
21
|
+
- finish
|
22
|
+
|
23
|
+
start: validate_client
|
data/lib/shards/base.rb
CHANGED
@@ -5,6 +5,7 @@ module Shards
|
|
5
5
|
class BaseYamlObject < Shards::Base
|
6
6
|
|
7
7
|
attr_reader :stage, :yaml
|
8
|
+
attr_writer :internal_host
|
8
9
|
|
9
10
|
def initialize stage
|
10
11
|
set_config
|
@@ -18,7 +19,7 @@ module Shards
|
|
18
19
|
end
|
19
20
|
|
20
21
|
def file
|
21
|
-
file_path ENV['
|
22
|
+
file_path ENV['SHARDS_FILE_NAME']
|
22
23
|
end
|
23
24
|
|
24
25
|
def file_path file_name
|
@@ -29,6 +30,55 @@ module Shards
|
|
29
30
|
File.write file, YAML.dump(yaml)
|
30
31
|
end
|
31
32
|
|
33
|
+
def remote_file
|
34
|
+
File.join ENV['REMOTE_SERVER_CURRENT_PATH'],'config', File.basename(file)
|
35
|
+
end
|
36
|
+
|
37
|
+
def proxy_read_remote_file
|
38
|
+
ssh! command_to_read_remote
|
39
|
+
end
|
40
|
+
|
41
|
+
def ssh! command
|
42
|
+
ssh_conn do |ssh|
|
43
|
+
ssh.exec! command
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def internal_host
|
48
|
+
@internal_host || stage.internal_host
|
49
|
+
end
|
50
|
+
|
51
|
+
def command_to_read_remote
|
52
|
+
command_to_remote internal_host, fingerprint_command(remote_file)
|
53
|
+
end
|
54
|
+
|
55
|
+
def command_to_remote host, command
|
56
|
+
"ssh -q -o StrictHostKeychecking=no #{host} '#{command}'"
|
57
|
+
end
|
58
|
+
|
59
|
+
def fingerprint_command file_full_name
|
60
|
+
"md5sum #{file_full_name}"
|
61
|
+
end
|
62
|
+
|
63
|
+
def local_fingerprint
|
64
|
+
`#{fingerprint_command(file)}`.split.first
|
65
|
+
end
|
66
|
+
|
67
|
+
def remote_fingerprint
|
68
|
+
proxy_read_remote_file.split.first
|
69
|
+
end
|
70
|
+
|
71
|
+
def same_fingerprint?
|
72
|
+
local_fingerprint == remote_fingerprint
|
73
|
+
end
|
74
|
+
|
75
|
+
def fingerprints
|
76
|
+
x={}
|
77
|
+
x[:local]= local_fingerprint
|
78
|
+
x[:remote]= remote_fingerprint
|
79
|
+
x[:equal]= x[:local]==x[:remote]
|
80
|
+
x
|
81
|
+
end
|
32
82
|
end
|
33
83
|
end
|
34
84
|
|
data/lib/shards/client.rb
CHANGED
@@ -16,6 +16,16 @@ module Shards
|
|
16
16
|
|
17
17
|
end
|
18
18
|
|
19
|
+
desc 'loop_fingerprint_check LOCATION STAGE CLIENT DOMAIN', 'Run a fingerprint check in a loop and then a dns upsert.'
|
20
|
+
|
21
|
+
def loop_fingerprint_check location, stage, client, domain
|
22
|
+
|
23
|
+
fast_terminal=Shards::Workflow::FingerprintTerminal.new config
|
24
|
+
fast_terminal.params= { client: client, domain: domain, location: location.downcase, stage: stage }
|
25
|
+
fast_terminal.start
|
26
|
+
|
27
|
+
end
|
28
|
+
|
19
29
|
# desc 'create_all LOCATION CLIENT DOMAIN', 'Fast creation shard tool in all stages'
|
20
30
|
#
|
21
31
|
# def create_all location, client, domain
|
@@ -176,6 +186,36 @@ module Shards
|
|
176
186
|
|
177
187
|
end
|
178
188
|
|
189
|
+
|
190
|
+
desc "fingerprints LOCATION STAGE", "Check files fingerprints in remote sever by LOCATION and STAGE"
|
191
|
+
|
192
|
+
def fingerprints(location, stage)
|
193
|
+
|
194
|
+
s = get_stage location, stage
|
195
|
+
|
196
|
+
puts s.check_remote_process remote_check: :fingerprints
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
desc "puppet LOCATION STAGE", "Run puppet agent -t in the STAGE"
|
201
|
+
|
202
|
+
def puppet(location, stage)
|
203
|
+
|
204
|
+
s = get_stage location, stage
|
205
|
+
|
206
|
+
puts s.check_remote_process remote_check: :puppet_agent!
|
207
|
+
|
208
|
+
end
|
209
|
+
|
210
|
+
desc "reload_shards LOCATION STAGE", "Run reload shards rake task in the STAGE"
|
211
|
+
|
212
|
+
def reload_shards(location, stage)
|
213
|
+
|
214
|
+
s = get_stage location, stage
|
215
|
+
|
216
|
+
puts s.check_remote_process remote_check: :reload_shards!
|
217
|
+
|
218
|
+
end
|
179
219
|
private
|
180
220
|
|
181
221
|
def config
|
data/lib/shards/config.rb
CHANGED
@@ -7,7 +7,9 @@ module Shards
|
|
7
7
|
|
8
8
|
class Config
|
9
9
|
|
10
|
-
ENV_VARS = %w(ENGINEERING_ROOT_PATH CONFIG_FILE PROXY_SUBDOMAIN DOMAIN CONFIG_FILES_ROOT_PATH
|
10
|
+
ENV_VARS = %w(ENGINEERING_ROOT_PATH CONFIG_FILE PROXY_SUBDOMAIN DOMAIN CONFIG_FILES_ROOT_PATH ) +
|
11
|
+
%w(SHARDS_FILE_NAME SITES_FILE_NAME CONFIG_DIR_SUFFIX DNS_ZONE_CODE REMOTE_SERVER_CURRENT_PATH) +
|
12
|
+
%w(PATH_TO_ENGINEERING_YAML)
|
11
13
|
|
12
14
|
attr_accessor :locations, :settings_file, :repo
|
13
15
|
|
data/lib/shards/db.rb
CHANGED
@@ -100,11 +100,6 @@ module Shards
|
|
100
100
|
server_databases.select { |k,v| v.include?database_to_drop }.keys
|
101
101
|
end
|
102
102
|
|
103
|
-
def ssh_conn
|
104
|
-
Net::SSH.start(proxy, proxy_user) do |ssh|
|
105
|
-
yield ssh
|
106
|
-
end
|
107
|
-
end
|
108
103
|
|
109
104
|
def parse_databases_on_servers ssh_output
|
110
105
|
@server_databases[server]=ssh_output.split("\n").reject { |x| x.include?"Warning" }
|
data/lib/shards/dns.rb
CHANGED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require "aws-sdk-elasticloadbalancing"
|
3
|
+
require "aws-sdk-elasticloadbalancingv2"
|
4
|
+
require "aws-sdk-ec2"
|
5
|
+
|
6
|
+
module Shards
|
7
|
+
class IpSearch
|
8
|
+
|
9
|
+
attr_reader :stage
|
10
|
+
|
11
|
+
def initialize stage
|
12
|
+
@stage=stage
|
13
|
+
end
|
14
|
+
|
15
|
+
def base_path
|
16
|
+
File.join ENV['ENGINEERING_ROOT_PATH'], ENV['PATH_TO_ENGINEERING_YAML'] , stage.location_name
|
17
|
+
end
|
18
|
+
|
19
|
+
def file_list
|
20
|
+
Dir.glob( File.join base_path, "*production*.yml" )
|
21
|
+
end
|
22
|
+
|
23
|
+
def all_servers_file
|
24
|
+
File.join base_path , 'all_servers.yml'
|
25
|
+
end
|
26
|
+
|
27
|
+
def load_balancers
|
28
|
+
region=nil
|
29
|
+
list1=[]
|
30
|
+
list2=[]
|
31
|
+
file_list.each do |file|
|
32
|
+
yaml=YAML.load_file file
|
33
|
+
all_yaml=YAML.load_file all_servers_file
|
34
|
+
yaml=all_yaml.merge(yaml)
|
35
|
+
list1+= (yaml['loadbalancer_list'] || [] )
|
36
|
+
list2+= (yaml['loadbalancer2_list'] || [] )
|
37
|
+
region||= yaml['loadbalancer_region']
|
38
|
+
end
|
39
|
+
|
40
|
+
{ region: region, list1: list1.uniq, list2: list2.uniq }
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
def lbs
|
45
|
+
@lbs||= load_balancers
|
46
|
+
end
|
47
|
+
|
48
|
+
def lb1_instances
|
49
|
+
client=Aws::ElasticLoadBalancing::Client.new region: lbs[:region]
|
50
|
+
|
51
|
+
lbs[:list1].map do |name|
|
52
|
+
client.describe_instance_health( { load_balancer_name: name }).instance_states.select do |state|
|
53
|
+
state.state=='InService'
|
54
|
+
end.map { |x| x.instance_id }
|
55
|
+
end.flatten.uniq
|
56
|
+
end
|
57
|
+
|
58
|
+
def lb2_instances
|
59
|
+
if lbs[:list2].empty?
|
60
|
+
[]
|
61
|
+
else
|
62
|
+
target_group_instances
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def lb2_client
|
67
|
+
@lb2_client||=Aws::ElasticLoadBalancingV2::Client.new( region: lbs[:region] )
|
68
|
+
end
|
69
|
+
|
70
|
+
def lb2_arns
|
71
|
+
lb2_client.describe_load_balancers({ names: lbs[:list2] }).load_balancers.map do |lb|
|
72
|
+
lb.load_balancer_arn
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def target_groups_arns
|
77
|
+
lb2_arns.map do |arn|
|
78
|
+
lb2_client.describe_target_groups({ load_balancer_arn: arn }).target_groups.map do |tg|
|
79
|
+
tg.target_group_arn
|
80
|
+
end
|
81
|
+
end.flatten.uniq
|
82
|
+
end
|
83
|
+
|
84
|
+
def target_group_instances
|
85
|
+
target_groups_arns.map do |arn|
|
86
|
+
lb2_client.describe_target_health({ target_group_arn: arn }).target_health_descriptions.select do |d|
|
87
|
+
d.target_health.state='healthy'
|
88
|
+
end.map { |d| d.target.id }
|
89
|
+
end.flatten.uniq
|
90
|
+
end
|
91
|
+
|
92
|
+
def instances
|
93
|
+
( lb1_instances + lb2_instances ).uniq
|
94
|
+
end
|
95
|
+
|
96
|
+
def ips
|
97
|
+
client = Aws::EC2::Client.new region: lbs[:region]
|
98
|
+
client.describe_instances({ instance_ids: instances }).reservations.map do |r|
|
99
|
+
r.instances.map do |i|
|
100
|
+
i.private_ip_address
|
101
|
+
end
|
102
|
+
end.flatten.uniq
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
data/lib/shards/meta/meta.rb
CHANGED
@@ -51,6 +51,20 @@ module Shards
|
|
51
51
|
get_config(class_name).keys
|
52
52
|
end
|
53
53
|
|
54
|
+
def ssh_conn
|
55
|
+
Net::SSH.start(proxy, proxy_user) do |ssh|
|
56
|
+
yield ssh
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def proxy
|
61
|
+
stage.get 'proxy'
|
62
|
+
end
|
63
|
+
|
64
|
+
def proxy_user
|
65
|
+
ENV['PROXY_USER']
|
66
|
+
end
|
67
|
+
|
54
68
|
end
|
55
69
|
|
56
70
|
end
|
data/lib/shards/stage.rb
CHANGED
@@ -3,13 +3,12 @@ require 'shards/site'
|
|
3
3
|
require 'shards/db'
|
4
4
|
require 'shards/dns'
|
5
5
|
require 'shards/base'
|
6
|
+
require 'shards/ip_search'
|
6
7
|
|
7
8
|
module Shards
|
8
9
|
|
9
10
|
class Stage < Shards::Base
|
10
11
|
|
11
|
-
include Shards::Meta
|
12
|
-
|
13
12
|
method_list('Shards::Stage').each do |m|
|
14
13
|
define_method(m.to_sym) { run __method__ }
|
15
14
|
end
|
@@ -28,7 +27,6 @@ module Shards
|
|
28
27
|
|
29
28
|
def shard
|
30
29
|
@shard||=Shards::Shard.new self
|
31
|
-
@shard
|
32
30
|
end
|
33
31
|
|
34
32
|
def add_domain domain, dryrun: false
|
@@ -59,6 +57,49 @@ module Shards
|
|
59
57
|
@variables.keys
|
60
58
|
end
|
61
59
|
|
60
|
+
def fingerprints
|
61
|
+
site = Shards::Site.new self
|
62
|
+
{ shards: shard.fingerprints, sites: site.fingerprints }
|
63
|
+
end
|
64
|
+
|
65
|
+
def puppet_agent!
|
66
|
+
command_to_remote 'sudo puppet agent -t'
|
67
|
+
end
|
68
|
+
|
69
|
+
def reload_shards!
|
70
|
+
cmd="cd #{ENV['REMOTE_SERVER_CURRENT_PATH']} && RAILS_ENV=production bundle exec rake reload_shards"
|
71
|
+
command_to_remote cmd
|
72
|
+
end
|
73
|
+
|
74
|
+
def command_to_remote cmd
|
75
|
+
resp=shard.ssh! shard.command_to_remote( internal_host, cmd)
|
76
|
+
message=resp.exitstatus==0 ? 'successfully' : resp.to_s
|
77
|
+
{ status: resp.exitstatus, message: message }
|
78
|
+
end
|
79
|
+
|
80
|
+
def hosts_to_check
|
81
|
+
name=='prod' ? get_production_ips : [ internal_host ]
|
82
|
+
end
|
83
|
+
|
84
|
+
def get_production_ips
|
85
|
+
ip_search=Shards::IpSearch.new self
|
86
|
+
@ips||=ip_search.ips
|
87
|
+
end
|
88
|
+
|
89
|
+
def internal_host
|
90
|
+
@internal_host || route53_name
|
91
|
+
end
|
92
|
+
|
93
|
+
def route53_name
|
94
|
+
get('route53').split('.').first
|
95
|
+
end
|
96
|
+
|
97
|
+
def check_remote_process remote_check: :internal_host
|
98
|
+
hosts_to_check.map do |h|
|
99
|
+
@internal_host=h
|
100
|
+
[ h , send(remote_check) ]
|
101
|
+
end.to_h
|
102
|
+
end
|
62
103
|
end
|
63
104
|
end
|
64
105
|
|
data/lib/shards/version.rb
CHANGED
data/lib/shards/workflow/base.rb
CHANGED
@@ -9,6 +9,7 @@ module Shards
|
|
9
9
|
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params, :blank_db
|
10
10
|
|
11
11
|
attr_writer :wf
|
12
|
+
attr_reader :current_loop
|
12
13
|
|
13
14
|
def initialize config
|
14
15
|
@config=config
|
@@ -248,6 +249,46 @@ module Shards
|
|
248
249
|
@flow['write_yaml_files']['correct']='ask_blank_db'
|
249
250
|
end
|
250
251
|
|
252
|
+
def check_fingerprints
|
253
|
+
resp=stage.check_remote_process remote_check: :fingerprints
|
254
|
+
errors=filter_errors resp
|
255
|
+
raise_wrong_text errors if errors.count>0
|
256
|
+
puts step['message']
|
257
|
+
end
|
258
|
+
|
259
|
+
def loop_step
|
260
|
+
@current_loop||=0
|
261
|
+
@total_loops||=step['total_loops']
|
262
|
+
@current_loop+=1
|
263
|
+
raise_wrong_text [current_loop,@total_loops] if current_loop>@total_loops
|
264
|
+
puts step['message'] % [step['correct'], step['seconds_between_loop']]
|
265
|
+
sleep step['seconds_between_loop']
|
266
|
+
end
|
267
|
+
|
268
|
+
|
269
|
+
def application_reload_shards_task
|
270
|
+
resp=stage.check_remote_process remote_check: :reload_shards!
|
271
|
+
errors=filter_status_errors resp
|
272
|
+
raise_wrong_text errors if errors.count>0
|
273
|
+
puts step['message']
|
274
|
+
end
|
275
|
+
|
276
|
+
def puppet_agent_update
|
277
|
+
resp=stage.check_remote_process remote_check: :puppet_agent!
|
278
|
+
errors=filter_status_errors resp
|
279
|
+
raise_wrong_text errors if errors.count>0
|
280
|
+
puts step['message']
|
281
|
+
end
|
282
|
+
|
283
|
+
private
|
284
|
+
|
285
|
+
def filter_errors resp
|
286
|
+
resp.select { |k,x| x.values.count { |v| v[:remote]!=v[:local] } > 0 }
|
287
|
+
end
|
288
|
+
|
289
|
+
def filter_status_errors resp
|
290
|
+
resp.select { |k,x| x[:status]==1 }
|
291
|
+
end
|
251
292
|
end
|
252
293
|
|
253
294
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative './base.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class FingerprintTerminal < Shards::Workflow::Base
|
8
|
+
|
9
|
+
def validate_client
|
10
|
+
|
11
|
+
cli=params[:client]
|
12
|
+
|
13
|
+
validate_min_size cli, step['min_size']
|
14
|
+
|
15
|
+
@client=cli
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
def validate_domain
|
20
|
+
|
21
|
+
domain_name=params[:domain]
|
22
|
+
|
23
|
+
validate_min_size domain_name, step['min_size']
|
24
|
+
|
25
|
+
validate_regex domain_name, step['regex']
|
26
|
+
|
27
|
+
@domain=domain_name
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
def dns_upsert
|
32
|
+
stage.add_domain domain, dryrun: true
|
33
|
+
stage.shard.dns.set dryrun: false
|
34
|
+
raise_wrong_text host unless stage.shard.dns.exist?
|
35
|
+
puts step['message'] % host
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/shards.gemspec
CHANGED
@@ -39,6 +39,9 @@ Gem::Specification.new do |spec|
|
|
39
39
|
spec.add_dependency "net-ssh", "~> 5.0"
|
40
40
|
spec.add_dependency "net-scp", "~> 1.2"
|
41
41
|
spec.add_dependency "aws-sdk-route53", "~> 1.9"
|
42
|
+
spec.add_dependency "aws-sdk-elasticloadbalancing", "~> 1.5"
|
43
|
+
spec.add_dependency "aws-sdk-elasticloadbalancingv2", "~> 1.14"
|
44
|
+
spec.add_dependency "aws-sdk-ec2", "~> 1.48"
|
42
45
|
spec.add_dependency "git", "~> 1.4"
|
43
46
|
spec.add_dependency "thor", "~> 0.20"
|
44
47
|
spec.add_dependency "terminal-table", "~> 1.8"
|
data/test.env
CHANGED
@@ -1,15 +1,28 @@
|
|
1
1
|
ENGINEERING_ROOT_PATH=tmp/testrepo/shards_test_repo
|
2
|
+
|
2
3
|
CONFIG_FILE=file.yml
|
4
|
+
|
3
5
|
PROXY_SUBDOMAIN=proxy.example.com
|
6
|
+
|
4
7
|
DOMAIN=example.com
|
8
|
+
|
5
9
|
CONFIG_FILES_ROOT_PATH=path/to
|
10
|
+
|
11
|
+
PATH_TO_ENGINEERING_YAML=path/to/engineering/files
|
12
|
+
|
6
13
|
SHARDS_FILE_NAME=shards.yml
|
14
|
+
|
7
15
|
SITES_FILE_NAME=sites.yml
|
16
|
+
|
8
17
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
18
|
+
|
9
19
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
20
|
+
|
10
21
|
DNS_ZONE_CODE=TestCodeDNSZone
|
22
|
+
|
11
23
|
PROXY_USER=proxy_username
|
12
|
-
|
24
|
+
|
25
|
+
REMOTE_SERVER_CURRENT_PATH=/rails/project/current_directory
|
13
26
|
|
14
27
|
# optional:
|
15
28
|
TEST=true #only for test.env delete this var in settings.env
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diego PL
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-09-
|
11
|
+
date: 2018-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -136,6 +136,48 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '1.9'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: aws-sdk-elasticloadbalancing
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '1.5'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '1.5'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: aws-sdk-elasticloadbalancingv2
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '1.14'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '1.14'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: aws-sdk-ec2
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '1.48'
|
174
|
+
type: :runtime
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '1.48'
|
139
181
|
- !ruby/object:Gem::Dependency
|
140
182
|
name: git
|
141
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +248,7 @@ files:
|
|
206
248
|
- lib/settings/workflows/delete_by_terminal.yaml
|
207
249
|
- lib/settings/workflows/fast_terminal.yaml
|
208
250
|
- lib/settings/workflows/fast_terminal_all.yaml
|
251
|
+
- lib/settings/workflows/fingerprint_terminal.yaml
|
209
252
|
- lib/settings/workflows/terminal.yaml
|
210
253
|
- lib/shards.rb
|
211
254
|
- lib/shards/base.rb
|
@@ -214,6 +257,7 @@ files:
|
|
214
257
|
- lib/shards/config.rb
|
215
258
|
- lib/shards/db.rb
|
216
259
|
- lib/shards/dns.rb
|
260
|
+
- lib/shards/ip_search.rb
|
217
261
|
- lib/shards/location.rb
|
218
262
|
- lib/shards/meta/list.rb
|
219
263
|
- lib/shards/meta/meta.rb
|
@@ -227,6 +271,7 @@ files:
|
|
227
271
|
- lib/shards/workflow/delete_by_terminal.rb
|
228
272
|
- lib/shards/workflow/fast_terminal.rb
|
229
273
|
- lib/shards/workflow/fast_terminal_all.rb
|
274
|
+
- lib/shards/workflow/fingerprint_terminal.rb
|
230
275
|
- lib/shards/workflow/terminal.rb
|
231
276
|
- lib/shards/workflow/workflow.rb
|
232
277
|
- shards.gemspec
|