shards 2.0.1 → 2.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +14 -2
- data/README.md +21 -0
- data/lib/settings/format_methods.yaml +2 -3
- data/lib/settings/workflows/base.yaml +26 -0
- data/lib/settings/workflows/fast_terminal.yaml +3 -0
- data/lib/settings/workflows/fingerprint_terminal.yaml +23 -0
- data/lib/shards/base.rb +3 -0
- data/lib/shards/base_yaml_object.rb +51 -1
- data/lib/shards/client.rb +40 -0
- data/lib/shards/config.rb +3 -1
- data/lib/shards/db.rb +0 -5
- data/lib/shards/dns.rb +1 -1
- data/lib/shards/ip_search.rb +107 -0
- data/lib/shards/meta/meta.rb +14 -0
- data/lib/shards/stage.rb +44 -3
- data/lib/shards/version.rb +1 -1
- data/lib/shards/workflow/base.rb +41 -0
- data/lib/shards/workflow/fast_terminal.rb +1 -1
- data/lib/shards/workflow/fingerprint_terminal.rb +42 -0
- data/lib/shards/workflow/terminal.rb +1 -1
- data/lib/shards/workflow/workflow.rb +1 -0
- data/shards.gemspec +3 -0
- data/test.env +14 -1
- metadata +47 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 84a189cde1dd20ee0c703592580ddccbca10bdc5
|
4
|
+
data.tar.gz: bc439f40ae05dcdc97111c061933aa01da8e257a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d7dcbd6cfdb02e578542507c18187aeb76f16292e48bae745311960fb79be9124d427ff4ce1943590bafb68ea67141064c746efbe96b37ba05973a744afb38cf
|
7
|
+
data.tar.gz: 34676aa6982b0d2b532e7087815ab74d141f60d6ccab91f5b41aef3a158fe8a336ff1c228b022267e894bee95dbb18c6bfec424cbef11bdf11ca3392885665c5
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,10 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
shards (2.
|
4
|
+
shards (2.1.6)
|
5
|
+
aws-sdk-ec2 (~> 1.48)
|
6
|
+
aws-sdk-elasticloadbalancing (~> 1.5)
|
7
|
+
aws-sdk-elasticloadbalancingv2 (~> 1.14)
|
5
8
|
aws-sdk-route53 (~> 1.9)
|
6
9
|
dotenv (~> 2.4)
|
7
10
|
git (~> 1.4)
|
@@ -21,12 +24,21 @@ GEM
|
|
21
24
|
rspec-expectations (>= 2.99)
|
22
25
|
thor (~> 0.19)
|
23
26
|
aws-eventstream (1.0.1)
|
24
|
-
aws-partitions (1.
|
27
|
+
aws-partitions (1.104.0)
|
25
28
|
aws-sdk-core (3.27.0)
|
26
29
|
aws-eventstream (~> 1.0)
|
27
30
|
aws-partitions (~> 1.0)
|
28
31
|
aws-sigv4 (~> 1.0)
|
29
32
|
jmespath (~> 1.0)
|
33
|
+
aws-sdk-ec2 (1.48.0)
|
34
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
35
|
+
aws-sigv4 (~> 1.0)
|
36
|
+
aws-sdk-elasticloadbalancing (1.5.0)
|
37
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
38
|
+
aws-sigv4 (~> 1.0)
|
39
|
+
aws-sdk-elasticloadbalancingv2 (1.14.0)
|
40
|
+
aws-sdk-core (~> 3, >= 3.26.0)
|
41
|
+
aws-sigv4 (~> 1.0)
|
30
42
|
aws-sdk-route53 (1.12.0)
|
31
43
|
aws-sdk-core (~> 3, >= 3.26.0)
|
32
44
|
aws-sigv4 (~> 1.0)
|
data/README.md
CHANGED
@@ -30,26 +30,47 @@ Add a settings.env file in your root application directory with the variables:
|
|
30
30
|
```bash
|
31
31
|
|
32
32
|
ENGINEERING_ROOT_PATH=/your/root/path/to/engineering/repo/project
|
33
|
+
|
33
34
|
CONFIG_FILE=relative/path/to/config.yml
|
35
|
+
|
34
36
|
PROXY_SUBDOMAIN=proxy.example.com
|
37
|
+
|
35
38
|
DOMAIN=example.com
|
39
|
+
|
36
40
|
CONFIG_FILES_ROOT_PATH=puppet/environments/common/bbug_config/files
|
41
|
+
|
42
|
+
PATH_TO_ENGINEERING_YAML=puppet/environments/common/bbug_engineering_config/files
|
43
|
+
|
37
44
|
SHARDS_FILE_NAME=shards.yml
|
45
|
+
|
38
46
|
SITES_FILE_NAME=sites.yml
|
47
|
+
|
39
48
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
49
|
+
|
40
50
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
51
|
+
|
41
52
|
DNS_ZONE_CODE=DNS Zone Code
|
53
|
+
|
42
54
|
PROXY_USER=proxy_username
|
43
55
|
|
56
|
+
REMOTE_SERVER_CURRENT_PATH=/rails/project/current_directory
|
57
|
+
|
44
58
|
# optional:
|
59
|
+
|
45
60
|
TEST=true #only for test.env delete this var in settings.env
|
61
|
+
|
46
62
|
CONFIG_REPLACE={ 'prod' => 'web' }
|
63
|
+
|
47
64
|
LIVE_BRANCH=shards_test # set this var to use in test.env or to change the default value 'v5'
|
65
|
+
|
48
66
|
WORKFLOW_YAML_PATH=path/to/your/workflows/ # you could overwrite yaml files using the same name in the defined path.
|
49
67
|
|
68
|
+
|
50
69
|
# You could overwrite methods using this metaprograming way ading your on your_file.yaml
|
51
70
|
# Then add this var:
|
71
|
+
|
52
72
|
# FORMAT_METHODS_FILE=your/path/to/your_file.yaml
|
73
|
+
|
53
74
|
# Finally to overwrite the default behavior write your code in your_file.yaml. The yaml will be merged so you need define just the methods thay you need overwrite.
|
54
75
|
# The default definitions are in spec/support/format_methods.yaml
|
55
76
|
|
@@ -61,8 +61,6 @@ Shards::Db:
|
|
61
61
|
net_server: blank_db_connection['host']
|
62
62
|
user: shard.username_by_server(server)
|
63
63
|
password: shard.password_by_server(server)
|
64
|
-
proxy: stage.get 'proxy'
|
65
|
-
proxy_user: ENV['PROXY_USER']
|
66
64
|
file: |
|
67
65
|
'/tmp/createdb' + dns.name + '.sh'
|
68
66
|
init_script: add "#!/bin/bash"
|
@@ -148,7 +146,8 @@ Shards::Shard:
|
|
148
146
|
[stage.downcase_location , domain , stage.name].join('_').downcase
|
149
147
|
|
150
148
|
Shards::Stage:
|
151
|
-
route53:
|
149
|
+
route53: |
|
150
|
+
[config_name + '01', downcase_location , ENV['DOMAIN']].join('.')
|
152
151
|
config_dir: |
|
153
152
|
[downcase_location, downcase_location + '_' + config_name, ENV['CONFIG_DIR_SUFFIX'] ].join('/')
|
154
153
|
blank_db: |
|
@@ -42,9 +42,35 @@ shared_steps:
|
|
42
42
|
commit_changes:
|
43
43
|
message: The changes has been committed and pushed to the remote repository.
|
44
44
|
wrong_message: We had an issue to commit and push the changes.
|
45
|
+
correct: check_fingerprints
|
46
|
+
wrong: finish
|
47
|
+
|
48
|
+
check_fingerprints:
|
49
|
+
message: The yaml files have been updated correctly in server(s).
|
50
|
+
wrong_message: Fingerprints are not matching %s
|
51
|
+
correct: application_reload_shards_task
|
52
|
+
wrong: loop_step
|
53
|
+
|
54
|
+
application_reload_shards_task:
|
55
|
+
message: Reload task have been executed successfully.
|
56
|
+
wrong_message: Errors found in the reload task %s.
|
45
57
|
correct: dns_upsert
|
46
58
|
wrong: finish
|
47
59
|
|
60
|
+
puppet_agent_update:
|
61
|
+
message: Puppet has been executed successfully.
|
62
|
+
correct: check_fingerprints
|
63
|
+
wrong_message: Errors executing the puppet update % s
|
64
|
+
wrong: check_fingerprints
|
65
|
+
|
66
|
+
loop_step:
|
67
|
+
correct: puppet_agent_update
|
68
|
+
message: The step %s will be executed after wait %d seconds.
|
69
|
+
wrong: finish
|
70
|
+
wrong_message: This is the %dth loop. The maximun is (%d).
|
71
|
+
total_loops: 3
|
72
|
+
seconds_between_loop: 180
|
73
|
+
|
48
74
|
dns_upsert:
|
49
75
|
message: The dns record with the host %s has been succesfully created.
|
50
76
|
wrong_message: Error. Could not register the host %s as a dns record.
|
@@ -0,0 +1,23 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
fingerprint_terminal:
|
5
|
+
|
6
|
+
validate_stage:
|
7
|
+
correct: check_fingerprints
|
8
|
+
wrong_message: Stage %s does not exist.
|
9
|
+
wrong: clean_repo_and_exit
|
10
|
+
|
11
|
+
correct_steps_summary:
|
12
|
+
- validate_client
|
13
|
+
- validate_domain
|
14
|
+
- validate_location
|
15
|
+
- validate_stage
|
16
|
+
- check_fingerprints
|
17
|
+
- loop_step
|
18
|
+
# It runs a puppet agent -t 3 times when the above check fails.
|
19
|
+
- application_reload_shards_task
|
20
|
+
- dns_upsert
|
21
|
+
- finish
|
22
|
+
|
23
|
+
start: validate_client
|
data/lib/shards/base.rb
CHANGED
@@ -5,6 +5,7 @@ module Shards
|
|
5
5
|
class BaseYamlObject < Shards::Base
|
6
6
|
|
7
7
|
attr_reader :stage, :yaml
|
8
|
+
attr_writer :internal_host
|
8
9
|
|
9
10
|
def initialize stage
|
10
11
|
set_config
|
@@ -18,7 +19,7 @@ module Shards
|
|
18
19
|
end
|
19
20
|
|
20
21
|
def file
|
21
|
-
file_path ENV['
|
22
|
+
file_path ENV['SHARDS_FILE_NAME']
|
22
23
|
end
|
23
24
|
|
24
25
|
def file_path file_name
|
@@ -29,6 +30,55 @@ module Shards
|
|
29
30
|
File.write file, YAML.dump(yaml)
|
30
31
|
end
|
31
32
|
|
33
|
+
def remote_file
|
34
|
+
File.join ENV['REMOTE_SERVER_CURRENT_PATH'],'config', File.basename(file)
|
35
|
+
end
|
36
|
+
|
37
|
+
def proxy_read_remote_file
|
38
|
+
ssh! command_to_read_remote
|
39
|
+
end
|
40
|
+
|
41
|
+
def ssh! command
|
42
|
+
ssh_conn do |ssh|
|
43
|
+
ssh.exec! command
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def internal_host
|
48
|
+
@internal_host || stage.internal_host
|
49
|
+
end
|
50
|
+
|
51
|
+
def command_to_read_remote
|
52
|
+
command_to_remote internal_host, fingerprint_command(remote_file)
|
53
|
+
end
|
54
|
+
|
55
|
+
def command_to_remote host, command
|
56
|
+
"ssh -q -o StrictHostKeychecking=no #{host} '#{command}'"
|
57
|
+
end
|
58
|
+
|
59
|
+
def fingerprint_command file_full_name
|
60
|
+
"md5sum #{file_full_name}"
|
61
|
+
end
|
62
|
+
|
63
|
+
def local_fingerprint
|
64
|
+
`#{fingerprint_command(file)}`.split.first
|
65
|
+
end
|
66
|
+
|
67
|
+
def remote_fingerprint
|
68
|
+
proxy_read_remote_file.split.first
|
69
|
+
end
|
70
|
+
|
71
|
+
def same_fingerprint?
|
72
|
+
local_fingerprint == remote_fingerprint
|
73
|
+
end
|
74
|
+
|
75
|
+
def fingerprints
|
76
|
+
x={}
|
77
|
+
x[:local]= local_fingerprint
|
78
|
+
x[:remote]= remote_fingerprint
|
79
|
+
x[:equal]= x[:local]==x[:remote]
|
80
|
+
x
|
81
|
+
end
|
32
82
|
end
|
33
83
|
end
|
34
84
|
|
data/lib/shards/client.rb
CHANGED
@@ -16,6 +16,16 @@ module Shards
|
|
16
16
|
|
17
17
|
end
|
18
18
|
|
19
|
+
desc 'loop_fingerprint_check LOCATION STAGE CLIENT DOMAIN', 'Run a fingerprint check in a loop and then a dns upsert.'
|
20
|
+
|
21
|
+
def loop_fingerprint_check location, stage, client, domain
|
22
|
+
|
23
|
+
fast_terminal=Shards::Workflow::FingerprintTerminal.new config
|
24
|
+
fast_terminal.params= { client: client, domain: domain, location: location.downcase, stage: stage }
|
25
|
+
fast_terminal.start
|
26
|
+
|
27
|
+
end
|
28
|
+
|
19
29
|
# desc 'create_all LOCATION CLIENT DOMAIN', 'Fast creation shard tool in all stages'
|
20
30
|
#
|
21
31
|
# def create_all location, client, domain
|
@@ -176,6 +186,36 @@ module Shards
|
|
176
186
|
|
177
187
|
end
|
178
188
|
|
189
|
+
|
190
|
+
desc "fingerprints LOCATION STAGE", "Check files fingerprints in remote sever by LOCATION and STAGE"
|
191
|
+
|
192
|
+
def fingerprints(location, stage)
|
193
|
+
|
194
|
+
s = get_stage location, stage
|
195
|
+
|
196
|
+
puts s.check_remote_process remote_check: :fingerprints
|
197
|
+
|
198
|
+
end
|
199
|
+
|
200
|
+
desc "puppet LOCATION STAGE", "Run puppet agent -t in the STAGE"
|
201
|
+
|
202
|
+
def puppet(location, stage)
|
203
|
+
|
204
|
+
s = get_stage location, stage
|
205
|
+
|
206
|
+
puts s.check_remote_process remote_check: :puppet_agent!
|
207
|
+
|
208
|
+
end
|
209
|
+
|
210
|
+
desc "reload_shards LOCATION STAGE", "Run reload shards rake task in the STAGE"
|
211
|
+
|
212
|
+
def reload_shards(location, stage)
|
213
|
+
|
214
|
+
s = get_stage location, stage
|
215
|
+
|
216
|
+
puts s.check_remote_process remote_check: :reload_shards!
|
217
|
+
|
218
|
+
end
|
179
219
|
private
|
180
220
|
|
181
221
|
def config
|
data/lib/shards/config.rb
CHANGED
@@ -7,7 +7,9 @@ module Shards
|
|
7
7
|
|
8
8
|
class Config
|
9
9
|
|
10
|
-
ENV_VARS = %w(ENGINEERING_ROOT_PATH CONFIG_FILE PROXY_SUBDOMAIN DOMAIN CONFIG_FILES_ROOT_PATH
|
10
|
+
ENV_VARS = %w(ENGINEERING_ROOT_PATH CONFIG_FILE PROXY_SUBDOMAIN DOMAIN CONFIG_FILES_ROOT_PATH ) +
|
11
|
+
%w(SHARDS_FILE_NAME SITES_FILE_NAME CONFIG_DIR_SUFFIX DNS_ZONE_CODE REMOTE_SERVER_CURRENT_PATH) +
|
12
|
+
%w(PATH_TO_ENGINEERING_YAML)
|
11
13
|
|
12
14
|
attr_accessor :locations, :settings_file, :repo
|
13
15
|
|
data/lib/shards/db.rb
CHANGED
@@ -100,11 +100,6 @@ module Shards
|
|
100
100
|
server_databases.select { |k,v| v.include?database_to_drop }.keys
|
101
101
|
end
|
102
102
|
|
103
|
-
def ssh_conn
|
104
|
-
Net::SSH.start(proxy, proxy_user) do |ssh|
|
105
|
-
yield ssh
|
106
|
-
end
|
107
|
-
end
|
108
103
|
|
109
104
|
def parse_databases_on_servers ssh_output
|
110
105
|
@server_databases[server]=ssh_output.split("\n").reject { |x| x.include?"Warning" }
|
data/lib/shards/dns.rb
CHANGED
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require "aws-sdk-elasticloadbalancing"
|
3
|
+
require "aws-sdk-elasticloadbalancingv2"
|
4
|
+
require "aws-sdk-ec2"
|
5
|
+
|
6
|
+
module Shards
|
7
|
+
class IpSearch
|
8
|
+
|
9
|
+
attr_reader :stage
|
10
|
+
|
11
|
+
def initialize stage
|
12
|
+
@stage=stage
|
13
|
+
end
|
14
|
+
|
15
|
+
def base_path
|
16
|
+
File.join ENV['ENGINEERING_ROOT_PATH'], ENV['PATH_TO_ENGINEERING_YAML'] , stage.location_name
|
17
|
+
end
|
18
|
+
|
19
|
+
def file_list
|
20
|
+
Dir.glob( File.join base_path, "*production*.yml" )
|
21
|
+
end
|
22
|
+
|
23
|
+
def all_servers_file
|
24
|
+
File.join base_path , 'all_servers.yml'
|
25
|
+
end
|
26
|
+
|
27
|
+
def load_balancers
|
28
|
+
region=nil
|
29
|
+
list1=[]
|
30
|
+
list2=[]
|
31
|
+
file_list.each do |file|
|
32
|
+
yaml=YAML.load_file file
|
33
|
+
all_yaml=YAML.load_file all_servers_file
|
34
|
+
yaml=all_yaml.merge(yaml)
|
35
|
+
list1+= (yaml['loadbalancer_list'] || [] )
|
36
|
+
list2+= (yaml['loadbalancer2_list'] || [] )
|
37
|
+
region||= yaml['loadbalancer_region']
|
38
|
+
end
|
39
|
+
|
40
|
+
{ region: region, list1: list1.uniq, list2: list2.uniq }
|
41
|
+
|
42
|
+
end
|
43
|
+
|
44
|
+
def lbs
|
45
|
+
@lbs||= load_balancers
|
46
|
+
end
|
47
|
+
|
48
|
+
def lb1_instances
|
49
|
+
client=Aws::ElasticLoadBalancing::Client.new region: lbs[:region]
|
50
|
+
|
51
|
+
lbs[:list1].map do |name|
|
52
|
+
client.describe_instance_health( { load_balancer_name: name }).instance_states.select do |state|
|
53
|
+
state.state=='InService'
|
54
|
+
end.map { |x| x.instance_id }
|
55
|
+
end.flatten.uniq
|
56
|
+
end
|
57
|
+
|
58
|
+
def lb2_instances
|
59
|
+
if lbs[:list2].empty?
|
60
|
+
[]
|
61
|
+
else
|
62
|
+
target_group_instances
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def lb2_client
|
67
|
+
@lb2_client||=Aws::ElasticLoadBalancingV2::Client.new( region: lbs[:region] )
|
68
|
+
end
|
69
|
+
|
70
|
+
def lb2_arns
|
71
|
+
lb2_client.describe_load_balancers({ names: lbs[:list2] }).load_balancers.map do |lb|
|
72
|
+
lb.load_balancer_arn
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def target_groups_arns
|
77
|
+
lb2_arns.map do |arn|
|
78
|
+
lb2_client.describe_target_groups({ load_balancer_arn: arn }).target_groups.map do |tg|
|
79
|
+
tg.target_group_arn
|
80
|
+
end
|
81
|
+
end.flatten.uniq
|
82
|
+
end
|
83
|
+
|
84
|
+
def target_group_instances
|
85
|
+
target_groups_arns.map do |arn|
|
86
|
+
lb2_client.describe_target_health({ target_group_arn: arn }).target_health_descriptions.select do |d|
|
87
|
+
d.target_health.state='healthy'
|
88
|
+
end.map { |d| d.target.id }
|
89
|
+
end.flatten.uniq
|
90
|
+
end
|
91
|
+
|
92
|
+
def instances
|
93
|
+
( lb1_instances + lb2_instances ).uniq
|
94
|
+
end
|
95
|
+
|
96
|
+
def ips
|
97
|
+
client = Aws::EC2::Client.new region: lbs[:region]
|
98
|
+
client.describe_instances({ instance_ids: instances }).reservations.map do |r|
|
99
|
+
r.instances.map do |i|
|
100
|
+
i.private_ip_address
|
101
|
+
end
|
102
|
+
end.flatten.uniq
|
103
|
+
end
|
104
|
+
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
data/lib/shards/meta/meta.rb
CHANGED
@@ -51,6 +51,20 @@ module Shards
|
|
51
51
|
get_config(class_name).keys
|
52
52
|
end
|
53
53
|
|
54
|
+
def ssh_conn
|
55
|
+
Net::SSH.start(proxy, proxy_user) do |ssh|
|
56
|
+
yield ssh
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def proxy
|
61
|
+
stage.get 'proxy'
|
62
|
+
end
|
63
|
+
|
64
|
+
def proxy_user
|
65
|
+
ENV['PROXY_USER']
|
66
|
+
end
|
67
|
+
|
54
68
|
end
|
55
69
|
|
56
70
|
end
|
data/lib/shards/stage.rb
CHANGED
@@ -3,13 +3,12 @@ require 'shards/site'
|
|
3
3
|
require 'shards/db'
|
4
4
|
require 'shards/dns'
|
5
5
|
require 'shards/base'
|
6
|
+
require 'shards/ip_search'
|
6
7
|
|
7
8
|
module Shards
|
8
9
|
|
9
10
|
class Stage < Shards::Base
|
10
11
|
|
11
|
-
include Shards::Meta
|
12
|
-
|
13
12
|
method_list('Shards::Stage').each do |m|
|
14
13
|
define_method(m.to_sym) { run __method__ }
|
15
14
|
end
|
@@ -28,7 +27,6 @@ module Shards
|
|
28
27
|
|
29
28
|
def shard
|
30
29
|
@shard||=Shards::Shard.new self
|
31
|
-
@shard
|
32
30
|
end
|
33
31
|
|
34
32
|
def add_domain domain, dryrun: false
|
@@ -59,6 +57,49 @@ module Shards
|
|
59
57
|
@variables.keys
|
60
58
|
end
|
61
59
|
|
60
|
+
def fingerprints
|
61
|
+
site = Shards::Site.new self
|
62
|
+
{ shards: shard.fingerprints, sites: site.fingerprints }
|
63
|
+
end
|
64
|
+
|
65
|
+
def puppet_agent!
|
66
|
+
command_to_remote 'sudo puppet agent -t'
|
67
|
+
end
|
68
|
+
|
69
|
+
def reload_shards!
|
70
|
+
cmd="cd #{ENV['REMOTE_SERVER_CURRENT_PATH']} && RAILS_ENV=production bundle exec rake reload_shards"
|
71
|
+
command_to_remote cmd
|
72
|
+
end
|
73
|
+
|
74
|
+
def command_to_remote cmd
|
75
|
+
resp=shard.ssh! shard.command_to_remote( internal_host, cmd)
|
76
|
+
message=resp.exitstatus==0 ? 'successfully' : resp.to_s
|
77
|
+
{ status: resp.exitstatus, message: message }
|
78
|
+
end
|
79
|
+
|
80
|
+
def hosts_to_check
|
81
|
+
name=='prod' ? get_production_ips : [ internal_host ]
|
82
|
+
end
|
83
|
+
|
84
|
+
def get_production_ips
|
85
|
+
ip_search=Shards::IpSearch.new self
|
86
|
+
@ips||=ip_search.ips
|
87
|
+
end
|
88
|
+
|
89
|
+
def internal_host
|
90
|
+
@internal_host || route53_name
|
91
|
+
end
|
92
|
+
|
93
|
+
def route53_name
|
94
|
+
get('route53').split('.').first
|
95
|
+
end
|
96
|
+
|
97
|
+
def check_remote_process remote_check: :internal_host
|
98
|
+
hosts_to_check.map do |h|
|
99
|
+
@internal_host=h
|
100
|
+
[ h , send(remote_check) ]
|
101
|
+
end.to_h
|
102
|
+
end
|
62
103
|
end
|
63
104
|
end
|
64
105
|
|
data/lib/shards/version.rb
CHANGED
data/lib/shards/workflow/base.rb
CHANGED
@@ -9,6 +9,7 @@ module Shards
|
|
9
9
|
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params, :blank_db
|
10
10
|
|
11
11
|
attr_writer :wf
|
12
|
+
attr_reader :current_loop
|
12
13
|
|
13
14
|
def initialize config
|
14
15
|
@config=config
|
@@ -248,6 +249,46 @@ module Shards
|
|
248
249
|
@flow['write_yaml_files']['correct']='ask_blank_db'
|
249
250
|
end
|
250
251
|
|
252
|
+
def check_fingerprints
|
253
|
+
resp=stage.check_remote_process remote_check: :fingerprints
|
254
|
+
errors=filter_errors resp
|
255
|
+
raise_wrong_text errors if errors.count>0
|
256
|
+
puts step['message']
|
257
|
+
end
|
258
|
+
|
259
|
+
def loop_step
|
260
|
+
@current_loop||=0
|
261
|
+
@total_loops||=step['total_loops']
|
262
|
+
@current_loop+=1
|
263
|
+
raise_wrong_text [current_loop,@total_loops] if current_loop>@total_loops
|
264
|
+
puts step['message'] % [step['correct'], step['seconds_between_loop']]
|
265
|
+
sleep step['seconds_between_loop']
|
266
|
+
end
|
267
|
+
|
268
|
+
|
269
|
+
def application_reload_shards_task
|
270
|
+
resp=stage.check_remote_process remote_check: :reload_shards!
|
271
|
+
errors=filter_status_errors resp
|
272
|
+
raise_wrong_text errors if errors.count>0
|
273
|
+
puts step['message']
|
274
|
+
end
|
275
|
+
|
276
|
+
def puppet_agent_update
|
277
|
+
resp=stage.check_remote_process remote_check: :puppet_agent!
|
278
|
+
errors=filter_status_errors resp
|
279
|
+
raise_wrong_text errors if errors.count>0
|
280
|
+
puts step['message']
|
281
|
+
end
|
282
|
+
|
283
|
+
private
|
284
|
+
|
285
|
+
def filter_errors resp
|
286
|
+
resp.select { |k,x| x.values.count { |v| v[:remote]!=v[:local] } > 0 }
|
287
|
+
end
|
288
|
+
|
289
|
+
def filter_status_errors resp
|
290
|
+
resp.select { |k,x| x[:status]==1 }
|
291
|
+
end
|
251
292
|
end
|
252
293
|
|
253
294
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require_relative './base.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class FingerprintTerminal < Shards::Workflow::Base
|
8
|
+
|
9
|
+
def validate_client
|
10
|
+
|
11
|
+
cli=params[:client]
|
12
|
+
|
13
|
+
validate_min_size cli, step['min_size']
|
14
|
+
|
15
|
+
@client=cli
|
16
|
+
|
17
|
+
end
|
18
|
+
|
19
|
+
def validate_domain
|
20
|
+
|
21
|
+
domain_name=params[:domain]
|
22
|
+
|
23
|
+
validate_min_size domain_name, step['min_size']
|
24
|
+
|
25
|
+
validate_regex domain_name, step['regex']
|
26
|
+
|
27
|
+
@domain=domain_name
|
28
|
+
|
29
|
+
end
|
30
|
+
|
31
|
+
def dns_upsert
|
32
|
+
stage.add_domain domain, dryrun: true
|
33
|
+
stage.shard.dns.set dryrun: false
|
34
|
+
raise_wrong_text host unless stage.shard.dns.exist?
|
35
|
+
puts step['message'] % host
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
data/shards.gemspec
CHANGED
@@ -39,6 +39,9 @@ Gem::Specification.new do |spec|
|
|
39
39
|
spec.add_dependency "net-ssh", "~> 5.0"
|
40
40
|
spec.add_dependency "net-scp", "~> 1.2"
|
41
41
|
spec.add_dependency "aws-sdk-route53", "~> 1.9"
|
42
|
+
spec.add_dependency "aws-sdk-elasticloadbalancing", "~> 1.5"
|
43
|
+
spec.add_dependency "aws-sdk-elasticloadbalancingv2", "~> 1.14"
|
44
|
+
spec.add_dependency "aws-sdk-ec2", "~> 1.48"
|
42
45
|
spec.add_dependency "git", "~> 1.4"
|
43
46
|
spec.add_dependency "thor", "~> 0.20"
|
44
47
|
spec.add_dependency "terminal-table", "~> 1.8"
|
data/test.env
CHANGED
@@ -1,15 +1,28 @@
|
|
1
1
|
ENGINEERING_ROOT_PATH=tmp/testrepo/shards_test_repo
|
2
|
+
|
2
3
|
CONFIG_FILE=file.yml
|
4
|
+
|
3
5
|
PROXY_SUBDOMAIN=proxy.example.com
|
6
|
+
|
4
7
|
DOMAIN=example.com
|
8
|
+
|
5
9
|
CONFIG_FILES_ROOT_PATH=path/to
|
10
|
+
|
11
|
+
PATH_TO_ENGINEERING_YAML=path/to/engineering/files
|
12
|
+
|
6
13
|
SHARDS_FILE_NAME=shards.yml
|
14
|
+
|
7
15
|
SITES_FILE_NAME=sites.yml
|
16
|
+
|
8
17
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
18
|
+
|
9
19
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
20
|
+
|
10
21
|
DNS_ZONE_CODE=TestCodeDNSZone
|
22
|
+
|
11
23
|
PROXY_USER=proxy_username
|
12
|
-
|
24
|
+
|
25
|
+
REMOTE_SERVER_CURRENT_PATH=/rails/project/current_directory
|
13
26
|
|
14
27
|
# optional:
|
15
28
|
TEST=true #only for test.env delete this var in settings.env
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.1.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diego PL
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-09-
|
11
|
+
date: 2018-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -136,6 +136,48 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '1.9'
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: aws-sdk-elasticloadbalancing
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '1.5'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '1.5'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: aws-sdk-elasticloadbalancingv2
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '1.14'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '1.14'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: aws-sdk-ec2
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '1.48'
|
174
|
+
type: :runtime
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '1.48'
|
139
181
|
- !ruby/object:Gem::Dependency
|
140
182
|
name: git
|
141
183
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +248,7 @@ files:
|
|
206
248
|
- lib/settings/workflows/delete_by_terminal.yaml
|
207
249
|
- lib/settings/workflows/fast_terminal.yaml
|
208
250
|
- lib/settings/workflows/fast_terminal_all.yaml
|
251
|
+
- lib/settings/workflows/fingerprint_terminal.yaml
|
209
252
|
- lib/settings/workflows/terminal.yaml
|
210
253
|
- lib/shards.rb
|
211
254
|
- lib/shards/base.rb
|
@@ -214,6 +257,7 @@ files:
|
|
214
257
|
- lib/shards/config.rb
|
215
258
|
- lib/shards/db.rb
|
216
259
|
- lib/shards/dns.rb
|
260
|
+
- lib/shards/ip_search.rb
|
217
261
|
- lib/shards/location.rb
|
218
262
|
- lib/shards/meta/list.rb
|
219
263
|
- lib/shards/meta/meta.rb
|
@@ -227,6 +271,7 @@ files:
|
|
227
271
|
- lib/shards/workflow/delete_by_terminal.rb
|
228
272
|
- lib/shards/workflow/fast_terminal.rb
|
229
273
|
- lib/shards/workflow/fast_terminal_all.rb
|
274
|
+
- lib/shards/workflow/fingerprint_terminal.rb
|
230
275
|
- lib/shards/workflow/terminal.rb
|
231
276
|
- lib/shards/workflow/workflow.rb
|
232
277
|
- shards.gemspec
|