shards 1.2.6 → 2.0.1.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/:w +50 -0
- data/Gemfile.lock +1 -1
- data/README.md +1 -2
- data/lib/settings/format_methods.yaml +18 -15
- data/lib/settings/workflows/base.yaml +27 -0
- data/lib/settings/workflows/create_by_api.yaml +28 -0
- data/lib/settings/workflows/delete_by_terminal.yaml +11 -3
- data/lib/settings/workflows/fast_terminal.yaml +14 -17
- data/lib/settings/workflows/fast_terminal_all.yaml +16 -0
- data/lib/settings/workflows/terminal.yaml +1 -1
- data/lib/shards/client.rb +41 -10
- data/lib/shards/config.rb +1 -1
- data/lib/shards/db.rb +39 -9
- data/lib/shards/dns.rb +21 -5
- data/lib/shards/shard.rb +32 -1
- data/lib/shards/stage.rb +2 -2
- data/lib/shards/version.rb +1 -1
- data/lib/shards/workflow/base.rb +24 -3
- data/lib/shards/workflow/create_by_api.rb +22 -0
- data/lib/shards/workflow/delete_by_terminal.rb +55 -25
- data/lib/shards/workflow/fast_terminal_all.rb +24 -0
- data/lib/shards/workflow/terminal.rb +1 -0
- data/lib/shards/workflow/workflow.rb +1 -0
- data/test.env +1 -2
- metadata +9 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35764b0efc264216ed213b070745bcc8f0fd6cf6
|
4
|
+
data.tar.gz: 5409b1150b4e8e76c100f5469bf3f2e1fe015f4f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d46bc308507992fa107d586946008e6d11361fbaedddbe9746b2c765eecb9d3700bafbe2c69fc656d26dd191009fe2559a88f91915a5fe97e3b00af4f9746060
|
7
|
+
data.tar.gz: a95761c3bebdc6b120bb4984bd63cdfdfdcc6c54c00b6baa7ffdf67084c8bc37a9afed40de06866614af173e85239cf5b81c70818a26bf1adb71609ca7b74bbe
|
data/:w
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
delete_by_terminal:
|
5
|
+
|
6
|
+
validate_stage:
|
7
|
+
correct: validate_subdomain
|
8
|
+
wrong_message: Stage %s does not exist.
|
9
|
+
wrong: finish
|
10
|
+
|
11
|
+
validate_subdomain:
|
12
|
+
correct: delete_record
|
13
|
+
wrong_message: DNS record for host %s, %s.
|
14
|
+
wrong: ask_to_continue
|
15
|
+
|
16
|
+
ask_to_continue:
|
17
|
+
question: Do you want to continue?
|
18
|
+
correct: delete_record
|
19
|
+
wrong: finish
|
20
|
+
|
21
|
+
delete_record:
|
22
|
+
message: Record %s deleted in target %s
|
23
|
+
correct: delete_site_and_shard
|
24
|
+
wrong: finish
|
25
|
+
wrong_message: Could not delete the record %s.
|
26
|
+
|
27
|
+
delete_site_and_shard:
|
28
|
+
correct: write_yaml_files
|
29
|
+
wrong: clean_repo_and_exit
|
30
|
+
wrong_message: Could not delete site or shard.\n %s.
|
31
|
+
|
32
|
+
write_yaml_files:
|
33
|
+
question: Type Y to confirm the changes
|
34
|
+
wrong_message: Undo changes
|
35
|
+
correct: commit_changes
|
36
|
+
wrong: clean_repo_and_exit
|
37
|
+
|
38
|
+
commit_changes:
|
39
|
+
message: The changes has been committed and pushed to the remote repository.
|
40
|
+
wrong_message: We had an issue to commit and push the changes.
|
41
|
+
correct: delete_database
|
42
|
+
wrong: finish
|
43
|
+
|
44
|
+
delete_database:
|
45
|
+
wrong_message: The database %s could not be deleted in host %s.
|
46
|
+
message: Database %s was deleted in host %s.
|
47
|
+
correct: finish
|
48
|
+
wrong: finish
|
49
|
+
|
50
|
+
start: validate_location
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -12,6 +12,7 @@ Add this line to your application's Gemfile:
|
|
12
12
|
|
13
13
|
```ruby
|
14
14
|
gem 'shards'
|
15
|
+
|
15
16
|
```
|
16
17
|
|
17
18
|
And then execute:
|
@@ -38,8 +39,6 @@ SITES_FILE_NAME=sites.yml
|
|
38
39
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
39
40
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
40
41
|
DNS_ZONE_CODE=DNS Zone Code
|
41
|
-
DB_USER=db_user_name
|
42
|
-
DB_PASSWORD=HerePasswordDB
|
43
42
|
PROXY_USER=proxy_username
|
44
43
|
|
45
44
|
# optional:
|
@@ -25,6 +25,10 @@ Shards::Dns:
|
|
25
25
|
r53delete: |
|
26
26
|
@action='DELETE'
|
27
27
|
@begin_comment='Deleting'
|
28
|
+
data=record_data
|
29
|
+
@ttl=data[:ttl]
|
30
|
+
@type=data[:type]
|
31
|
+
@resource_records=data[:resource_records]
|
28
32
|
r53_operation
|
29
33
|
|
30
34
|
r53_operation: |
|
@@ -35,13 +39,9 @@ Shards::Dns:
|
|
35
39
|
action: action,
|
36
40
|
resource_record_set: {
|
37
41
|
name: host_param,
|
38
|
-
resource_records:
|
39
|
-
|
40
|
-
|
41
|
-
},
|
42
|
-
],
|
43
|
-
ttl: 300,
|
44
|
-
type: "CNAME",
|
42
|
+
resource_records: resource_records,
|
43
|
+
ttl:ttl,
|
44
|
+
type: type,
|
45
45
|
},
|
46
46
|
},
|
47
47
|
],
|
@@ -54,11 +54,13 @@ Shards::Dns:
|
|
54
54
|
Shards::Db:
|
55
55
|
name: |
|
56
56
|
[stage.downcase_location, dns.domain, stage.name ].join('_').gsub('-','_')
|
57
|
-
server:
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
57
|
+
server: |
|
58
|
+
@server||=shard.server_with_min_databases
|
59
|
+
net: blank_db_connection['database']
|
60
|
+
shard: stage.shard
|
61
|
+
net_server: blank_db_connection['host']
|
62
|
+
user: shard.username_by_server(server)
|
63
|
+
password: shard.password_by_server(server)
|
62
64
|
proxy: stage.get 'proxy'
|
63
65
|
proxy_user: ENV['PROXY_USER']
|
64
66
|
file: |
|
@@ -76,8 +78,10 @@ Shards::Db:
|
|
76
78
|
default_connection: |
|
77
79
|
"mysql #{conn_usr_pass} -h#{server}"
|
78
80
|
dump: |
|
79
|
-
"mysqldump #{
|
81
|
+
"mysqldump #{blank_db_conn_usr_pass} -h#{net_server}"
|
80
82
|
conn_usr_pass: '"-u#{user} -p#{password}"'
|
83
|
+
blank_db_conn_usr_pass: |
|
84
|
+
" -u#{blank_db_connection['username']} -p#{blank_db_connection['password']}"
|
81
85
|
setting_up_database_message: |
|
82
86
|
add ""
|
83
87
|
add "##### Setting Up Database"
|
@@ -105,7 +109,7 @@ Shards::Db:
|
|
105
109
|
show_command: |
|
106
110
|
conn + ' -N -B -e "show databases"'
|
107
111
|
drop_command: |
|
108
|
-
conn + '-v -v -e "DROP DATABASE ' + database_to_drop + '"'
|
112
|
+
conn + ' -v -v -e "DROP DATABASE ' + database_to_drop + '"'
|
109
113
|
|
110
114
|
|
111
115
|
Shards::Site:
|
@@ -149,7 +153,6 @@ Shards::Stage:
|
|
149
153
|
[downcase_location, downcase_location + '_' + config_name, ENV['CONFIG_DIR_SUFFIX'] ].join('/')
|
150
154
|
blank_db: |
|
151
155
|
[downcase_location, 'net', name].join('_')
|
152
|
-
blank_db_server: get 'db_server'
|
153
156
|
proxy: downcase_location + '.' + ENV['PROXY_SUBDOMAIN']
|
154
157
|
domain_suffix: |
|
155
158
|
'-' + name
|
@@ -58,3 +58,30 @@ shared_steps:
|
|
58
58
|
finish:
|
59
59
|
correct: final_step
|
60
60
|
|
61
|
+
ask_blank_db:
|
62
|
+
# this step could be called or not after the write_yaml_files
|
63
|
+
# to change the workflow dinamically call the methods
|
64
|
+
# add_ask_blank_db_step
|
65
|
+
question: Choose a blank database source, please.
|
66
|
+
wrong_message: The database %s is not in the list.
|
67
|
+
correct: create_database
|
68
|
+
wrong: ask_for_repeat
|
69
|
+
|
70
|
+
validate_client:
|
71
|
+
correct: validate_domain
|
72
|
+
wrong: clean_repo_and_exit
|
73
|
+
wrong_message: The client name %s.
|
74
|
+
min_size: 2
|
75
|
+
|
76
|
+
validate_domain:
|
77
|
+
correct: validate_location
|
78
|
+
wrong: clean_repo_and_exit
|
79
|
+
wrong_message: The domain %s.
|
80
|
+
min_size: 3
|
81
|
+
regex: ^[a-z0-9_]*$
|
82
|
+
|
83
|
+
validate_stage:
|
84
|
+
correct: check_shard_presence_in_shards
|
85
|
+
wrong_message: Stage %s does not exist.
|
86
|
+
wrong: clean_repo_and_exit
|
87
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
create_by_api:
|
5
|
+
|
6
|
+
commit_changes:
|
7
|
+
message: The changes has been committed and pushed to the remote repository.
|
8
|
+
wrong_message: We had an issue to commit and push the changes.
|
9
|
+
correct: finish
|
10
|
+
wrong: finish
|
11
|
+
|
12
|
+
|
13
|
+
correct_steps_summary:
|
14
|
+
- validate_client
|
15
|
+
- validate_domain
|
16
|
+
- validate_location
|
17
|
+
- validate_stage
|
18
|
+
- check_shard_presence_in_shards
|
19
|
+
- check_database_presence_in_server
|
20
|
+
- check_domain_dns_presence
|
21
|
+
- check_domain_presence_in_sites
|
22
|
+
- write_yaml_files
|
23
|
+
- create_database
|
24
|
+
- commit_changes
|
25
|
+
- finish
|
26
|
+
|
27
|
+
|
28
|
+
start: validate_client
|
@@ -10,7 +10,12 @@ delete_by_terminal:
|
|
10
10
|
|
11
11
|
validate_subdomain:
|
12
12
|
correct: delete_record
|
13
|
-
wrong_message:
|
13
|
+
wrong_message: DNS record for host %s, %s.
|
14
|
+
wrong: ask_to_continue
|
15
|
+
|
16
|
+
ask_to_continue:
|
17
|
+
question: Type (y) to continue, or another string to exit.
|
18
|
+
correct: delete_record
|
14
19
|
wrong: finish
|
15
20
|
|
16
21
|
delete_record:
|
@@ -36,9 +41,12 @@ delete_by_terminal:
|
|
36
41
|
correct: delete_database
|
37
42
|
wrong: finish
|
38
43
|
|
44
|
+
# todo add steps to run the puppet agent,
|
45
|
+
# and restart sidekiq before the database deletion.
|
46
|
+
|
39
47
|
delete_database:
|
40
|
-
wrong_message: The database %s could not be deleted
|
41
|
-
message: Database %s
|
48
|
+
wrong_message: The database %s could not be deleted.
|
49
|
+
message: Database %s has been deleted.
|
42
50
|
correct: finish
|
43
51
|
wrong: finish
|
44
52
|
|
@@ -3,22 +3,19 @@
|
|
3
3
|
|
4
4
|
fast_terminal:
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
correct: check_shard_presence_in_shards
|
21
|
-
wrong_message: Stage %s does not exist.
|
22
|
-
wrong: clean_repo_and_exit
|
6
|
+
correct_steps_summary:
|
7
|
+
- validate_client
|
8
|
+
- validate_domain
|
9
|
+
- validate_location
|
10
|
+
- validate_stage
|
11
|
+
- check_shard_presence_in_shards
|
12
|
+
- check_database_presence_in_server
|
13
|
+
- check_domain_dns_presence
|
14
|
+
- check_domain_presence_in_sites
|
15
|
+
- write_yaml_files
|
16
|
+
- create_database
|
17
|
+
- commit_changes
|
18
|
+
- dns_upsert
|
19
|
+
- finish
|
23
20
|
|
24
21
|
start: validate_client
|
@@ -0,0 +1,16 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
fast_terminal_all:
|
5
|
+
# this workflow inherits of fast_terminal, so here are the overwrited steps
|
6
|
+
# and new start pointer
|
7
|
+
|
8
|
+
validate_location:
|
9
|
+
correct: stage_iteration
|
10
|
+
wrong_message: Location %s does not exist.
|
11
|
+
wrong: clean_repo_and_exit
|
12
|
+
|
13
|
+
#TODO define step iteration
|
14
|
+
# and overwrite the end of the loop
|
15
|
+
|
16
|
+
start: validate_client
|
data/lib/shards/client.rb
CHANGED
@@ -11,17 +11,28 @@ module Shards
|
|
11
11
|
def create location, stage, client, domain
|
12
12
|
|
13
13
|
fast_terminal=Shards::Workflow::FastTerminal.new config
|
14
|
-
fast_terminal.params= { client: client, domain: domain, location: location, stage: stage }
|
14
|
+
fast_terminal.params= { client: client, domain: domain, location: location.downcase, stage: stage }
|
15
15
|
fast_terminal.start
|
16
16
|
|
17
17
|
end
|
18
18
|
|
19
|
+
# desc 'create_all LOCATION CLIENT DOMAIN', 'Fast creation shard tool in all stages'
|
20
|
+
#
|
21
|
+
# def create_all location, client, domain
|
22
|
+
#
|
23
|
+
# fast_terminal=Shards::Workflow::FastTerminalAll.new config
|
24
|
+
#
|
25
|
+
# fast_terminal.params= { client: client, domain: domain, location: location.downcase }
|
26
|
+
# fast_terminal.start
|
27
|
+
#
|
28
|
+
# end
|
29
|
+
|
19
30
|
desc 'delete LOCATION STAGE SUBDOMAIN', 'Fast deletion tool. To delete dns record, shard, site and database.'
|
20
31
|
|
21
32
|
def delete location, stage, subdomain
|
22
33
|
|
23
34
|
terminal=Shards::Workflow::DeleteByTerminal.new config
|
24
|
-
terminal.params= { subdomain: subdomain, location: location, stage: stage }
|
35
|
+
terminal.params= { subdomain: subdomain, location: location.downcase, stage: stage }
|
25
36
|
terminal.start
|
26
37
|
|
27
38
|
end
|
@@ -47,6 +58,7 @@ module Shards
|
|
47
58
|
|
48
59
|
|
49
60
|
desc "list LOCATION STAGE", "Shard list by LOCATION and STAGE"
|
61
|
+
option :filter, desc: "substring to filter results"
|
50
62
|
|
51
63
|
def list(location, stage)
|
52
64
|
|
@@ -56,24 +68,37 @@ module Shards
|
|
56
68
|
[ k, v['database'], v['host'] ]
|
57
69
|
end
|
58
70
|
|
71
|
+
rows=filter rows, options[:filter]
|
72
|
+
|
59
73
|
columns %w(Shard Database Host), rows
|
60
74
|
|
61
75
|
end
|
62
76
|
|
77
|
+
|
63
78
|
desc "databases LOCATION STAGE", "List databases in server by LOCATION and STAGE"
|
79
|
+
option :filter, desc: "substring to filter results"
|
64
80
|
|
65
81
|
def databases(location, stage)
|
66
82
|
|
67
83
|
s = get_stage location, stage
|
68
|
-
|
69
|
-
db=s.shard.db
|
84
|
+
db=Shards::Db.new stage: s
|
70
85
|
db.show
|
71
86
|
|
72
|
-
rows=db.
|
73
|
-
[ v ]
|
74
|
-
end
|
87
|
+
rows=filter db.database_server, options[:filter]
|
75
88
|
|
76
|
-
columns %w(Database), rows
|
89
|
+
columns %w(Database Server), rows
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
desc "delete_db LOCATION STAGE DATABASE", "Delete database in all database servers"
|
94
|
+
|
95
|
+
def delete_db(location, stage, database)
|
96
|
+
|
97
|
+
s = get_stage location, stage
|
98
|
+
terminal=Shards::Workflow::DeleteByTerminal.new config
|
99
|
+
terminal.stage=s
|
100
|
+
terminal.database_to_drop=database
|
101
|
+
terminal.start_at 'delete_database'
|
77
102
|
|
78
103
|
end
|
79
104
|
|
@@ -82,7 +107,7 @@ module Shards
|
|
82
107
|
|
83
108
|
def dns(location, stage, domain)
|
84
109
|
|
85
|
-
d = get_dns location, stage, domain
|
110
|
+
d = get_dns location.downcase, stage, domain
|
86
111
|
|
87
112
|
puts "Host: #{d.host} , exists: #{d.exist?}"
|
88
113
|
|
@@ -134,6 +159,7 @@ module Shards
|
|
134
159
|
end
|
135
160
|
|
136
161
|
desc "sites LOCATION STAGE", "Site list by LOCATION and STAGE"
|
162
|
+
option :filter, desc: "substring to filter results"
|
137
163
|
|
138
164
|
def sites(location, stage)
|
139
165
|
|
@@ -145,6 +171,7 @@ module Shards
|
|
145
171
|
[ x['name'], x['subdomain'], x['shard'] ]
|
146
172
|
end
|
147
173
|
|
174
|
+
rows=filter rows, options[:filter]
|
148
175
|
columns %w(Name Subdomain Shard), rows
|
149
176
|
|
150
177
|
end
|
@@ -161,7 +188,7 @@ module Shards
|
|
161
188
|
end
|
162
189
|
|
163
190
|
def get_stage location, stage
|
164
|
-
l= config.locations[location]
|
191
|
+
l= config.locations[location.downcase]
|
165
192
|
l.stages[stage]
|
166
193
|
end
|
167
194
|
|
@@ -170,6 +197,10 @@ module Shards
|
|
170
197
|
Shards::Dns.new stage: s, domain: domain
|
171
198
|
end
|
172
199
|
|
200
|
+
def filter rows, text
|
201
|
+
rows=rows.select { |x| x.join(' ').downcase.include?text.downcase } if text
|
202
|
+
rows
|
203
|
+
end
|
173
204
|
end
|
174
205
|
|
175
206
|
end
|
data/lib/shards/config.rb
CHANGED
data/lib/shards/db.rb
CHANGED
@@ -12,15 +12,18 @@ module Shards
|
|
12
12
|
define_method(m.to_sym) { run __method__ }
|
13
13
|
end
|
14
14
|
|
15
|
-
attr_accessor :database_to_drop
|
16
|
-
attr_reader :stage, :dns, :
|
17
|
-
attr_writer :conn
|
15
|
+
attr_accessor :database_to_drop, :blank_db
|
16
|
+
attr_reader :stage, :dns, :databases_on_servers, :server_databases
|
17
|
+
attr_writer :conn, :server
|
18
18
|
|
19
|
-
def initialize stage:, dns:
|
19
|
+
def initialize stage:, dns: nil
|
20
20
|
set_config
|
21
21
|
@stage=stage
|
22
22
|
@dns=dns
|
23
|
-
|
23
|
+
@databases_on_servers=[]
|
24
|
+
@server_databases={}
|
25
|
+
|
26
|
+
reset_output
|
24
27
|
end
|
25
28
|
|
26
29
|
def write_file
|
@@ -69,9 +72,32 @@ module Shards
|
|
69
72
|
end
|
70
73
|
|
71
74
|
def show
|
75
|
+
@databases_on_servers=[]
|
76
|
+
@server_databases={}
|
77
|
+
|
72
78
|
ssh_conn do |ssh|
|
73
|
-
|
79
|
+
shard.servers.each do |s|
|
80
|
+
@server=s
|
81
|
+
parse_databases_on_servers( ssh.exec! show_command )
|
82
|
+
end
|
83
|
+
end
|
84
|
+
@server_databases.values.each do |databases|
|
85
|
+
@databases_on_servers+=databases
|
74
86
|
end
|
87
|
+
|
88
|
+
@databases_on_servers=databases_on_servers.uniq
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
def database_server
|
93
|
+
server_databases.each_pair.map do |server,db|
|
94
|
+
db.product [server]
|
95
|
+
end.flatten(1).sort
|
96
|
+
end
|
97
|
+
|
98
|
+
def servers_to_drop
|
99
|
+
raise "database_to_drop is nil" if database_to_drop.nil?
|
100
|
+
server_databases.select { |k,v| v.include?database_to_drop }.keys
|
75
101
|
end
|
76
102
|
|
77
103
|
def ssh_conn
|
@@ -80,14 +106,14 @@ module Shards
|
|
80
106
|
end
|
81
107
|
end
|
82
108
|
|
83
|
-
def
|
84
|
-
@
|
109
|
+
def parse_databases_on_servers ssh_output
|
110
|
+
@server_databases[server]=ssh_output.split("\n").reject { |x| x.include?"Warning" }
|
85
111
|
end
|
86
112
|
|
87
113
|
def exist? searched=nil
|
88
114
|
searched||=name
|
89
115
|
show
|
90
|
-
|
116
|
+
databases_on_servers.include? searched
|
91
117
|
end
|
92
118
|
|
93
119
|
def conn
|
@@ -104,6 +130,10 @@ module Shards
|
|
104
130
|
|
105
131
|
end
|
106
132
|
|
133
|
+
def blank_db_connection
|
134
|
+
shard.list[blank_db || stage.get('blank_db')]
|
135
|
+
end
|
136
|
+
|
107
137
|
end
|
108
138
|
end
|
109
139
|
|
data/lib/shards/dns.rb
CHANGED
@@ -10,7 +10,8 @@ module Shards
|
|
10
10
|
end
|
11
11
|
|
12
12
|
attr_reader :stage, :domain, :action, :begin_comment
|
13
|
-
attr_accessor :aws_route53_resp, :r53
|
13
|
+
attr_accessor :aws_route53_resp, :r53, :ttl, :type
|
14
|
+
attr_writer :resource_records
|
14
15
|
|
15
16
|
def initialize stage:, domain: nil
|
16
17
|
set_config
|
@@ -18,6 +19,12 @@ module Shards
|
|
18
19
|
@domain=domain.downcase if domain
|
19
20
|
reset_output
|
20
21
|
@r53 = Aws::Route53::Client.new()
|
22
|
+
@ttl=300
|
23
|
+
@type='CNAME'
|
24
|
+
end
|
25
|
+
|
26
|
+
def resource_records
|
27
|
+
@resource_records||= [ { value: target } ]
|
21
28
|
end
|
22
29
|
|
23
30
|
def set dryrun: true
|
@@ -40,14 +47,23 @@ module Shards
|
|
40
47
|
end
|
41
48
|
|
42
49
|
def exist?
|
43
|
-
!record_list
|
50
|
+
!record_list.empty?
|
51
|
+
end
|
52
|
+
|
53
|
+
def record_data
|
54
|
+
data=record_list.first || {}
|
55
|
+
data.to_h
|
56
|
+
end
|
57
|
+
|
58
|
+
def include_target?
|
59
|
+
exist? and record_data[:resource_records].include?( { value: target} )
|
44
60
|
end
|
45
61
|
|
46
|
-
def record_list
|
47
|
-
record_sets
|
62
|
+
def record_list
|
63
|
+
record_sets.select { |x| x.name.include? host_param }
|
48
64
|
end
|
49
65
|
|
50
|
-
def record_sets
|
66
|
+
def record_sets
|
51
67
|
r53.list_resource_record_sets({
|
52
68
|
hosted_zone_id: zone,
|
53
69
|
start_record_name: host_param,
|
data/lib/shards/shard.rb
CHANGED
@@ -9,7 +9,7 @@ module Shards
|
|
9
9
|
end
|
10
10
|
|
11
11
|
attr_accessor :domain, :db, :dns, :database
|
12
|
-
attr_writer :name_param
|
12
|
+
attr_writer :name_param, :server
|
13
13
|
|
14
14
|
def shard_list_key
|
15
15
|
ENV['SHARD_LIST_KEY_ROOT_PATH'].split.map do |k|
|
@@ -86,6 +86,37 @@ module Shards
|
|
86
86
|
by_database.keys.count
|
87
87
|
end
|
88
88
|
|
89
|
+
def servers
|
90
|
+
list.values.map { |x| x['host'] }.uniq
|
91
|
+
end
|
92
|
+
|
93
|
+
def server_databases
|
94
|
+
servers.map do |s|
|
95
|
+
[s,databases_by_host(s).count ]
|
96
|
+
end.to_h
|
97
|
+
end
|
98
|
+
|
99
|
+
def databases_by_host server
|
100
|
+
list.values.select { |x| x['host']==server }.map { |x| x['database'] }.uniq
|
101
|
+
end
|
102
|
+
|
103
|
+
def server_with_min_databases
|
104
|
+
min=server_databases.values.min
|
105
|
+
server_databases.invert[min]
|
106
|
+
end
|
107
|
+
|
108
|
+
def by_server server
|
109
|
+
list.values.select { |v| v['host']==server }.first
|
110
|
+
end
|
111
|
+
|
112
|
+
def username_by_server server
|
113
|
+
by_server(server)['username']
|
114
|
+
end
|
115
|
+
|
116
|
+
def password_by_server server
|
117
|
+
by_server(server)['password']
|
118
|
+
end
|
119
|
+
|
89
120
|
end
|
90
121
|
end
|
91
122
|
|
data/lib/shards/stage.rb
CHANGED
data/lib/shards/version.rb
CHANGED
data/lib/shards/workflow/base.rb
CHANGED
@@ -6,11 +6,10 @@ module Shards
|
|
6
6
|
|
7
7
|
class Base
|
8
8
|
|
9
|
-
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params
|
9
|
+
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params, :blank_db
|
10
10
|
|
11
11
|
attr_writer :wf
|
12
12
|
|
13
|
-
|
14
13
|
def initialize config
|
15
14
|
@config=config
|
16
15
|
config.repo.clean
|
@@ -71,6 +70,12 @@ module Shards
|
|
71
70
|
run_step
|
72
71
|
end
|
73
72
|
|
73
|
+
def start_at pointer_step
|
74
|
+
@flow=shared_steps.merge yaml[wf]
|
75
|
+
@pointer=pointer_step
|
76
|
+
run_step
|
77
|
+
end
|
78
|
+
|
74
79
|
def run_step
|
75
80
|
|
76
81
|
@step=flow[pointer]
|
@@ -81,7 +86,7 @@ module Shards
|
|
81
86
|
send pointer
|
82
87
|
@pointer=step['correct']
|
83
88
|
rescue => e
|
84
|
-
|
89
|
+
output_message e.message
|
85
90
|
@pointer=step['wrong']
|
86
91
|
end
|
87
92
|
|
@@ -91,6 +96,11 @@ module Shards
|
|
91
96
|
|
92
97
|
end
|
93
98
|
|
99
|
+
def output_message message
|
100
|
+
puts message
|
101
|
+
end
|
102
|
+
|
103
|
+
|
94
104
|
def ask_question
|
95
105
|
puts step['question']
|
96
106
|
@answer=$stdin.gets.chomp
|
@@ -190,6 +200,7 @@ module Shards
|
|
190
200
|
end
|
191
201
|
|
192
202
|
def create_database
|
203
|
+
stage.shard.db.blank_db = blank_db
|
193
204
|
stage.shard.db.create dryrun: false
|
194
205
|
message_params= [stage.shard.db.name, stage.shard.db.server]
|
195
206
|
raise_wrong_text message_params unless stage.shard.db.exist?
|
@@ -227,6 +238,16 @@ module Shards
|
|
227
238
|
@stage=location.stages[st]
|
228
239
|
end
|
229
240
|
|
241
|
+
def ask_blank_db
|
242
|
+
shards=stage.shard.list.keys
|
243
|
+
choose_one "Databases:", shards
|
244
|
+
@blank_db=shards[answer.to_i]
|
245
|
+
end
|
246
|
+
|
247
|
+
def add_ask_blank_db_step
|
248
|
+
@flow['write_yaml_files']['correct']='ask_blank_db'
|
249
|
+
end
|
250
|
+
|
230
251
|
end
|
231
252
|
|
232
253
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative './base.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class CreateByApi < Shards::Workflow::Base
|
8
|
+
|
9
|
+
def write_yaml_files
|
10
|
+
stage.add_domain domain
|
11
|
+
site.add_site
|
12
|
+
end
|
13
|
+
|
14
|
+
def output_message message
|
15
|
+
message
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -6,19 +6,31 @@ module Shards
|
|
6
6
|
|
7
7
|
class DeleteByTerminal < Shards::Workflow::Base
|
8
8
|
|
9
|
-
attr_accessor :dns, :db, :
|
9
|
+
attr_accessor :dns, :db, :database_to_drop
|
10
10
|
|
11
11
|
def validate_subdomain
|
12
12
|
@dns=Shards::Dns.new stage: stage
|
13
13
|
dns.subdomain_param=params[:subdomain]
|
14
|
-
|
14
|
+
|
15
|
+
raise_wrong_text [dns.host_param, " doesn't exist"] unless dns.exist?
|
16
|
+
|
17
|
+
unless dns.include_target?
|
18
|
+
raise_wrong_text [dns.host_param, target_message_error ]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def ask_to_continue
|
23
|
+
ask_question
|
24
|
+
raise step['wrong_message'] if answer.strip.downcase!='y'
|
15
25
|
end
|
16
26
|
|
17
27
|
def delete_record
|
18
|
-
dns.
|
19
|
-
|
20
|
-
|
21
|
-
|
28
|
+
if dns.exist?
|
29
|
+
dns.delete
|
30
|
+
message_params= [dns.host_param, dns.resource_records.to_s]
|
31
|
+
raise_wrong_text message_params if dns.exist?
|
32
|
+
puts step['message'] % message_params
|
33
|
+
end
|
22
34
|
end
|
23
35
|
|
24
36
|
def delete_site_and_shard
|
@@ -31,7 +43,7 @@ module Shards
|
|
31
43
|
shard.name_param = @shard_name
|
32
44
|
shard.delete
|
33
45
|
|
34
|
-
@
|
46
|
+
@database_to_drop=shard.database['database']
|
35
47
|
|
36
48
|
end
|
37
49
|
|
@@ -49,35 +61,53 @@ module Shards
|
|
49
61
|
end
|
50
62
|
|
51
63
|
def delete_database
|
52
|
-
@db=Shards::Db.new stage: stage, dns: dns
|
53
|
-
message_params= [db_name, db_host]
|
54
|
-
raise "Database %s does not exist in %s." % message_params unless db_exist?
|
55
64
|
|
56
|
-
db.
|
57
|
-
db.conn="mysql -u#{db_conn['username']} -p#{db_conn['password']} -h#{db_host} "
|
58
|
-
db.drop
|
65
|
+
@db=Shards::Db.new stage: stage
|
59
66
|
|
60
|
-
|
61
|
-
puts step['message'] % message_params
|
62
|
-
end
|
67
|
+
db.database_to_drop=database_to_drop
|
63
68
|
|
69
|
+
raise "Database %s does is not in present db servers" % database_to_drop unless db.exist?(database_to_drop)
|
64
70
|
|
65
|
-
|
71
|
+
db.servers_to_drop.each do |host|
|
66
72
|
|
67
|
-
|
68
|
-
|
69
|
-
|
73
|
+
puts
|
74
|
+
puts "Database:\t #{database_to_drop}"
|
75
|
+
puts "Host: \t #{host}"
|
76
|
+
puts
|
77
|
+
puts "Do you want to delete the above database? (y) to confirm, another string to not."
|
78
|
+
|
79
|
+
answer=$stdin.gets
|
80
|
+
|
81
|
+
puts
|
82
|
+
|
83
|
+
if answer.chomp.downcase=='y'
|
84
|
+
puts "DELETING database #{database_to_drop} in #{host}"
|
85
|
+
db.server=host
|
86
|
+
db.drop
|
87
|
+
else
|
88
|
+
puts "KEEP database #{database_to_drop} in #{host}"
|
89
|
+
end
|
90
|
+
puts
|
91
|
+
puts "-" * 50
|
92
|
+
puts
|
93
|
+
end
|
70
94
|
|
71
|
-
|
72
|
-
|
95
|
+
raise_wrong_text database_to_drop if db.exist? database_to_drop
|
96
|
+
puts step['message'] % database_to_drop
|
73
97
|
end
|
74
98
|
|
75
|
-
|
76
|
-
|
99
|
+
private
|
100
|
+
|
101
|
+
def target_message_error
|
102
|
+
message = []
|
103
|
+
message<< "Defined target: #{dns.target},"
|
104
|
+
message<< "not found in resource record values:"
|
105
|
+
message<< dns.record_data.to_s
|
106
|
+
message.join("\n\t")
|
77
107
|
end
|
78
108
|
|
79
109
|
end
|
80
110
|
|
81
111
|
end
|
82
112
|
|
83
|
-
|
113
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require_relative './fast_terminal.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class FastTerminalAll < Shards::Workflow::FastTerminal
|
8
|
+
|
9
|
+
def initialize config
|
10
|
+
super(config)
|
11
|
+
@wf='fast_terminal'
|
12
|
+
end
|
13
|
+
|
14
|
+
def start
|
15
|
+
@flow=shared_steps.merge yaml[wf]
|
16
|
+
|
17
|
+
@pointer=yaml['start']
|
18
|
+
run_step
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
data/test.env
CHANGED
@@ -8,10 +8,9 @@ SITES_FILE_NAME=sites.yml
|
|
8
8
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
9
9
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
10
10
|
DNS_ZONE_CODE=TestCodeDNSZone
|
11
|
-
DB_USER=db_user_name
|
12
|
-
DB_PASSWORD=HerePasswordDB
|
13
11
|
PROXY_USER=proxy_username
|
14
12
|
BASE_CONFIG_FILE=shards.yml
|
13
|
+
|
15
14
|
# optional:
|
16
15
|
TEST=true #only for test.env delete this var in settings.env
|
17
16
|
CONFIG_REPLACE={ 'prod' => 'web' }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.0.1.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diego PL
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -191,6 +191,7 @@ files:
|
|
191
191
|
- ".rspec"
|
192
192
|
- ".ruby-version"
|
193
193
|
- ".travis.yml"
|
194
|
+
- ":w"
|
194
195
|
- CODE_OF_CONDUCT.md
|
195
196
|
- Gemfile
|
196
197
|
- Gemfile.lock
|
@@ -202,8 +203,10 @@ files:
|
|
202
203
|
- exe/shards
|
203
204
|
- lib/settings/format_methods.yaml
|
204
205
|
- lib/settings/workflows/base.yaml
|
206
|
+
- lib/settings/workflows/create_by_api.yaml
|
205
207
|
- lib/settings/workflows/delete_by_terminal.yaml
|
206
208
|
- lib/settings/workflows/fast_terminal.yaml
|
209
|
+
- lib/settings/workflows/fast_terminal_all.yaml
|
207
210
|
- lib/settings/workflows/terminal.yaml
|
208
211
|
- lib/shards.rb
|
209
212
|
- lib/shards/base.rb
|
@@ -221,8 +224,10 @@ files:
|
|
221
224
|
- lib/shards/stage.rb
|
222
225
|
- lib/shards/version.rb
|
223
226
|
- lib/shards/workflow/base.rb
|
227
|
+
- lib/shards/workflow/create_by_api.rb
|
224
228
|
- lib/shards/workflow/delete_by_terminal.rb
|
225
229
|
- lib/shards/workflow/fast_terminal.rb
|
230
|
+
- lib/shards/workflow/fast_terminal_all.rb
|
226
231
|
- lib/shards/workflow/terminal.rb
|
227
232
|
- lib/shards/workflow/workflow.rb
|
228
233
|
- shards.gemspec
|
@@ -243,9 +248,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
243
248
|
version: '0'
|
244
249
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
245
250
|
requirements:
|
246
|
-
- - "
|
251
|
+
- - ">"
|
247
252
|
- !ruby/object:Gem::Version
|
248
|
-
version:
|
253
|
+
version: 1.3.1
|
249
254
|
requirements: []
|
250
255
|
rubyforge_project:
|
251
256
|
rubygems_version: 2.6.14
|