shards 1.2.6 → 2.0.1.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/:w +50 -0
- data/Gemfile.lock +1 -1
- data/README.md +1 -2
- data/lib/settings/format_methods.yaml +18 -15
- data/lib/settings/workflows/base.yaml +27 -0
- data/lib/settings/workflows/create_by_api.yaml +28 -0
- data/lib/settings/workflows/delete_by_terminal.yaml +11 -3
- data/lib/settings/workflows/fast_terminal.yaml +14 -17
- data/lib/settings/workflows/fast_terminal_all.yaml +16 -0
- data/lib/settings/workflows/terminal.yaml +1 -1
- data/lib/shards/client.rb +41 -10
- data/lib/shards/config.rb +1 -1
- data/lib/shards/db.rb +39 -9
- data/lib/shards/dns.rb +21 -5
- data/lib/shards/shard.rb +32 -1
- data/lib/shards/stage.rb +2 -2
- data/lib/shards/version.rb +1 -1
- data/lib/shards/workflow/base.rb +24 -3
- data/lib/shards/workflow/create_by_api.rb +22 -0
- data/lib/shards/workflow/delete_by_terminal.rb +55 -25
- data/lib/shards/workflow/fast_terminal_all.rb +24 -0
- data/lib/shards/workflow/terminal.rb +1 -0
- data/lib/shards/workflow/workflow.rb +1 -0
- data/test.env +1 -2
- metadata +9 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 35764b0efc264216ed213b070745bcc8f0fd6cf6
|
4
|
+
data.tar.gz: 5409b1150b4e8e76c100f5469bf3f2e1fe015f4f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d46bc308507992fa107d586946008e6d11361fbaedddbe9746b2c765eecb9d3700bafbe2c69fc656d26dd191009fe2559a88f91915a5fe97e3b00af4f9746060
|
7
|
+
data.tar.gz: a95761c3bebdc6b120bb4984bd63cdfdfdcc6c54c00b6baa7ffdf67084c8bc37a9afed40de06866614af173e85239cf5b81c70818a26bf1adb71609ca7b74bbe
|
data/:w
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
delete_by_terminal:
|
5
|
+
|
6
|
+
validate_stage:
|
7
|
+
correct: validate_subdomain
|
8
|
+
wrong_message: Stage %s does not exist.
|
9
|
+
wrong: finish
|
10
|
+
|
11
|
+
validate_subdomain:
|
12
|
+
correct: delete_record
|
13
|
+
wrong_message: DNS record for host %s, %s.
|
14
|
+
wrong: ask_to_continue
|
15
|
+
|
16
|
+
ask_to_continue:
|
17
|
+
question: Do you want to continue?
|
18
|
+
correct: delete_record
|
19
|
+
wrong: finish
|
20
|
+
|
21
|
+
delete_record:
|
22
|
+
message: Record %s deleted in target %s
|
23
|
+
correct: delete_site_and_shard
|
24
|
+
wrong: finish
|
25
|
+
wrong_message: Could not delete the record %s.
|
26
|
+
|
27
|
+
delete_site_and_shard:
|
28
|
+
correct: write_yaml_files
|
29
|
+
wrong: clean_repo_and_exit
|
30
|
+
wrong_message: Could not delete site or shard.\n %s.
|
31
|
+
|
32
|
+
write_yaml_files:
|
33
|
+
question: Type Y to confirm the changes
|
34
|
+
wrong_message: Undo changes
|
35
|
+
correct: commit_changes
|
36
|
+
wrong: clean_repo_and_exit
|
37
|
+
|
38
|
+
commit_changes:
|
39
|
+
message: The changes has been committed and pushed to the remote repository.
|
40
|
+
wrong_message: We had an issue to commit and push the changes.
|
41
|
+
correct: delete_database
|
42
|
+
wrong: finish
|
43
|
+
|
44
|
+
delete_database:
|
45
|
+
wrong_message: The database %s could not be deleted in host %s.
|
46
|
+
message: Database %s was deleted in host %s.
|
47
|
+
correct: finish
|
48
|
+
wrong: finish
|
49
|
+
|
50
|
+
start: validate_location
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -12,6 +12,7 @@ Add this line to your application's Gemfile:
|
|
12
12
|
|
13
13
|
```ruby
|
14
14
|
gem 'shards'
|
15
|
+
|
15
16
|
```
|
16
17
|
|
17
18
|
And then execute:
|
@@ -38,8 +39,6 @@ SITES_FILE_NAME=sites.yml
|
|
38
39
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
39
40
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
40
41
|
DNS_ZONE_CODE=DNS Zone Code
|
41
|
-
DB_USER=db_user_name
|
42
|
-
DB_PASSWORD=HerePasswordDB
|
43
42
|
PROXY_USER=proxy_username
|
44
43
|
|
45
44
|
# optional:
|
@@ -25,6 +25,10 @@ Shards::Dns:
|
|
25
25
|
r53delete: |
|
26
26
|
@action='DELETE'
|
27
27
|
@begin_comment='Deleting'
|
28
|
+
data=record_data
|
29
|
+
@ttl=data[:ttl]
|
30
|
+
@type=data[:type]
|
31
|
+
@resource_records=data[:resource_records]
|
28
32
|
r53_operation
|
29
33
|
|
30
34
|
r53_operation: |
|
@@ -35,13 +39,9 @@ Shards::Dns:
|
|
35
39
|
action: action,
|
36
40
|
resource_record_set: {
|
37
41
|
name: host_param,
|
38
|
-
resource_records:
|
39
|
-
|
40
|
-
|
41
|
-
},
|
42
|
-
],
|
43
|
-
ttl: 300,
|
44
|
-
type: "CNAME",
|
42
|
+
resource_records: resource_records,
|
43
|
+
ttl:ttl,
|
44
|
+
type: type,
|
45
45
|
},
|
46
46
|
},
|
47
47
|
],
|
@@ -54,11 +54,13 @@ Shards::Dns:
|
|
54
54
|
Shards::Db:
|
55
55
|
name: |
|
56
56
|
[stage.downcase_location, dns.domain, stage.name ].join('_').gsub('-','_')
|
57
|
-
server:
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
57
|
+
server: |
|
58
|
+
@server||=shard.server_with_min_databases
|
59
|
+
net: blank_db_connection['database']
|
60
|
+
shard: stage.shard
|
61
|
+
net_server: blank_db_connection['host']
|
62
|
+
user: shard.username_by_server(server)
|
63
|
+
password: shard.password_by_server(server)
|
62
64
|
proxy: stage.get 'proxy'
|
63
65
|
proxy_user: ENV['PROXY_USER']
|
64
66
|
file: |
|
@@ -76,8 +78,10 @@ Shards::Db:
|
|
76
78
|
default_connection: |
|
77
79
|
"mysql #{conn_usr_pass} -h#{server}"
|
78
80
|
dump: |
|
79
|
-
"mysqldump #{
|
81
|
+
"mysqldump #{blank_db_conn_usr_pass} -h#{net_server}"
|
80
82
|
conn_usr_pass: '"-u#{user} -p#{password}"'
|
83
|
+
blank_db_conn_usr_pass: |
|
84
|
+
" -u#{blank_db_connection['username']} -p#{blank_db_connection['password']}"
|
81
85
|
setting_up_database_message: |
|
82
86
|
add ""
|
83
87
|
add "##### Setting Up Database"
|
@@ -105,7 +109,7 @@ Shards::Db:
|
|
105
109
|
show_command: |
|
106
110
|
conn + ' -N -B -e "show databases"'
|
107
111
|
drop_command: |
|
108
|
-
conn + '-v -v -e "DROP DATABASE ' + database_to_drop + '"'
|
112
|
+
conn + ' -v -v -e "DROP DATABASE ' + database_to_drop + '"'
|
109
113
|
|
110
114
|
|
111
115
|
Shards::Site:
|
@@ -149,7 +153,6 @@ Shards::Stage:
|
|
149
153
|
[downcase_location, downcase_location + '_' + config_name, ENV['CONFIG_DIR_SUFFIX'] ].join('/')
|
150
154
|
blank_db: |
|
151
155
|
[downcase_location, 'net', name].join('_')
|
152
|
-
blank_db_server: get 'db_server'
|
153
156
|
proxy: downcase_location + '.' + ENV['PROXY_SUBDOMAIN']
|
154
157
|
domain_suffix: |
|
155
158
|
'-' + name
|
@@ -58,3 +58,30 @@ shared_steps:
|
|
58
58
|
finish:
|
59
59
|
correct: final_step
|
60
60
|
|
61
|
+
ask_blank_db:
|
62
|
+
# this step could be called or not after the write_yaml_files
|
63
|
+
# to change the workflow dinamically call the methods
|
64
|
+
# add_ask_blank_db_step
|
65
|
+
question: Choose a blank database source, please.
|
66
|
+
wrong_message: The database %s is not in the list.
|
67
|
+
correct: create_database
|
68
|
+
wrong: ask_for_repeat
|
69
|
+
|
70
|
+
validate_client:
|
71
|
+
correct: validate_domain
|
72
|
+
wrong: clean_repo_and_exit
|
73
|
+
wrong_message: The client name %s.
|
74
|
+
min_size: 2
|
75
|
+
|
76
|
+
validate_domain:
|
77
|
+
correct: validate_location
|
78
|
+
wrong: clean_repo_and_exit
|
79
|
+
wrong_message: The domain %s.
|
80
|
+
min_size: 3
|
81
|
+
regex: ^[a-z0-9_]*$
|
82
|
+
|
83
|
+
validate_stage:
|
84
|
+
correct: check_shard_presence_in_shards
|
85
|
+
wrong_message: Stage %s does not exist.
|
86
|
+
wrong: clean_repo_and_exit
|
87
|
+
|
@@ -0,0 +1,28 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
create_by_api:
|
5
|
+
|
6
|
+
commit_changes:
|
7
|
+
message: The changes has been committed and pushed to the remote repository.
|
8
|
+
wrong_message: We had an issue to commit and push the changes.
|
9
|
+
correct: finish
|
10
|
+
wrong: finish
|
11
|
+
|
12
|
+
|
13
|
+
correct_steps_summary:
|
14
|
+
- validate_client
|
15
|
+
- validate_domain
|
16
|
+
- validate_location
|
17
|
+
- validate_stage
|
18
|
+
- check_shard_presence_in_shards
|
19
|
+
- check_database_presence_in_server
|
20
|
+
- check_domain_dns_presence
|
21
|
+
- check_domain_presence_in_sites
|
22
|
+
- write_yaml_files
|
23
|
+
- create_database
|
24
|
+
- commit_changes
|
25
|
+
- finish
|
26
|
+
|
27
|
+
|
28
|
+
start: validate_client
|
@@ -10,7 +10,12 @@ delete_by_terminal:
|
|
10
10
|
|
11
11
|
validate_subdomain:
|
12
12
|
correct: delete_record
|
13
|
-
wrong_message:
|
13
|
+
wrong_message: DNS record for host %s, %s.
|
14
|
+
wrong: ask_to_continue
|
15
|
+
|
16
|
+
ask_to_continue:
|
17
|
+
question: Type (y) to continue, or another string to exit.
|
18
|
+
correct: delete_record
|
14
19
|
wrong: finish
|
15
20
|
|
16
21
|
delete_record:
|
@@ -36,9 +41,12 @@ delete_by_terminal:
|
|
36
41
|
correct: delete_database
|
37
42
|
wrong: finish
|
38
43
|
|
44
|
+
# todo add steps to run the puppet agent,
|
45
|
+
# and restart sidekiq before the database deletion.
|
46
|
+
|
39
47
|
delete_database:
|
40
|
-
wrong_message: The database %s could not be deleted
|
41
|
-
message: Database %s
|
48
|
+
wrong_message: The database %s could not be deleted.
|
49
|
+
message: Database %s has been deleted.
|
42
50
|
correct: finish
|
43
51
|
wrong: finish
|
44
52
|
|
@@ -3,22 +3,19 @@
|
|
3
3
|
|
4
4
|
fast_terminal:
|
5
5
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
correct: check_shard_presence_in_shards
|
21
|
-
wrong_message: Stage %s does not exist.
|
22
|
-
wrong: clean_repo_and_exit
|
6
|
+
correct_steps_summary:
|
7
|
+
- validate_client
|
8
|
+
- validate_domain
|
9
|
+
- validate_location
|
10
|
+
- validate_stage
|
11
|
+
- check_shard_presence_in_shards
|
12
|
+
- check_database_presence_in_server
|
13
|
+
- check_domain_dns_presence
|
14
|
+
- check_domain_presence_in_sites
|
15
|
+
- write_yaml_files
|
16
|
+
- create_database
|
17
|
+
- commit_changes
|
18
|
+
- dns_upsert
|
19
|
+
- finish
|
23
20
|
|
24
21
|
start: validate_client
|
@@ -0,0 +1,16 @@
|
|
1
|
+
---
|
2
|
+
# see shared_steps in base.yaml to find the steps not defined in a workflow
|
3
|
+
|
4
|
+
fast_terminal_all:
|
5
|
+
# this workflow inherits of fast_terminal, so here are the overwrited steps
|
6
|
+
# and new start pointer
|
7
|
+
|
8
|
+
validate_location:
|
9
|
+
correct: stage_iteration
|
10
|
+
wrong_message: Location %s does not exist.
|
11
|
+
wrong: clean_repo_and_exit
|
12
|
+
|
13
|
+
#TODO define step iteration
|
14
|
+
# and overwrite the end of the loop
|
15
|
+
|
16
|
+
start: validate_client
|
data/lib/shards/client.rb
CHANGED
@@ -11,17 +11,28 @@ module Shards
|
|
11
11
|
def create location, stage, client, domain
|
12
12
|
|
13
13
|
fast_terminal=Shards::Workflow::FastTerminal.new config
|
14
|
-
fast_terminal.params= { client: client, domain: domain, location: location, stage: stage }
|
14
|
+
fast_terminal.params= { client: client, domain: domain, location: location.downcase, stage: stage }
|
15
15
|
fast_terminal.start
|
16
16
|
|
17
17
|
end
|
18
18
|
|
19
|
+
# desc 'create_all LOCATION CLIENT DOMAIN', 'Fast creation shard tool in all stages'
|
20
|
+
#
|
21
|
+
# def create_all location, client, domain
|
22
|
+
#
|
23
|
+
# fast_terminal=Shards::Workflow::FastTerminalAll.new config
|
24
|
+
#
|
25
|
+
# fast_terminal.params= { client: client, domain: domain, location: location.downcase }
|
26
|
+
# fast_terminal.start
|
27
|
+
#
|
28
|
+
# end
|
29
|
+
|
19
30
|
desc 'delete LOCATION STAGE SUBDOMAIN', 'Fast deletion tool. To delete dns record, shard, site and database.'
|
20
31
|
|
21
32
|
def delete location, stage, subdomain
|
22
33
|
|
23
34
|
terminal=Shards::Workflow::DeleteByTerminal.new config
|
24
|
-
terminal.params= { subdomain: subdomain, location: location, stage: stage }
|
35
|
+
terminal.params= { subdomain: subdomain, location: location.downcase, stage: stage }
|
25
36
|
terminal.start
|
26
37
|
|
27
38
|
end
|
@@ -47,6 +58,7 @@ module Shards
|
|
47
58
|
|
48
59
|
|
49
60
|
desc "list LOCATION STAGE", "Shard list by LOCATION and STAGE"
|
61
|
+
option :filter, desc: "substring to filter results"
|
50
62
|
|
51
63
|
def list(location, stage)
|
52
64
|
|
@@ -56,24 +68,37 @@ module Shards
|
|
56
68
|
[ k, v['database'], v['host'] ]
|
57
69
|
end
|
58
70
|
|
71
|
+
rows=filter rows, options[:filter]
|
72
|
+
|
59
73
|
columns %w(Shard Database Host), rows
|
60
74
|
|
61
75
|
end
|
62
76
|
|
77
|
+
|
63
78
|
desc "databases LOCATION STAGE", "List databases in server by LOCATION and STAGE"
|
79
|
+
option :filter, desc: "substring to filter results"
|
64
80
|
|
65
81
|
def databases(location, stage)
|
66
82
|
|
67
83
|
s = get_stage location, stage
|
68
|
-
|
69
|
-
db=s.shard.db
|
84
|
+
db=Shards::Db.new stage: s
|
70
85
|
db.show
|
71
86
|
|
72
|
-
rows=db.
|
73
|
-
[ v ]
|
74
|
-
end
|
87
|
+
rows=filter db.database_server, options[:filter]
|
75
88
|
|
76
|
-
columns %w(Database), rows
|
89
|
+
columns %w(Database Server), rows
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
desc "delete_db LOCATION STAGE DATABASE", "Delete database in all database servers"
|
94
|
+
|
95
|
+
def delete_db(location, stage, database)
|
96
|
+
|
97
|
+
s = get_stage location, stage
|
98
|
+
terminal=Shards::Workflow::DeleteByTerminal.new config
|
99
|
+
terminal.stage=s
|
100
|
+
terminal.database_to_drop=database
|
101
|
+
terminal.start_at 'delete_database'
|
77
102
|
|
78
103
|
end
|
79
104
|
|
@@ -82,7 +107,7 @@ module Shards
|
|
82
107
|
|
83
108
|
def dns(location, stage, domain)
|
84
109
|
|
85
|
-
d = get_dns location, stage, domain
|
110
|
+
d = get_dns location.downcase, stage, domain
|
86
111
|
|
87
112
|
puts "Host: #{d.host} , exists: #{d.exist?}"
|
88
113
|
|
@@ -134,6 +159,7 @@ module Shards
|
|
134
159
|
end
|
135
160
|
|
136
161
|
desc "sites LOCATION STAGE", "Site list by LOCATION and STAGE"
|
162
|
+
option :filter, desc: "substring to filter results"
|
137
163
|
|
138
164
|
def sites(location, stage)
|
139
165
|
|
@@ -145,6 +171,7 @@ module Shards
|
|
145
171
|
[ x['name'], x['subdomain'], x['shard'] ]
|
146
172
|
end
|
147
173
|
|
174
|
+
rows=filter rows, options[:filter]
|
148
175
|
columns %w(Name Subdomain Shard), rows
|
149
176
|
|
150
177
|
end
|
@@ -161,7 +188,7 @@ module Shards
|
|
161
188
|
end
|
162
189
|
|
163
190
|
def get_stage location, stage
|
164
|
-
l= config.locations[location]
|
191
|
+
l= config.locations[location.downcase]
|
165
192
|
l.stages[stage]
|
166
193
|
end
|
167
194
|
|
@@ -170,6 +197,10 @@ module Shards
|
|
170
197
|
Shards::Dns.new stage: s, domain: domain
|
171
198
|
end
|
172
199
|
|
200
|
+
def filter rows, text
|
201
|
+
rows=rows.select { |x| x.join(' ').downcase.include?text.downcase } if text
|
202
|
+
rows
|
203
|
+
end
|
173
204
|
end
|
174
205
|
|
175
206
|
end
|
data/lib/shards/config.rb
CHANGED
data/lib/shards/db.rb
CHANGED
@@ -12,15 +12,18 @@ module Shards
|
|
12
12
|
define_method(m.to_sym) { run __method__ }
|
13
13
|
end
|
14
14
|
|
15
|
-
attr_accessor :database_to_drop
|
16
|
-
attr_reader :stage, :dns, :
|
17
|
-
attr_writer :conn
|
15
|
+
attr_accessor :database_to_drop, :blank_db
|
16
|
+
attr_reader :stage, :dns, :databases_on_servers, :server_databases
|
17
|
+
attr_writer :conn, :server
|
18
18
|
|
19
|
-
def initialize stage:, dns:
|
19
|
+
def initialize stage:, dns: nil
|
20
20
|
set_config
|
21
21
|
@stage=stage
|
22
22
|
@dns=dns
|
23
|
-
|
23
|
+
@databases_on_servers=[]
|
24
|
+
@server_databases={}
|
25
|
+
|
26
|
+
reset_output
|
24
27
|
end
|
25
28
|
|
26
29
|
def write_file
|
@@ -69,9 +72,32 @@ module Shards
|
|
69
72
|
end
|
70
73
|
|
71
74
|
def show
|
75
|
+
@databases_on_servers=[]
|
76
|
+
@server_databases={}
|
77
|
+
|
72
78
|
ssh_conn do |ssh|
|
73
|
-
|
79
|
+
shard.servers.each do |s|
|
80
|
+
@server=s
|
81
|
+
parse_databases_on_servers( ssh.exec! show_command )
|
82
|
+
end
|
83
|
+
end
|
84
|
+
@server_databases.values.each do |databases|
|
85
|
+
@databases_on_servers+=databases
|
74
86
|
end
|
87
|
+
|
88
|
+
@databases_on_servers=databases_on_servers.uniq
|
89
|
+
|
90
|
+
end
|
91
|
+
|
92
|
+
def database_server
|
93
|
+
server_databases.each_pair.map do |server,db|
|
94
|
+
db.product [server]
|
95
|
+
end.flatten(1).sort
|
96
|
+
end
|
97
|
+
|
98
|
+
def servers_to_drop
|
99
|
+
raise "database_to_drop is nil" if database_to_drop.nil?
|
100
|
+
server_databases.select { |k,v| v.include?database_to_drop }.keys
|
75
101
|
end
|
76
102
|
|
77
103
|
def ssh_conn
|
@@ -80,14 +106,14 @@ module Shards
|
|
80
106
|
end
|
81
107
|
end
|
82
108
|
|
83
|
-
def
|
84
|
-
@
|
109
|
+
def parse_databases_on_servers ssh_output
|
110
|
+
@server_databases[server]=ssh_output.split("\n").reject { |x| x.include?"Warning" }
|
85
111
|
end
|
86
112
|
|
87
113
|
def exist? searched=nil
|
88
114
|
searched||=name
|
89
115
|
show
|
90
|
-
|
116
|
+
databases_on_servers.include? searched
|
91
117
|
end
|
92
118
|
|
93
119
|
def conn
|
@@ -104,6 +130,10 @@ module Shards
|
|
104
130
|
|
105
131
|
end
|
106
132
|
|
133
|
+
def blank_db_connection
|
134
|
+
shard.list[blank_db || stage.get('blank_db')]
|
135
|
+
end
|
136
|
+
|
107
137
|
end
|
108
138
|
end
|
109
139
|
|
data/lib/shards/dns.rb
CHANGED
@@ -10,7 +10,8 @@ module Shards
|
|
10
10
|
end
|
11
11
|
|
12
12
|
attr_reader :stage, :domain, :action, :begin_comment
|
13
|
-
attr_accessor :aws_route53_resp, :r53
|
13
|
+
attr_accessor :aws_route53_resp, :r53, :ttl, :type
|
14
|
+
attr_writer :resource_records
|
14
15
|
|
15
16
|
def initialize stage:, domain: nil
|
16
17
|
set_config
|
@@ -18,6 +19,12 @@ module Shards
|
|
18
19
|
@domain=domain.downcase if domain
|
19
20
|
reset_output
|
20
21
|
@r53 = Aws::Route53::Client.new()
|
22
|
+
@ttl=300
|
23
|
+
@type='CNAME'
|
24
|
+
end
|
25
|
+
|
26
|
+
def resource_records
|
27
|
+
@resource_records||= [ { value: target } ]
|
21
28
|
end
|
22
29
|
|
23
30
|
def set dryrun: true
|
@@ -40,14 +47,23 @@ module Shards
|
|
40
47
|
end
|
41
48
|
|
42
49
|
def exist?
|
43
|
-
!record_list
|
50
|
+
!record_list.empty?
|
51
|
+
end
|
52
|
+
|
53
|
+
def record_data
|
54
|
+
data=record_list.first || {}
|
55
|
+
data.to_h
|
56
|
+
end
|
57
|
+
|
58
|
+
def include_target?
|
59
|
+
exist? and record_data[:resource_records].include?( { value: target} )
|
44
60
|
end
|
45
61
|
|
46
|
-
def record_list
|
47
|
-
record_sets
|
62
|
+
def record_list
|
63
|
+
record_sets.select { |x| x.name.include? host_param }
|
48
64
|
end
|
49
65
|
|
50
|
-
def record_sets
|
66
|
+
def record_sets
|
51
67
|
r53.list_resource_record_sets({
|
52
68
|
hosted_zone_id: zone,
|
53
69
|
start_record_name: host_param,
|
data/lib/shards/shard.rb
CHANGED
@@ -9,7 +9,7 @@ module Shards
|
|
9
9
|
end
|
10
10
|
|
11
11
|
attr_accessor :domain, :db, :dns, :database
|
12
|
-
attr_writer :name_param
|
12
|
+
attr_writer :name_param, :server
|
13
13
|
|
14
14
|
def shard_list_key
|
15
15
|
ENV['SHARD_LIST_KEY_ROOT_PATH'].split.map do |k|
|
@@ -86,6 +86,37 @@ module Shards
|
|
86
86
|
by_database.keys.count
|
87
87
|
end
|
88
88
|
|
89
|
+
def servers
|
90
|
+
list.values.map { |x| x['host'] }.uniq
|
91
|
+
end
|
92
|
+
|
93
|
+
def server_databases
|
94
|
+
servers.map do |s|
|
95
|
+
[s,databases_by_host(s).count ]
|
96
|
+
end.to_h
|
97
|
+
end
|
98
|
+
|
99
|
+
def databases_by_host server
|
100
|
+
list.values.select { |x| x['host']==server }.map { |x| x['database'] }.uniq
|
101
|
+
end
|
102
|
+
|
103
|
+
def server_with_min_databases
|
104
|
+
min=server_databases.values.min
|
105
|
+
server_databases.invert[min]
|
106
|
+
end
|
107
|
+
|
108
|
+
def by_server server
|
109
|
+
list.values.select { |v| v['host']==server }.first
|
110
|
+
end
|
111
|
+
|
112
|
+
def username_by_server server
|
113
|
+
by_server(server)['username']
|
114
|
+
end
|
115
|
+
|
116
|
+
def password_by_server server
|
117
|
+
by_server(server)['password']
|
118
|
+
end
|
119
|
+
|
89
120
|
end
|
90
121
|
end
|
91
122
|
|
data/lib/shards/stage.rb
CHANGED
data/lib/shards/version.rb
CHANGED
data/lib/shards/workflow/base.rb
CHANGED
@@ -6,11 +6,10 @@ module Shards
|
|
6
6
|
|
7
7
|
class Base
|
8
8
|
|
9
|
-
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params
|
9
|
+
attr_accessor :config, :flow, :pointer, :step, :answer, :pointer_history, :client, :location, :stage, :domain, :site, :params, :blank_db
|
10
10
|
|
11
11
|
attr_writer :wf
|
12
12
|
|
13
|
-
|
14
13
|
def initialize config
|
15
14
|
@config=config
|
16
15
|
config.repo.clean
|
@@ -71,6 +70,12 @@ module Shards
|
|
71
70
|
run_step
|
72
71
|
end
|
73
72
|
|
73
|
+
def start_at pointer_step
|
74
|
+
@flow=shared_steps.merge yaml[wf]
|
75
|
+
@pointer=pointer_step
|
76
|
+
run_step
|
77
|
+
end
|
78
|
+
|
74
79
|
def run_step
|
75
80
|
|
76
81
|
@step=flow[pointer]
|
@@ -81,7 +86,7 @@ module Shards
|
|
81
86
|
send pointer
|
82
87
|
@pointer=step['correct']
|
83
88
|
rescue => e
|
84
|
-
|
89
|
+
output_message e.message
|
85
90
|
@pointer=step['wrong']
|
86
91
|
end
|
87
92
|
|
@@ -91,6 +96,11 @@ module Shards
|
|
91
96
|
|
92
97
|
end
|
93
98
|
|
99
|
+
def output_message message
|
100
|
+
puts message
|
101
|
+
end
|
102
|
+
|
103
|
+
|
94
104
|
def ask_question
|
95
105
|
puts step['question']
|
96
106
|
@answer=$stdin.gets.chomp
|
@@ -190,6 +200,7 @@ module Shards
|
|
190
200
|
end
|
191
201
|
|
192
202
|
def create_database
|
203
|
+
stage.shard.db.blank_db = blank_db
|
193
204
|
stage.shard.db.create dryrun: false
|
194
205
|
message_params= [stage.shard.db.name, stage.shard.db.server]
|
195
206
|
raise_wrong_text message_params unless stage.shard.db.exist?
|
@@ -227,6 +238,16 @@ module Shards
|
|
227
238
|
@stage=location.stages[st]
|
228
239
|
end
|
229
240
|
|
241
|
+
def ask_blank_db
|
242
|
+
shards=stage.shard.list.keys
|
243
|
+
choose_one "Databases:", shards
|
244
|
+
@blank_db=shards[answer.to_i]
|
245
|
+
end
|
246
|
+
|
247
|
+
def add_ask_blank_db_step
|
248
|
+
@flow['write_yaml_files']['correct']='ask_blank_db'
|
249
|
+
end
|
250
|
+
|
230
251
|
end
|
231
252
|
|
232
253
|
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require_relative './base.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class CreateByApi < Shards::Workflow::Base
|
8
|
+
|
9
|
+
def write_yaml_files
|
10
|
+
stage.add_domain domain
|
11
|
+
site.add_site
|
12
|
+
end
|
13
|
+
|
14
|
+
def output_message message
|
15
|
+
message
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
@@ -6,19 +6,31 @@ module Shards
|
|
6
6
|
|
7
7
|
class DeleteByTerminal < Shards::Workflow::Base
|
8
8
|
|
9
|
-
attr_accessor :dns, :db, :
|
9
|
+
attr_accessor :dns, :db, :database_to_drop
|
10
10
|
|
11
11
|
def validate_subdomain
|
12
12
|
@dns=Shards::Dns.new stage: stage
|
13
13
|
dns.subdomain_param=params[:subdomain]
|
14
|
-
|
14
|
+
|
15
|
+
raise_wrong_text [dns.host_param, " doesn't exist"] unless dns.exist?
|
16
|
+
|
17
|
+
unless dns.include_target?
|
18
|
+
raise_wrong_text [dns.host_param, target_message_error ]
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def ask_to_continue
|
23
|
+
ask_question
|
24
|
+
raise step['wrong_message'] if answer.strip.downcase!='y'
|
15
25
|
end
|
16
26
|
|
17
27
|
def delete_record
|
18
|
-
dns.
|
19
|
-
|
20
|
-
|
21
|
-
|
28
|
+
if dns.exist?
|
29
|
+
dns.delete
|
30
|
+
message_params= [dns.host_param, dns.resource_records.to_s]
|
31
|
+
raise_wrong_text message_params if dns.exist?
|
32
|
+
puts step['message'] % message_params
|
33
|
+
end
|
22
34
|
end
|
23
35
|
|
24
36
|
def delete_site_and_shard
|
@@ -31,7 +43,7 @@ module Shards
|
|
31
43
|
shard.name_param = @shard_name
|
32
44
|
shard.delete
|
33
45
|
|
34
|
-
@
|
46
|
+
@database_to_drop=shard.database['database']
|
35
47
|
|
36
48
|
end
|
37
49
|
|
@@ -49,35 +61,53 @@ module Shards
|
|
49
61
|
end
|
50
62
|
|
51
63
|
def delete_database
|
52
|
-
@db=Shards::Db.new stage: stage, dns: dns
|
53
|
-
message_params= [db_name, db_host]
|
54
|
-
raise "Database %s does not exist in %s." % message_params unless db_exist?
|
55
64
|
|
56
|
-
db.
|
57
|
-
db.conn="mysql -u#{db_conn['username']} -p#{db_conn['password']} -h#{db_host} "
|
58
|
-
db.drop
|
65
|
+
@db=Shards::Db.new stage: stage
|
59
66
|
|
60
|
-
|
61
|
-
puts step['message'] % message_params
|
62
|
-
end
|
67
|
+
db.database_to_drop=database_to_drop
|
63
68
|
|
69
|
+
raise "Database %s does is not in present db servers" % database_to_drop unless db.exist?(database_to_drop)
|
64
70
|
|
65
|
-
|
71
|
+
db.servers_to_drop.each do |host|
|
66
72
|
|
67
|
-
|
68
|
-
|
69
|
-
|
73
|
+
puts
|
74
|
+
puts "Database:\t #{database_to_drop}"
|
75
|
+
puts "Host: \t #{host}"
|
76
|
+
puts
|
77
|
+
puts "Do you want to delete the above database? (y) to confirm, another string to not."
|
78
|
+
|
79
|
+
answer=$stdin.gets
|
80
|
+
|
81
|
+
puts
|
82
|
+
|
83
|
+
if answer.chomp.downcase=='y'
|
84
|
+
puts "DELETING database #{database_to_drop} in #{host}"
|
85
|
+
db.server=host
|
86
|
+
db.drop
|
87
|
+
else
|
88
|
+
puts "KEEP database #{database_to_drop} in #{host}"
|
89
|
+
end
|
90
|
+
puts
|
91
|
+
puts "-" * 50
|
92
|
+
puts
|
93
|
+
end
|
70
94
|
|
71
|
-
|
72
|
-
|
95
|
+
raise_wrong_text database_to_drop if db.exist? database_to_drop
|
96
|
+
puts step['message'] % database_to_drop
|
73
97
|
end
|
74
98
|
|
75
|
-
|
76
|
-
|
99
|
+
private
|
100
|
+
|
101
|
+
def target_message_error
|
102
|
+
message = []
|
103
|
+
message<< "Defined target: #{dns.target},"
|
104
|
+
message<< "not found in resource record values:"
|
105
|
+
message<< dns.record_data.to_s
|
106
|
+
message.join("\n\t")
|
77
107
|
end
|
78
108
|
|
79
109
|
end
|
80
110
|
|
81
111
|
end
|
82
112
|
|
83
|
-
|
113
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require_relative './fast_terminal.rb'
|
2
|
+
|
3
|
+
module Shards
|
4
|
+
|
5
|
+
module Workflow
|
6
|
+
|
7
|
+
class FastTerminalAll < Shards::Workflow::FastTerminal
|
8
|
+
|
9
|
+
def initialize config
|
10
|
+
super(config)
|
11
|
+
@wf='fast_terminal'
|
12
|
+
end
|
13
|
+
|
14
|
+
def start
|
15
|
+
@flow=shared_steps.merge yaml[wf]
|
16
|
+
|
17
|
+
@pointer=yaml['start']
|
18
|
+
run_step
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
data/test.env
CHANGED
@@ -8,10 +8,9 @@ SITES_FILE_NAME=sites.yml
|
|
8
8
|
SHARD_LIST_KEY_ROOT_PATH=octopus production migration_group_shards
|
9
9
|
CONFIG_DIR_SUFFIX=config_dir_suffix
|
10
10
|
DNS_ZONE_CODE=TestCodeDNSZone
|
11
|
-
DB_USER=db_user_name
|
12
|
-
DB_PASSWORD=HerePasswordDB
|
13
11
|
PROXY_USER=proxy_username
|
14
12
|
BASE_CONFIG_FILE=shards.yml
|
13
|
+
|
15
14
|
# optional:
|
16
15
|
TEST=true #only for test.env delete this var in settings.env
|
17
16
|
CONFIG_REPLACE={ 'prod' => 'web' }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: shards
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 2.0.1.pre.alpha
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Diego PL
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -191,6 +191,7 @@ files:
|
|
191
191
|
- ".rspec"
|
192
192
|
- ".ruby-version"
|
193
193
|
- ".travis.yml"
|
194
|
+
- ":w"
|
194
195
|
- CODE_OF_CONDUCT.md
|
195
196
|
- Gemfile
|
196
197
|
- Gemfile.lock
|
@@ -202,8 +203,10 @@ files:
|
|
202
203
|
- exe/shards
|
203
204
|
- lib/settings/format_methods.yaml
|
204
205
|
- lib/settings/workflows/base.yaml
|
206
|
+
- lib/settings/workflows/create_by_api.yaml
|
205
207
|
- lib/settings/workflows/delete_by_terminal.yaml
|
206
208
|
- lib/settings/workflows/fast_terminal.yaml
|
209
|
+
- lib/settings/workflows/fast_terminal_all.yaml
|
207
210
|
- lib/settings/workflows/terminal.yaml
|
208
211
|
- lib/shards.rb
|
209
212
|
- lib/shards/base.rb
|
@@ -221,8 +224,10 @@ files:
|
|
221
224
|
- lib/shards/stage.rb
|
222
225
|
- lib/shards/version.rb
|
223
226
|
- lib/shards/workflow/base.rb
|
227
|
+
- lib/shards/workflow/create_by_api.rb
|
224
228
|
- lib/shards/workflow/delete_by_terminal.rb
|
225
229
|
- lib/shards/workflow/fast_terminal.rb
|
230
|
+
- lib/shards/workflow/fast_terminal_all.rb
|
226
231
|
- lib/shards/workflow/terminal.rb
|
227
232
|
- lib/shards/workflow/workflow.rb
|
228
233
|
- shards.gemspec
|
@@ -243,9 +248,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
243
248
|
version: '0'
|
244
249
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
245
250
|
requirements:
|
246
|
-
- - "
|
251
|
+
- - ">"
|
247
252
|
- !ruby/object:Gem::Version
|
248
|
-
version:
|
253
|
+
version: 1.3.1
|
249
254
|
requirements: []
|
250
255
|
rubyforge_project:
|
251
256
|
rubygems_version: 2.6.14
|