smartmachine 0.8.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +661 -0
- data/README.md +147 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/exe/smartmachine +3 -0
- data/lib/smart_machine/apps/app.rb +165 -181
- data/lib/smart_machine/apps/container.rb +120 -0
- data/lib/smart_machine/apps/manager.rb +182 -0
- data/lib/smart_machine/base.rb +24 -6
- data/lib/smart_machine/buildpackers/buildpacker.rb +95 -0
- data/lib/smart_machine/{engine/buildpacks → buildpackers}/rails/Dockerfile +3 -3
- data/lib/smart_machine/buildpackers/rails.rb +221 -0
- data/lib/smart_machine/commands/app.rb +112 -0
- data/lib/smart_machine/commands/buildpacker.rb +53 -0
- data/lib/smart_machine/commands/credentials.rb +17 -0
- data/lib/smart_machine/commands/docker.rb +23 -0
- data/lib/smart_machine/commands/engine.rb +27 -0
- data/lib/smart_machine/commands/grid.rb +33 -0
- data/lib/smart_machine/commands/grid_commands/elasticsearch.rb +68 -0
- data/lib/smart_machine/commands/grid_commands/minio.rb +65 -0
- data/lib/smart_machine/commands/grid_commands/mysql.rb +71 -0
- data/lib/smart_machine/commands/grid_commands/nginx.rb +53 -0
- data/lib/smart_machine/commands/grid_commands/prereceiver.rb +96 -0
- data/lib/smart_machine/commands/grid_commands/redis.rb +65 -0
- data/lib/smart_machine/commands/grid_commands/scheduler.rb +15 -0
- data/lib/smart_machine/commands/grid_commands/sub_thor.rb +15 -0
- data/lib/smart_machine/commands/machine.rb +24 -0
- data/lib/smart_machine/commands/syncer.rb +23 -0
- data/lib/smart_machine/commands/utilities.rb +37 -0
- data/lib/smart_machine/commands.rb +66 -0
- data/lib/smart_machine/configuration.rb +79 -0
- data/lib/smart_machine/credentials.rb +93 -95
- data/lib/smart_machine/docker.rb +144 -143
- data/lib/smart_machine/engine/Dockerfile +7 -5
- data/lib/smart_machine/engine.rb +104 -79
- data/lib/smart_machine/grids/elasticsearch.rb +70 -89
- data/lib/smart_machine/grids/minio.rb +79 -68
- data/lib/smart_machine/grids/mysql.rb +207 -202
- data/lib/smart_machine/grids/nginx.rb +176 -135
- data/lib/smart_machine/grids/prereceiver/Dockerfile +2 -2
- data/lib/smart_machine/grids/prereceiver/pre-receive +17 -0
- data/lib/smart_machine/grids/prereceiver.rb +169 -169
- data/lib/smart_machine/grids/redis.rb +75 -57
- data/lib/smart_machine/grids/scheduler/Dockerfile +1 -1
- data/lib/smart_machine/logger.rb +26 -26
- data/lib/smart_machine/machine.rb +128 -194
- data/lib/smart_machine/scp.rb +15 -0
- data/lib/smart_machine/ssh.rb +69 -40
- data/lib/smart_machine/syncer.rb +133 -0
- data/lib/smart_machine/templates/dotsmartmachine/Gemfile +7 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/elasticsearch.yml +5 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/environment.rb +0 -9
- data/lib/smart_machine/templates/dotsmartmachine/config/minio.yml +13 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb +3 -3
- data/lib/smart_machine/templates/dotsmartmachine/config/mysql.yml +13 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/prereceiver.yml +7 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/redis.yml +15 -0
- data/lib/smart_machine/templates/dotsmartmachine/config/users.yml +1 -1
- data/lib/smart_machine/templates/dotsmartmachine/gitignore-template +47 -0
- data/lib/smart_machine/templates/dotsmartmachine/{grids/elasticsearch/data → vendor}/.keep +0 -0
- data/lib/smart_machine/version.rb +30 -6
- data/lib/smart_machine.rb +42 -16
- metadata +97 -47
- data/CHANGELOG.rdoc +0 -0
- data/MIT-LICENSE +0 -21
- data/README.rdoc +0 -87
- data/bin/buildpacker +0 -8
- data/bin/prereceiver +0 -8
- data/bin/scheduler +0 -18
- data/bin/smartmachine +0 -81
- data/bin/smartrunner +0 -33
- data/lib/smart_machine/apps/rails.rb +0 -250
- data/lib/smart_machine/apps.rb +0 -14
- data/lib/smart_machine/boot.rb +0 -32
- data/lib/smart_machine/buildpacker.rb +0 -106
- data/lib/smart_machine/gem_version.rb +0 -17
- data/lib/smart_machine/grids.rb +0 -18
- data/lib/smart_machine/sync.rb +0 -120
- data/lib/smart_machine/templates/dotsmartmachine/grids/elasticsearch/logs/.keep +0 -0
- data/lib/smart_machine/templates/dotsmartmachine/grids/minio/data/.keep +0 -0
- data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/backups/.keep +0 -0
- data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/data/.keep +0 -0
- data/lib/smart_machine/templates/dotsmartmachine/grids/prereceiver/pre-receive +0 -17
- data/lib/smart_machine/templates/dotsmartmachine/grids/redis/data/.keep +0 -0
- data/lib/smart_machine/user.rb +0 -38
@@ -1,203 +1,208 @@
|
|
1
|
-
# The main SmartMachine Grids Mysql driver
|
2
1
|
module SmartMachine
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
2
|
+
class Grids
|
3
|
+
class Mysql < SmartMachine::Base
|
4
|
+
def initialize(name:)
|
5
|
+
config = SmartMachine.config.grids.mysql.dig(name.to_sym)
|
6
|
+
raise "mysql config for #{name} not found." unless config
|
7
|
+
|
8
|
+
@port = config.dig(:port)
|
9
|
+
@root_password = config.dig(:root_password)
|
10
|
+
@username = config.dig(:username)
|
11
|
+
@password = config.dig(:password)
|
12
|
+
@database_name = config.dig(:database_name)
|
13
|
+
|
14
|
+
@name = name.to_s
|
15
|
+
@home_dir = File.expand_path('~')
|
16
|
+
@backups_path = "#{@home_dir}/smartmachine/grids/mysql/#{@name}/backups"
|
17
|
+
end
|
18
|
+
|
19
|
+
def uper
|
20
|
+
# Creating networks
|
21
|
+
unless system("docker network inspect #{@name}-network", [:out, :err] => File::NULL)
|
22
|
+
puts "-----> Creating network #{@name}-network ... "
|
23
|
+
if system("docker network create #{@name}-network", out: File::NULL)
|
24
|
+
puts "done"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/data")
|
29
|
+
FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/backups")
|
30
|
+
|
31
|
+
# Creating & Starting containers
|
32
|
+
puts "-----> Creating container #{@name} ... "
|
33
|
+
command = [
|
34
|
+
"docker create",
|
35
|
+
"--name='#{@name}'",
|
36
|
+
"--env MYSQL_ROOT_PASSWORD=#{@root_password}",
|
37
|
+
"--env MYSQL_USER=#{@username}",
|
38
|
+
"--env MYSQL_PASSWORD=#{@password}",
|
39
|
+
"--env MYSQL_DATABASE=#{@database_name}",
|
40
|
+
"--user `id -u`:`id -g`",
|
41
|
+
"--publish='#{@port}:#{@port}'",
|
42
|
+
"--volume='#{@home_dir}/smartmachine/grids/mysql/#{@name}/data:/var/lib/mysql'",
|
43
|
+
"--restart='always'",
|
44
|
+
"--network='#{@name}-network'",
|
45
|
+
"mysql:8.0.18"
|
46
|
+
]
|
47
|
+
if system(command.compact.join(" "), out: File::NULL)
|
48
|
+
puts "done"
|
49
|
+
puts "-----> Starting container #{@name} ... "
|
50
|
+
if system("docker start #{@name}", out: File::NULL)
|
51
|
+
puts "done"
|
52
|
+
else
|
53
|
+
raise "Error: Could not start the created #{@name} container"
|
54
|
+
end
|
55
|
+
else
|
56
|
+
raise "Error: Could not create #{@name} container"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# Stopping & Removing containers - in reverse order
|
61
|
+
def downer
|
62
|
+
puts "-----> Stopping container #{@name} ... "
|
63
|
+
if system("docker stop '#{@name}'", out: File::NULL)
|
64
|
+
puts "done"
|
65
|
+
puts "-----> Removing container #{@name} ... "
|
66
|
+
if system("docker rm '#{@name}'", out: File::NULL)
|
67
|
+
puts "done"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
# Removing networks
|
72
|
+
puts "-----> Removing network #{@name}-network ... "
|
73
|
+
if system("docker network rm #{@name}-network", out: File::NULL)
|
74
|
+
puts "done"
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Flushing logs
|
79
|
+
def flushlogs(*args)
|
80
|
+
puts "-----> Flushing logs for #{@name} ... "
|
81
|
+
if system("docker exec #{@name} sh -c \
|
82
|
+
'exec mysqladmin \
|
83
|
+
--user=root \
|
84
|
+
--password=#{@root_password} \
|
85
|
+
flush-logs'")
|
86
|
+
|
87
|
+
puts "done"
|
88
|
+
else
|
89
|
+
puts "error"
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
# Create backup using the grids backup command
|
94
|
+
def backup(*args)
|
95
|
+
args.flatten!
|
96
|
+
type = args.empty? ? '--snapshot' : args.shift
|
97
|
+
|
98
|
+
if type == "--daily"
|
99
|
+
run_backup(type: "daily")
|
100
|
+
elsif type == "--promote-to-weekly"
|
101
|
+
run_backup(type: "weekly")
|
102
|
+
elsif type == "--snapshot"
|
103
|
+
run_backup(type: "snapshot")
|
104
|
+
elsif type == "--transfer"
|
105
|
+
transfer_backups_to_external_storage
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
private
|
110
|
+
|
111
|
+
# Transfer all current backups to external storage
|
112
|
+
def transfer_backups_to_external_storage
|
113
|
+
end
|
114
|
+
|
115
|
+
def run_backup(type:)
|
116
|
+
FileUtils.mkdir_p("#{@backups_path}/#{type}")
|
117
|
+
|
118
|
+
unless type == "weekly"
|
119
|
+
standard_backup(type: type)
|
120
|
+
else
|
121
|
+
weekly_backup_from_latest_daily
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
def restore(type:, version:)
|
126
|
+
printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
|
127
|
+
prompt = STDIN.gets.chomp
|
128
|
+
return unless prompt == 'YES'
|
129
|
+
|
130
|
+
puts "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{@name} ... "
|
131
|
+
if system("docker exec -i #{@name} sh -c \
|
132
|
+
'exec xz < #{@backups_path}/#{type}/#{version}.sql.xz \
|
133
|
+
| mysql \
|
134
|
+
--user=root \
|
135
|
+
--password=#{@root_password}")
|
136
|
+
|
137
|
+
puts "done"
|
138
|
+
else
|
139
|
+
puts "error... check data & try again"
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
# Create a standard backup
|
144
|
+
def standard_backup(type:)
|
145
|
+
# Note: There should be no space between + and " in version.
|
146
|
+
# Note: date will be UTC date until timezone has been changed.
|
147
|
+
version = `date +"%Y%m%d%H%M%S"`.chomp!
|
148
|
+
backup_version_file = "#{version}.sql.xz"
|
149
|
+
|
150
|
+
puts "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{@name} ... "
|
151
|
+
if system("docker exec #{@name} sh -c \
|
152
|
+
'exec mysqldump \
|
153
|
+
--user=root \
|
154
|
+
--password=#{@root_password} \
|
155
|
+
--all-databases \
|
156
|
+
--single-transaction \
|
157
|
+
--flush-logs \
|
158
|
+
--master-data=2 \
|
159
|
+
--events \
|
160
|
+
--routines \
|
161
|
+
--triggers' \
|
162
|
+
| xz -9 > #{@backups_path}/#{type}/#{backup_version_file}")
|
163
|
+
|
164
|
+
puts "done"
|
165
|
+
|
166
|
+
clean_up(type: type)
|
167
|
+
else
|
168
|
+
puts "error... check data & try again"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
# Copy weekly backup from the daily backup
|
173
|
+
def weekly_backup_from_latest_daily
|
174
|
+
Dir.chdir("#{@backups_path}/daily") do
|
175
|
+
backup_versions = Dir.glob('*').sort
|
176
|
+
backup_version = backup_versions.last
|
177
|
+
|
178
|
+
if backup_version
|
179
|
+
puts "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
|
180
|
+
system("cp ./#{backup_version} ../weekly/#{backup_version}")
|
181
|
+
puts "done"
|
182
|
+
|
183
|
+
clean_up(type: "weekly")
|
184
|
+
else
|
185
|
+
puts "-----> Could not find daily backup to copy to weekly ... error"
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
# Clean up very old versions
|
191
|
+
def clean_up(type:)
|
192
|
+
keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
|
193
|
+
|
194
|
+
Dir.chdir("#{@backups_path}/#{type}") do
|
195
|
+
backup_versions = Dir.glob('*').sort
|
196
|
+
destroy_count = backup_versions.count - keep_releases[type.to_sym]
|
197
|
+
if destroy_count > 0
|
198
|
+
puts "Deleting older #{type} backups ... "
|
199
|
+
destroy_count.times do
|
200
|
+
FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
|
201
|
+
end
|
202
|
+
puts "done"
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|