smartmachine 0.8.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +661 -0
  3. data/README.md +147 -0
  4. data/bin/console +14 -0
  5. data/bin/setup +8 -0
  6. data/exe/smartmachine +3 -0
  7. data/lib/smart_machine/apps/app.rb +165 -181
  8. data/lib/smart_machine/apps/container.rb +120 -0
  9. data/lib/smart_machine/apps/manager.rb +182 -0
  10. data/lib/smart_machine/base.rb +24 -6
  11. data/lib/smart_machine/buildpackers/buildpacker.rb +95 -0
  12. data/lib/smart_machine/{engine/buildpacks → buildpackers}/rails/Dockerfile +3 -3
  13. data/lib/smart_machine/buildpackers/rails.rb +221 -0
  14. data/lib/smart_machine/commands/app.rb +112 -0
  15. data/lib/smart_machine/commands/buildpacker.rb +53 -0
  16. data/lib/smart_machine/commands/credentials.rb +17 -0
  17. data/lib/smart_machine/commands/docker.rb +23 -0
  18. data/lib/smart_machine/commands/engine.rb +27 -0
  19. data/lib/smart_machine/commands/grid.rb +33 -0
  20. data/lib/smart_machine/commands/grid_commands/elasticsearch.rb +68 -0
  21. data/lib/smart_machine/commands/grid_commands/minio.rb +65 -0
  22. data/lib/smart_machine/commands/grid_commands/mysql.rb +71 -0
  23. data/lib/smart_machine/commands/grid_commands/nginx.rb +53 -0
  24. data/lib/smart_machine/commands/grid_commands/prereceiver.rb +96 -0
  25. data/lib/smart_machine/commands/grid_commands/redis.rb +65 -0
  26. data/lib/smart_machine/commands/grid_commands/scheduler.rb +15 -0
  27. data/lib/smart_machine/commands/grid_commands/sub_thor.rb +15 -0
  28. data/lib/smart_machine/commands/machine.rb +24 -0
  29. data/lib/smart_machine/commands/syncer.rb +23 -0
  30. data/lib/smart_machine/commands/utilities.rb +37 -0
  31. data/lib/smart_machine/commands.rb +66 -0
  32. data/lib/smart_machine/configuration.rb +79 -0
  33. data/lib/smart_machine/credentials.rb +93 -95
  34. data/lib/smart_machine/docker.rb +144 -143
  35. data/lib/smart_machine/engine/Dockerfile +7 -5
  36. data/lib/smart_machine/engine.rb +104 -79
  37. data/lib/smart_machine/grids/elasticsearch.rb +70 -89
  38. data/lib/smart_machine/grids/minio.rb +79 -68
  39. data/lib/smart_machine/grids/mysql.rb +207 -202
  40. data/lib/smart_machine/grids/nginx.rb +176 -135
  41. data/lib/smart_machine/grids/prereceiver/Dockerfile +2 -2
  42. data/lib/smart_machine/grids/prereceiver/pre-receive +17 -0
  43. data/lib/smart_machine/grids/prereceiver.rb +169 -169
  44. data/lib/smart_machine/grids/redis.rb +75 -57
  45. data/lib/smart_machine/grids/scheduler/Dockerfile +1 -1
  46. data/lib/smart_machine/logger.rb +26 -26
  47. data/lib/smart_machine/machine.rb +128 -194
  48. data/lib/smart_machine/scp.rb +15 -0
  49. data/lib/smart_machine/ssh.rb +69 -40
  50. data/lib/smart_machine/syncer.rb +133 -0
  51. data/lib/smart_machine/templates/dotsmartmachine/Gemfile +7 -0
  52. data/lib/smart_machine/templates/dotsmartmachine/config/elasticsearch.yml +5 -0
  53. data/lib/smart_machine/templates/dotsmartmachine/config/environment.rb +0 -9
  54. data/lib/smart_machine/templates/dotsmartmachine/config/minio.yml +13 -0
  55. data/lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb +3 -3
  56. data/lib/smart_machine/templates/dotsmartmachine/config/mysql.yml +13 -0
  57. data/lib/smart_machine/templates/dotsmartmachine/config/prereceiver.yml +7 -0
  58. data/lib/smart_machine/templates/dotsmartmachine/config/redis.yml +15 -0
  59. data/lib/smart_machine/templates/dotsmartmachine/config/users.yml +1 -1
  60. data/lib/smart_machine/templates/dotsmartmachine/gitignore-template +47 -0
  61. data/lib/smart_machine/templates/dotsmartmachine/{grids/elasticsearch/data → vendor}/.keep +0 -0
  62. data/lib/smart_machine/version.rb +30 -6
  63. data/lib/smart_machine.rb +42 -16
  64. metadata +97 -47
  65. data/CHANGELOG.rdoc +0 -0
  66. data/MIT-LICENSE +0 -21
  67. data/README.rdoc +0 -87
  68. data/bin/buildpacker +0 -8
  69. data/bin/prereceiver +0 -8
  70. data/bin/scheduler +0 -18
  71. data/bin/smartmachine +0 -81
  72. data/bin/smartrunner +0 -33
  73. data/lib/smart_machine/apps/rails.rb +0 -250
  74. data/lib/smart_machine/apps.rb +0 -14
  75. data/lib/smart_machine/boot.rb +0 -32
  76. data/lib/smart_machine/buildpacker.rb +0 -106
  77. data/lib/smart_machine/gem_version.rb +0 -17
  78. data/lib/smart_machine/grids.rb +0 -18
  79. data/lib/smart_machine/sync.rb +0 -120
  80. data/lib/smart_machine/templates/dotsmartmachine/grids/elasticsearch/logs/.keep +0 -0
  81. data/lib/smart_machine/templates/dotsmartmachine/grids/minio/data/.keep +0 -0
  82. data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/backups/.keep +0 -0
  83. data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/data/.keep +0 -0
  84. data/lib/smart_machine/templates/dotsmartmachine/grids/prereceiver/pre-receive +0 -17
  85. data/lib/smart_machine/templates/dotsmartmachine/grids/redis/data/.keep +0 -0
  86. data/lib/smart_machine/user.rb +0 -38
@@ -1,203 +1,208 @@
1
- # The main SmartMachine Grids Mysql driver
2
1
  module SmartMachine
3
- class Grids
4
- class Mysql < SmartMachine::Base
5
- def initialize
6
- end
7
-
8
- def up(*args)
9
- args.flatten!
10
- exposed = args.empty? ? '' : args.shift
11
-
12
- if SmartMachine::Docker.running?
13
- # Creating networks
14
- unless system("docker network inspect mysql-network", [:out, :err] => File::NULL)
15
- print "-----> Creating network mysql-network ... "
16
- if system("docker network create mysql-network", out: File::NULL)
17
- puts "done"
18
- end
19
- end
20
-
21
- # Creating & Starting containers
22
- print "-----> Creating container mysql ... "
23
- if system("docker create \
24
- --name='#{container_name}' \
25
- --env MYSQL_ROOT_PASSWORD=#{SmartMachine.credentials.mysql[:root_password]} \
26
- --env MYSQL_USER=#{SmartMachine.credentials.mysql[:username]} \
27
- --env MYSQL_PASSWORD=#{SmartMachine.credentials.mysql[:password]} \
28
- --env MYSQL_DATABASE=#{SmartMachine.credentials.mysql[:database_name]} \
29
- --user `id -u`:`id -g` \
30
- #{"--publish='#{SmartMachine.credentials.mysql[:port]}:#{SmartMachine.credentials.mysql[:port]}'" if exposed == '--exposed'} \
31
- --volume='#{SmartMachine.config.user_home_path}/.smartmachine/grids/mysql/data:/var/lib/mysql' \
32
- --restart='always' \
33
- --network='mysql-network' \
34
- mysql:8.0.18", out: File::NULL)
35
-
36
- puts "done"
37
- print "-----> Starting container mysql ... "
38
- if system("docker start mysql", out: File::NULL)
39
- puts "done"
40
- end
41
- end
42
- end
43
- end
44
-
45
- def down
46
- if SmartMachine::Docker.running?
47
- # Stopping & Removing containers - in reverse order
48
- print "-----> Stopping container mysql ... "
49
- if system("docker stop 'mysql'", out: File::NULL)
50
- puts "done"
51
- print "-----> Removing container mysql ... "
52
- if system("docker rm 'mysql'", out: File::NULL)
53
- puts "done"
54
- end
55
- end
56
-
57
- # Removing networks
58
- print "-----> Removing network mysql-network ... "
59
- if system("docker network rm mysql-network", out: File::NULL)
60
- puts "done"
61
- end
62
- end
63
- end
64
-
65
- # Flushing logs
66
- def flushlogs(*args)
67
- print "-----> Flushing logs for #{container_name} ... "
68
- if system("docker exec #{container_name} sh -c \
69
- 'exec mysqladmin \
70
- --user=root \
71
- --password=#{SmartMachine.credentials.mysql[:root_password]} \
72
- flush-logs'")
73
-
74
- puts "done"
75
- else
76
- puts "error"
77
- end
78
- end
79
-
80
- # Create backup using the grids backup command
81
- def backup(*args)
82
- args.flatten!
83
- type = args.empty? ? '--snapshot' : args.shift
84
-
85
- if type == "--daily"
86
- run_backup(type: "daily")
87
- elsif type == "--promote-to-weekly"
88
- run_backup(type: "weekly")
89
- elsif type == "--snapshot"
90
- run_backup(type: "snapshot")
91
- elsif type == "--transfer"
92
- transfer_backups_to_external_storage
93
- end
94
- end
95
-
96
- private
97
-
98
- # Transfer all current backups to external storage
99
- def transfer_backups_to_external_storage
100
- end
101
-
102
- def run_backup(type:)
103
- FileUtils.mkdir_p("#{backups_path}/#{type}")
104
-
105
- unless type == "weekly"
106
- standard_backup(type: type)
107
- else
108
- weekly_backup_from_latest_daily
109
- end
110
- end
111
-
112
- def restore(type:, version:)
113
- printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
114
- prompt = STDIN.gets.chomp
115
- return unless prompt == 'YES'
116
-
117
- print "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{container_name} ... "
118
- if system("docker exec -i #{container_name} sh -c \
119
- 'exec xz < #{backups_path}/#{type}/#{version}.sql.xz \
120
- | mysql \
121
- --user=root \
122
- --password=#{SmartMachine.credentials.mysql[:root_password]}")
123
-
124
- puts "done"
125
- else
126
- puts "error... check data & try again"
127
- end
128
- end
129
-
130
- # Create a standard backup
131
- def standard_backup(type:)
132
- # Note: There should be no space between + and " in version.
133
- # Note: date will be UTC date until timezone has been changed.
134
- version = `date +"%Y%m%d%H%M%S"`.chomp!
135
- backup_version_file = "#{version}.sql.xz"
136
-
137
- print "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{container_name} ... "
138
- if system("docker exec #{container_name} sh -c \
139
- 'exec mysqldump \
140
- --user=root \
141
- --password=#{SmartMachine.credentials.mysql[:root_password]} \
142
- --all-databases \
143
- --single-transaction \
144
- --flush-logs \
145
- --master-data=2 \
146
- --events \
147
- --routines \
148
- --triggers' \
149
- | xz -9 > #{backups_path}/#{type}/#{backup_version_file}")
150
-
151
- puts "done"
152
-
153
- clean_up(type: type)
154
- else
155
- puts "error... check data & try again"
156
- end
157
- end
158
-
159
- # Copy weekly backup from the daily backup
160
- def weekly_backup_from_latest_daily
161
- Dir.chdir("#{backups_path}/daily") do
162
- backup_versions = Dir.glob('*').sort
163
- backup_version = backup_versions.last
164
-
165
- if backup_version
166
- print "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
167
- system("cp ./#{backup_version} ../weekly/#{backup_version}")
168
- puts "done"
169
-
170
- clean_up(type: "weekly")
171
- else
172
- puts "-----> Could not find daily backup to copy to weekly ... error"
173
- end
174
- end
175
- end
176
-
177
- # Clean up very old versions
178
- def clean_up(type:)
179
- keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
180
-
181
- Dir.chdir("#{backups_path}/#{type}") do
182
- backup_versions = Dir.glob('*').sort
183
- destroy_count = backup_versions.count - keep_releases[type.to_sym]
184
- if destroy_count > 0
185
- print "Deleting older #{type} backups ... "
186
- destroy_count.times do
187
- FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
188
- end
189
- puts "done"
190
- end
191
- end
192
- end
193
-
194
- def backups_path
195
- "#{SmartMachine.config.user_home_path}/.smartmachine/grids/mysql/backups"
196
- end
197
-
198
- def container_name
199
- "mysql"
200
- end
201
- end
202
- end
203
- end
2
+ class Grids
3
+ class Mysql < SmartMachine::Base
4
+ def initialize(name:)
5
+ config = SmartMachine.config.grids.mysql.dig(name.to_sym)
6
+ raise "mysql config for #{name} not found." unless config
7
+
8
+ @port = config.dig(:port)
9
+ @root_password = config.dig(:root_password)
10
+ @username = config.dig(:username)
11
+ @password = config.dig(:password)
12
+ @database_name = config.dig(:database_name)
13
+
14
+ @name = name.to_s
15
+ @home_dir = File.expand_path('~')
16
+ @backups_path = "#{@home_dir}/smartmachine/grids/mysql/#{@name}/backups"
17
+ end
18
+
19
+ def uper
20
+ # Creating networks
21
+ unless system("docker network inspect #{@name}-network", [:out, :err] => File::NULL)
22
+ puts "-----> Creating network #{@name}-network ... "
23
+ if system("docker network create #{@name}-network", out: File::NULL)
24
+ puts "done"
25
+ end
26
+ end
27
+
28
+ FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/data")
29
+ FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/backups")
30
+
31
+ # Creating & Starting containers
32
+ puts "-----> Creating container #{@name} ... "
33
+ command = [
34
+ "docker create",
35
+ "--name='#{@name}'",
36
+ "--env MYSQL_ROOT_PASSWORD=#{@root_password}",
37
+ "--env MYSQL_USER=#{@username}",
38
+ "--env MYSQL_PASSWORD=#{@password}",
39
+ "--env MYSQL_DATABASE=#{@database_name}",
40
+ "--user `id -u`:`id -g`",
41
+ "--publish='#{@port}:#{@port}'",
42
+ "--volume='#{@home_dir}/smartmachine/grids/mysql/#{@name}/data:/var/lib/mysql'",
43
+ "--restart='always'",
44
+ "--network='#{@name}-network'",
45
+ "mysql:8.0.18"
46
+ ]
47
+ if system(command.compact.join(" "), out: File::NULL)
48
+ puts "done"
49
+ puts "-----> Starting container #{@name} ... "
50
+ if system("docker start #{@name}", out: File::NULL)
51
+ puts "done"
52
+ else
53
+ raise "Error: Could not start the created #{@name} container"
54
+ end
55
+ else
56
+ raise "Error: Could not create #{@name} container"
57
+ end
58
+ end
59
+
60
+ # Stopping & Removing containers - in reverse order
61
+ def downer
62
+ puts "-----> Stopping container #{@name} ... "
63
+ if system("docker stop '#{@name}'", out: File::NULL)
64
+ puts "done"
65
+ puts "-----> Removing container #{@name} ... "
66
+ if system("docker rm '#{@name}'", out: File::NULL)
67
+ puts "done"
68
+ end
69
+ end
70
+
71
+ # Removing networks
72
+ puts "-----> Removing network #{@name}-network ... "
73
+ if system("docker network rm #{@name}-network", out: File::NULL)
74
+ puts "done"
75
+ end
76
+ end
77
+
78
+ # Flushing logs
79
+ def flushlogs(*args)
80
+ puts "-----> Flushing logs for #{@name} ... "
81
+ if system("docker exec #{@name} sh -c \
82
+ 'exec mysqladmin \
83
+ --user=root \
84
+ --password=#{@root_password} \
85
+ flush-logs'")
86
+
87
+ puts "done"
88
+ else
89
+ puts "error"
90
+ end
91
+ end
92
+
93
+ # Create backup using the grids backup command
94
+ def backup(*args)
95
+ args.flatten!
96
+ type = args.empty? ? '--snapshot' : args.shift
97
+
98
+ if type == "--daily"
99
+ run_backup(type: "daily")
100
+ elsif type == "--promote-to-weekly"
101
+ run_backup(type: "weekly")
102
+ elsif type == "--snapshot"
103
+ run_backup(type: "snapshot")
104
+ elsif type == "--transfer"
105
+ transfer_backups_to_external_storage
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ # Transfer all current backups to external storage
112
+ def transfer_backups_to_external_storage
113
+ end
114
+
115
+ def run_backup(type:)
116
+ FileUtils.mkdir_p("#{@backups_path}/#{type}")
117
+
118
+ unless type == "weekly"
119
+ standard_backup(type: type)
120
+ else
121
+ weekly_backup_from_latest_daily
122
+ end
123
+ end
124
+
125
+ def restore(type:, version:)
126
+ printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
127
+ prompt = STDIN.gets.chomp
128
+ return unless prompt == 'YES'
129
+
130
+ puts "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{@name} ... "
131
+ if system("docker exec -i #{@name} sh -c \
132
+ 'exec xz < #{@backups_path}/#{type}/#{version}.sql.xz \
133
+ | mysql \
134
+ --user=root \
135
+ --password=#{@root_password}")
136
+
137
+ puts "done"
138
+ else
139
+ puts "error... check data & try again"
140
+ end
141
+ end
142
+
143
+ # Create a standard backup
144
+ def standard_backup(type:)
145
+ # Note: There should be no space between + and " in version.
146
+ # Note: date will be UTC date until timezone has been changed.
147
+ version = `date +"%Y%m%d%H%M%S"`.chomp!
148
+ backup_version_file = "#{version}.sql.xz"
149
+
150
+ puts "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{@name} ... "
151
+ if system("docker exec #{@name} sh -c \
152
+ 'exec mysqldump \
153
+ --user=root \
154
+ --password=#{@root_password} \
155
+ --all-databases \
156
+ --single-transaction \
157
+ --flush-logs \
158
+ --master-data=2 \
159
+ --events \
160
+ --routines \
161
+ --triggers' \
162
+ | xz -9 > #{@backups_path}/#{type}/#{backup_version_file}")
163
+
164
+ puts "done"
165
+
166
+ clean_up(type: type)
167
+ else
168
+ puts "error... check data & try again"
169
+ end
170
+ end
171
+
172
+ # Copy weekly backup from the daily backup
173
+ def weekly_backup_from_latest_daily
174
+ Dir.chdir("#{@backups_path}/daily") do
175
+ backup_versions = Dir.glob('*').sort
176
+ backup_version = backup_versions.last
177
+
178
+ if backup_version
179
+ puts "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
180
+ system("cp ./#{backup_version} ../weekly/#{backup_version}")
181
+ puts "done"
182
+
183
+ clean_up(type: "weekly")
184
+ else
185
+ puts "-----> Could not find daily backup to copy to weekly ... error"
186
+ end
187
+ end
188
+ end
189
+
190
+ # Clean up very old versions
191
+ def clean_up(type:)
192
+ keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
193
+
194
+ Dir.chdir("#{@backups_path}/#{type}") do
195
+ backup_versions = Dir.glob('*').sort
196
+ destroy_count = backup_versions.count - keep_releases[type.to_sym]
197
+ if destroy_count > 0
198
+ puts "Deleting older #{type} backups ... "
199
+ destroy_count.times do
200
+ FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
201
+ end
202
+ puts "done"
203
+ end
204
+ end
205
+ end
206
+ end
207
+ end
208
+ end