smartmachine 0.8.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE.txt +661 -0
  3. data/README.md +147 -0
  4. data/bin/console +14 -0
  5. data/bin/setup +8 -0
  6. data/exe/smartmachine +3 -0
  7. data/lib/smart_machine/apps/app.rb +165 -181
  8. data/lib/smart_machine/apps/container.rb +120 -0
  9. data/lib/smart_machine/apps/manager.rb +182 -0
  10. data/lib/smart_machine/base.rb +24 -6
  11. data/lib/smart_machine/buildpackers/buildpacker.rb +95 -0
  12. data/lib/smart_machine/{engine/buildpacks → buildpackers}/rails/Dockerfile +3 -3
  13. data/lib/smart_machine/buildpackers/rails.rb +221 -0
  14. data/lib/smart_machine/commands/app.rb +112 -0
  15. data/lib/smart_machine/commands/buildpacker.rb +53 -0
  16. data/lib/smart_machine/commands/credentials.rb +17 -0
  17. data/lib/smart_machine/commands/docker.rb +23 -0
  18. data/lib/smart_machine/commands/engine.rb +27 -0
  19. data/lib/smart_machine/commands/grid.rb +33 -0
  20. data/lib/smart_machine/commands/grid_commands/elasticsearch.rb +68 -0
  21. data/lib/smart_machine/commands/grid_commands/minio.rb +65 -0
  22. data/lib/smart_machine/commands/grid_commands/mysql.rb +71 -0
  23. data/lib/smart_machine/commands/grid_commands/nginx.rb +53 -0
  24. data/lib/smart_machine/commands/grid_commands/prereceiver.rb +96 -0
  25. data/lib/smart_machine/commands/grid_commands/redis.rb +65 -0
  26. data/lib/smart_machine/commands/grid_commands/scheduler.rb +15 -0
  27. data/lib/smart_machine/commands/grid_commands/sub_thor.rb +15 -0
  28. data/lib/smart_machine/commands/machine.rb +24 -0
  29. data/lib/smart_machine/commands/syncer.rb +23 -0
  30. data/lib/smart_machine/commands/utilities.rb +37 -0
  31. data/lib/smart_machine/commands.rb +66 -0
  32. data/lib/smart_machine/configuration.rb +79 -0
  33. data/lib/smart_machine/credentials.rb +93 -95
  34. data/lib/smart_machine/docker.rb +144 -143
  35. data/lib/smart_machine/engine/Dockerfile +7 -5
  36. data/lib/smart_machine/engine.rb +104 -79
  37. data/lib/smart_machine/grids/elasticsearch.rb +70 -89
  38. data/lib/smart_machine/grids/minio.rb +79 -68
  39. data/lib/smart_machine/grids/mysql.rb +207 -202
  40. data/lib/smart_machine/grids/nginx.rb +176 -135
  41. data/lib/smart_machine/grids/prereceiver/Dockerfile +2 -2
  42. data/lib/smart_machine/grids/prereceiver/pre-receive +17 -0
  43. data/lib/smart_machine/grids/prereceiver.rb +169 -169
  44. data/lib/smart_machine/grids/redis.rb +75 -57
  45. data/lib/smart_machine/grids/scheduler/Dockerfile +1 -1
  46. data/lib/smart_machine/logger.rb +26 -26
  47. data/lib/smart_machine/machine.rb +128 -194
  48. data/lib/smart_machine/scp.rb +15 -0
  49. data/lib/smart_machine/ssh.rb +69 -40
  50. data/lib/smart_machine/syncer.rb +133 -0
  51. data/lib/smart_machine/templates/dotsmartmachine/Gemfile +7 -0
  52. data/lib/smart_machine/templates/dotsmartmachine/config/elasticsearch.yml +5 -0
  53. data/lib/smart_machine/templates/dotsmartmachine/config/environment.rb +0 -9
  54. data/lib/smart_machine/templates/dotsmartmachine/config/minio.yml +13 -0
  55. data/lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb +3 -3
  56. data/lib/smart_machine/templates/dotsmartmachine/config/mysql.yml +13 -0
  57. data/lib/smart_machine/templates/dotsmartmachine/config/prereceiver.yml +7 -0
  58. data/lib/smart_machine/templates/dotsmartmachine/config/redis.yml +15 -0
  59. data/lib/smart_machine/templates/dotsmartmachine/config/users.yml +1 -1
  60. data/lib/smart_machine/templates/dotsmartmachine/gitignore-template +47 -0
  61. data/lib/smart_machine/templates/dotsmartmachine/{grids/elasticsearch/data → vendor}/.keep +0 -0
  62. data/lib/smart_machine/version.rb +30 -6
  63. data/lib/smart_machine.rb +42 -16
  64. metadata +97 -47
  65. data/CHANGELOG.rdoc +0 -0
  66. data/MIT-LICENSE +0 -21
  67. data/README.rdoc +0 -87
  68. data/bin/buildpacker +0 -8
  69. data/bin/prereceiver +0 -8
  70. data/bin/scheduler +0 -18
  71. data/bin/smartmachine +0 -81
  72. data/bin/smartrunner +0 -33
  73. data/lib/smart_machine/apps/rails.rb +0 -250
  74. data/lib/smart_machine/apps.rb +0 -14
  75. data/lib/smart_machine/boot.rb +0 -32
  76. data/lib/smart_machine/buildpacker.rb +0 -106
  77. data/lib/smart_machine/gem_version.rb +0 -17
  78. data/lib/smart_machine/grids.rb +0 -18
  79. data/lib/smart_machine/sync.rb +0 -120
  80. data/lib/smart_machine/templates/dotsmartmachine/grids/elasticsearch/logs/.keep +0 -0
  81. data/lib/smart_machine/templates/dotsmartmachine/grids/minio/data/.keep +0 -0
  82. data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/backups/.keep +0 -0
  83. data/lib/smart_machine/templates/dotsmartmachine/grids/mysql/data/.keep +0 -0
  84. data/lib/smart_machine/templates/dotsmartmachine/grids/prereceiver/pre-receive +0 -17
  85. data/lib/smart_machine/templates/dotsmartmachine/grids/redis/data/.keep +0 -0
  86. data/lib/smart_machine/user.rb +0 -38
@@ -1,203 +1,208 @@
1
- # The main SmartMachine Grids Mysql driver
2
1
  module SmartMachine
3
- class Grids
4
- class Mysql < SmartMachine::Base
5
- def initialize
6
- end
7
-
8
- def up(*args)
9
- args.flatten!
10
- exposed = args.empty? ? '' : args.shift
11
-
12
- if SmartMachine::Docker.running?
13
- # Creating networks
14
- unless system("docker network inspect mysql-network", [:out, :err] => File::NULL)
15
- print "-----> Creating network mysql-network ... "
16
- if system("docker network create mysql-network", out: File::NULL)
17
- puts "done"
18
- end
19
- end
20
-
21
- # Creating & Starting containers
22
- print "-----> Creating container mysql ... "
23
- if system("docker create \
24
- --name='#{container_name}' \
25
- --env MYSQL_ROOT_PASSWORD=#{SmartMachine.credentials.mysql[:root_password]} \
26
- --env MYSQL_USER=#{SmartMachine.credentials.mysql[:username]} \
27
- --env MYSQL_PASSWORD=#{SmartMachine.credentials.mysql[:password]} \
28
- --env MYSQL_DATABASE=#{SmartMachine.credentials.mysql[:database_name]} \
29
- --user `id -u`:`id -g` \
30
- #{"--publish='#{SmartMachine.credentials.mysql[:port]}:#{SmartMachine.credentials.mysql[:port]}'" if exposed == '--exposed'} \
31
- --volume='#{SmartMachine.config.user_home_path}/.smartmachine/grids/mysql/data:/var/lib/mysql' \
32
- --restart='always' \
33
- --network='mysql-network' \
34
- mysql:8.0.18", out: File::NULL)
35
-
36
- puts "done"
37
- print "-----> Starting container mysql ... "
38
- if system("docker start mysql", out: File::NULL)
39
- puts "done"
40
- end
41
- end
42
- end
43
- end
44
-
45
- def down
46
- if SmartMachine::Docker.running?
47
- # Stopping & Removing containers - in reverse order
48
- print "-----> Stopping container mysql ... "
49
- if system("docker stop 'mysql'", out: File::NULL)
50
- puts "done"
51
- print "-----> Removing container mysql ... "
52
- if system("docker rm 'mysql'", out: File::NULL)
53
- puts "done"
54
- end
55
- end
56
-
57
- # Removing networks
58
- print "-----> Removing network mysql-network ... "
59
- if system("docker network rm mysql-network", out: File::NULL)
60
- puts "done"
61
- end
62
- end
63
- end
64
-
65
- # Flushing logs
66
- def flushlogs(*args)
67
- print "-----> Flushing logs for #{container_name} ... "
68
- if system("docker exec #{container_name} sh -c \
69
- 'exec mysqladmin \
70
- --user=root \
71
- --password=#{SmartMachine.credentials.mysql[:root_password]} \
72
- flush-logs'")
73
-
74
- puts "done"
75
- else
76
- puts "error"
77
- end
78
- end
79
-
80
- # Create backup using the grids backup command
81
- def backup(*args)
82
- args.flatten!
83
- type = args.empty? ? '--snapshot' : args.shift
84
-
85
- if type == "--daily"
86
- run_backup(type: "daily")
87
- elsif type == "--promote-to-weekly"
88
- run_backup(type: "weekly")
89
- elsif type == "--snapshot"
90
- run_backup(type: "snapshot")
91
- elsif type == "--transfer"
92
- transfer_backups_to_external_storage
93
- end
94
- end
95
-
96
- private
97
-
98
- # Transfer all current backups to external storage
99
- def transfer_backups_to_external_storage
100
- end
101
-
102
- def run_backup(type:)
103
- FileUtils.mkdir_p("#{backups_path}/#{type}")
104
-
105
- unless type == "weekly"
106
- standard_backup(type: type)
107
- else
108
- weekly_backup_from_latest_daily
109
- end
110
- end
111
-
112
- def restore(type:, version:)
113
- printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
114
- prompt = STDIN.gets.chomp
115
- return unless prompt == 'YES'
116
-
117
- print "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{container_name} ... "
118
- if system("docker exec -i #{container_name} sh -c \
119
- 'exec xz < #{backups_path}/#{type}/#{version}.sql.xz \
120
- | mysql \
121
- --user=root \
122
- --password=#{SmartMachine.credentials.mysql[:root_password]}")
123
-
124
- puts "done"
125
- else
126
- puts "error... check data & try again"
127
- end
128
- end
129
-
130
- # Create a standard backup
131
- def standard_backup(type:)
132
- # Note: There should be no space between + and " in version.
133
- # Note: date will be UTC date until timezone has been changed.
134
- version = `date +"%Y%m%d%H%M%S"`.chomp!
135
- backup_version_file = "#{version}.sql.xz"
136
-
137
- print "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{container_name} ... "
138
- if system("docker exec #{container_name} sh -c \
139
- 'exec mysqldump \
140
- --user=root \
141
- --password=#{SmartMachine.credentials.mysql[:root_password]} \
142
- --all-databases \
143
- --single-transaction \
144
- --flush-logs \
145
- --master-data=2 \
146
- --events \
147
- --routines \
148
- --triggers' \
149
- | xz -9 > #{backups_path}/#{type}/#{backup_version_file}")
150
-
151
- puts "done"
152
-
153
- clean_up(type: type)
154
- else
155
- puts "error... check data & try again"
156
- end
157
- end
158
-
159
- # Copy weekly backup from the daily backup
160
- def weekly_backup_from_latest_daily
161
- Dir.chdir("#{backups_path}/daily") do
162
- backup_versions = Dir.glob('*').sort
163
- backup_version = backup_versions.last
164
-
165
- if backup_version
166
- print "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
167
- system("cp ./#{backup_version} ../weekly/#{backup_version}")
168
- puts "done"
169
-
170
- clean_up(type: "weekly")
171
- else
172
- puts "-----> Could not find daily backup to copy to weekly ... error"
173
- end
174
- end
175
- end
176
-
177
- # Clean up very old versions
178
- def clean_up(type:)
179
- keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
180
-
181
- Dir.chdir("#{backups_path}/#{type}") do
182
- backup_versions = Dir.glob('*').sort
183
- destroy_count = backup_versions.count - keep_releases[type.to_sym]
184
- if destroy_count > 0
185
- print "Deleting older #{type} backups ... "
186
- destroy_count.times do
187
- FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
188
- end
189
- puts "done"
190
- end
191
- end
192
- end
193
-
194
- def backups_path
195
- "#{SmartMachine.config.user_home_path}/.smartmachine/grids/mysql/backups"
196
- end
197
-
198
- def container_name
199
- "mysql"
200
- end
201
- end
202
- end
203
- end
2
+ class Grids
3
+ class Mysql < SmartMachine::Base
4
+ def initialize(name:)
5
+ config = SmartMachine.config.grids.mysql.dig(name.to_sym)
6
+ raise "mysql config for #{name} not found." unless config
7
+
8
+ @port = config.dig(:port)
9
+ @root_password = config.dig(:root_password)
10
+ @username = config.dig(:username)
11
+ @password = config.dig(:password)
12
+ @database_name = config.dig(:database_name)
13
+
14
+ @name = name.to_s
15
+ @home_dir = File.expand_path('~')
16
+ @backups_path = "#{@home_dir}/smartmachine/grids/mysql/#{@name}/backups"
17
+ end
18
+
19
+ def uper
20
+ # Creating networks
21
+ unless system("docker network inspect #{@name}-network", [:out, :err] => File::NULL)
22
+ puts "-----> Creating network #{@name}-network ... "
23
+ if system("docker network create #{@name}-network", out: File::NULL)
24
+ puts "done"
25
+ end
26
+ end
27
+
28
+ FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/data")
29
+ FileUtils.mkdir_p("#{@home_dir}/machine/grids/mysql/#{@name}/backups")
30
+
31
+ # Creating & Starting containers
32
+ puts "-----> Creating container #{@name} ... "
33
+ command = [
34
+ "docker create",
35
+ "--name='#{@name}'",
36
+ "--env MYSQL_ROOT_PASSWORD=#{@root_password}",
37
+ "--env MYSQL_USER=#{@username}",
38
+ "--env MYSQL_PASSWORD=#{@password}",
39
+ "--env MYSQL_DATABASE=#{@database_name}",
40
+ "--user `id -u`:`id -g`",
41
+ "--publish='#{@port}:#{@port}'",
42
+ "--volume='#{@home_dir}/smartmachine/grids/mysql/#{@name}/data:/var/lib/mysql'",
43
+ "--restart='always'",
44
+ "--network='#{@name}-network'",
45
+ "mysql:8.0.18"
46
+ ]
47
+ if system(command.compact.join(" "), out: File::NULL)
48
+ puts "done"
49
+ puts "-----> Starting container #{@name} ... "
50
+ if system("docker start #{@name}", out: File::NULL)
51
+ puts "done"
52
+ else
53
+ raise "Error: Could not start the created #{@name} container"
54
+ end
55
+ else
56
+ raise "Error: Could not create #{@name} container"
57
+ end
58
+ end
59
+
60
+ # Stopping & Removing containers - in reverse order
61
+ def downer
62
+ puts "-----> Stopping container #{@name} ... "
63
+ if system("docker stop '#{@name}'", out: File::NULL)
64
+ puts "done"
65
+ puts "-----> Removing container #{@name} ... "
66
+ if system("docker rm '#{@name}'", out: File::NULL)
67
+ puts "done"
68
+ end
69
+ end
70
+
71
+ # Removing networks
72
+ puts "-----> Removing network #{@name}-network ... "
73
+ if system("docker network rm #{@name}-network", out: File::NULL)
74
+ puts "done"
75
+ end
76
+ end
77
+
78
+ # Flushing logs
79
+ def flushlogs(*args)
80
+ puts "-----> Flushing logs for #{@name} ... "
81
+ if system("docker exec #{@name} sh -c \
82
+ 'exec mysqladmin \
83
+ --user=root \
84
+ --password=#{@root_password} \
85
+ flush-logs'")
86
+
87
+ puts "done"
88
+ else
89
+ puts "error"
90
+ end
91
+ end
92
+
93
+ # Create backup using the grids backup command
94
+ def backup(*args)
95
+ args.flatten!
96
+ type = args.empty? ? '--snapshot' : args.shift
97
+
98
+ if type == "--daily"
99
+ run_backup(type: "daily")
100
+ elsif type == "--promote-to-weekly"
101
+ run_backup(type: "weekly")
102
+ elsif type == "--snapshot"
103
+ run_backup(type: "snapshot")
104
+ elsif type == "--transfer"
105
+ transfer_backups_to_external_storage
106
+ end
107
+ end
108
+
109
+ private
110
+
111
+ # Transfer all current backups to external storage
112
+ def transfer_backups_to_external_storage
113
+ end
114
+
115
+ def run_backup(type:)
116
+ FileUtils.mkdir_p("#{@backups_path}/#{type}")
117
+
118
+ unless type == "weekly"
119
+ standard_backup(type: type)
120
+ else
121
+ weekly_backup_from_latest_daily
122
+ end
123
+ end
124
+
125
+ def restore(type:, version:)
126
+ printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
127
+ prompt = STDIN.gets.chomp
128
+ return unless prompt == 'YES'
129
+
130
+ puts "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{@name} ... "
131
+ if system("docker exec -i #{@name} sh -c \
132
+ 'exec xz < #{@backups_path}/#{type}/#{version}.sql.xz \
133
+ | mysql \
134
+ --user=root \
135
+ --password=#{@root_password}")
136
+
137
+ puts "done"
138
+ else
139
+ puts "error... check data & try again"
140
+ end
141
+ end
142
+
143
+ # Create a standard backup
144
+ def standard_backup(type:)
145
+ # Note: There should be no space between + and " in version.
146
+ # Note: date will be UTC date until timezone has been changed.
147
+ version = `date +"%Y%m%d%H%M%S"`.chomp!
148
+ backup_version_file = "#{version}.sql.xz"
149
+
150
+ puts "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{@name} ... "
151
+ if system("docker exec #{@name} sh -c \
152
+ 'exec mysqldump \
153
+ --user=root \
154
+ --password=#{@root_password} \
155
+ --all-databases \
156
+ --single-transaction \
157
+ --flush-logs \
158
+ --master-data=2 \
159
+ --events \
160
+ --routines \
161
+ --triggers' \
162
+ | xz -9 > #{@backups_path}/#{type}/#{backup_version_file}")
163
+
164
+ puts "done"
165
+
166
+ clean_up(type: type)
167
+ else
168
+ puts "error... check data & try again"
169
+ end
170
+ end
171
+
172
+ # Copy weekly backup from the daily backup
173
+ def weekly_backup_from_latest_daily
174
+ Dir.chdir("#{@backups_path}/daily") do
175
+ backup_versions = Dir.glob('*').sort
176
+ backup_version = backup_versions.last
177
+
178
+ if backup_version
179
+ puts "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
180
+ system("cp ./#{backup_version} ../weekly/#{backup_version}")
181
+ puts "done"
182
+
183
+ clean_up(type: "weekly")
184
+ else
185
+ puts "-----> Could not find daily backup to copy to weekly ... error"
186
+ end
187
+ end
188
+ end
189
+
190
+ # Clean up very old versions
191
+ def clean_up(type:)
192
+ keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
193
+
194
+ Dir.chdir("#{@backups_path}/#{type}") do
195
+ backup_versions = Dir.glob('*').sort
196
+ destroy_count = backup_versions.count - keep_releases[type.to_sym]
197
+ if destroy_count > 0
198
+ puts "Deleting older #{type} backups ... "
199
+ destroy_count.times do
200
+ FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
201
+ end
202
+ puts "done"
203
+ end
204
+ end
205
+ end
206
+ end
207
+ end
208
+ end