smartmachine 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b3ee76cffa4f2e68ce10d0cf6e2ab0734520ab901fd4cd37dc53b6e15a1690a9
-  data.tar.gz: 9c48ce88369327fb8914ece890b48349bb27b389b1f71e384bb7611869ac3a9b
+  metadata.gz: abf46cdee1bfbf5a35837643f766eed7a83a824e04247cd2cb8a09af34dc6752
+  data.tar.gz: f51cc20363a52e15ba30f628d8dafd9897a15f9d5d4252e8bbf4d7af1d012282
 SHA512:
-  metadata.gz: c629d1bcf0900c3278fb21e9c794cee97f6f8842a9eadc411d13c4c606ffaa2b9e2ba4f2cc303dc79622e9bbf252e4cecaad72a39033f15747d22e7f972aee8b
-  data.tar.gz: 9bd5b57762ee72afad778b0883b6a96b430fa334784912e0e91502b9ea61b29befc687fc7bc4ee0bc39b2f7568c484d0fcf353764543f3afb34529b2c7ffc925
+  metadata.gz: cc1429cc9ad3bfb04741487b3620f17a5e103af6d817463631cabd9f6aacd85ad0d5921f3bb98275444f24bf78ead7e9c35c7eb17f4282c55946ebdc1e555039
+  data.tar.gz: 67879fc2681d7d32129795d7e8b60519997dfed8a40a60c174a2c7d19573e61b46b821fe80e0de7991fd8587451ddc983c865a6d7b1d161f3fd975b9593c55b3

data/bin/scheduler

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'smart_machine'
+
+command = ARGV.shift
+
+scheduler = SmartMachine::Grids::Scheduler.new
+
+case command
+when "start"
+	action = ARGV.shift
+	scheduler.mysql("start") if action == "--mysql"
+
+when "stop"
+	action = ARGV.shift
+	scheduler.mysql("stop") if action == "--mysql"
+
+end

data/bin/smartrunner

@@ -25,4 +25,9 @@ when "prereceiver"
 	prereceiver = SmartMachine::Grids::Prereceiver.new
 	prereceiver.public_send(action, *ARGV)
 
+when "scheduler"
+	action = ARGV.shift
+	scheduler = SmartMachine::Grids::Scheduler.new
+	scheduler.public_send(action, *ARGV)
+
 end

data/lib/smart_machine/boot.rb

@@ -24,6 +24,7 @@ require 'smart_machine/grids/minio'
 require 'smart_machine/grids/mysql'
 require 'smart_machine/grids/nginx'
 require 'smart_machine/grids/prereceiver'
+require 'smart_machine/grids/scheduler'
 require 'smart_machine/grids/solr'
 
 require 'smart_machine/apps'

data/lib/smart_machine/engine/Dockerfile

@@ -12,6 +12,7 @@ RUN sed -i "s/999/99/" /etc/group && \
 
 # Essentials
 RUN apk add --update build-base && \
+	apk add --update xz && \
 	apk add --update docker && \
 	apk add --update git && \
     rm -rf /var/cache/apk/*

data/lib/smart_machine/gem_version.rb

@@ -8,7 +8,7 @@ module SmartMachine
 
   module VERSION
     MAJOR = 0
-    MINOR = 7
+    MINOR = 8
     TINY  = 0
     PRE   = nil
 

data/lib/smart_machine/grids.rb

@@ -7,7 +7,10 @@ module SmartMachine
 			grid_name = args.shift
 			action = args.shift
 
-			raise "invalid action on the grid" unless ['up', 'down'].include? action
+			valid_actions = ['up', 'down']
+			valid_actions.push(*['backup', 'flushlogs']) if grid_name == 'mysql'
+			valid_actions.push(*["start", "stop"]) if grid_name == 'scheduler'
+			raise "invalid action on the grid #{grid_name}" unless valid_actions.include? action
 
 			Object.const_get("SmartMachine::Grids::#{grid_name.capitalize}").new.public_send(action, *args)
 		end

data/lib/smart_machine/grids/mysql.rb

@@ -21,7 +21,7 @@ module SmartMachine
 					# Creating & Starting containers
 					print "-----> Creating container mysql ... "
 					if system("docker create \
-						--name='mysql' \
+						--name='#{container_name}' \
 						--env MYSQL_ROOT_PASSWORD=#{SmartMachine.credentials.mysql[:root_password]} \
 						--env MYSQL_USER=#{SmartMachine.credentials.mysql[:username]} \
 						--env MYSQL_PASSWORD=#{SmartMachine.credentials.mysql[:password]} \
@@ -60,7 +60,144 @@ module SmartMachine
 						puts "done"
 					end
 				end
-			end			
+			end
+
+			# Flushing logs
+			def flushlogs(*args)
+				print "-----> Flushing logs for #{container_name} ... "
+				if system("docker exec #{container_name} sh -c \
+					'exec mysqladmin \
+					--user=root \
+					--password=#{SmartMachine.credentials.mysql[:root_password]} \
+					flush-logs'")
+
+					puts "done"
+				else
+					puts "error"
+				end
+			end
+
+			# Create backup using the grids backup command
+			def backup(*args)
+				args.flatten!
+				type = args.empty? ? '--snapshot' : args.shift
+
+				if type == "--daily"
+					run_backup(type: "daily")
+				elsif type == "--promote-to-weekly"
+					run_backup(type: "weekly")
+				elsif type == "--snapshot"
+					run_backup(type: "snapshot")
+				elsif type == "--transfer"
+					transfer_backups_to_external_storage
+				end
+			end
+
+			private
+
+			# Transfer all current backups to external storage
+			def transfer_backups_to_external_storage
+			end
+
+			def run_backup(type:)
+				FileUtils.mkdir_p("#{backups_path}/#{type}")
+
+				unless type == "weekly"
+					standard_backup(type: type)
+				else
+					weekly_backup_from_latest_daily
+				end
+			end
+
+			def restore(type:, version:)
+				printf "Are you sure you want to do this? It will destroy/overwrite all the current databases? Type 'YES' and press enter to continue: ".red
+				prompt = STDIN.gets.chomp
+				return unless prompt == 'YES'
+
+				print "-----> Restoring the backup of all databases with version #{version} (without binlogs) in #{container_name} ... "
+				if system("docker exec -i #{container_name} sh -c \
+					'exec xz < #{backups_path}/#{type}/#{version}.sql.xz \
+					| mysql \
+					--user=root \
+					--password=#{SmartMachine.credentials.mysql[:root_password]}")
+
+					puts "done"
+				else
+					puts "error... check data & try again"
+				end
+			end
+
+			# Create a standard backup
+			def standard_backup(type:)
+				# Note: There should be no space between + and " in version.
+				# Note: date will be UTC date until timezone has been changed.
+				version = `date +"%Y%m%d%H%M%S"`.chomp!
+				backup_version_file = "#{version}.sql.xz"
+
+				print "-----> Creating #{type} backup of all databases with backup version file #{backup_version_file} in #{container_name} ... "
+				if system("docker exec #{container_name} sh -c \
+					'exec mysqldump \
+					--user=root \
+					--password=#{SmartMachine.credentials.mysql[:root_password]} \
+					--all-databases \
+					--single-transaction \
+					--flush-logs \
+					--master-data=2 \
+					--events \
+					--routines \
+					--triggers' \
+					| xz -9 > #{backups_path}/#{type}/#{backup_version_file}")
+
+					puts "done"
+
+					clean_up(type: type)
+				else
+					puts "error... check data & try again"
+				end
+			end
+
+			# Copy weekly backup from the daily backup
+			def weekly_backup_from_latest_daily
+				Dir.chdir("#{backups_path}/daily") do
+					backup_versions = Dir.glob('*').sort
+					backup_version = backup_versions.last
+
+					if backup_version
+						print "-----> Creating weekly backup from daily backup version file #{backup_version} ... "
+						system("cp ./#{backup_version} ../weekly/#{backup_version}")
+						puts "done"
+
+						clean_up(type: "weekly")
+					else
+						puts "-----> Could not find daily backup to copy to weekly ... error"
+					end
+				end
+			end
+
+			# Clean up very old versions
+			def clean_up(type:)
+				keep_releases = { snapshot: 2, daily: 7, weekly: 3 }
+
+				Dir.chdir("#{backups_path}/#{type}") do
+					backup_versions = Dir.glob('*').sort
+					destroy_count = backup_versions.count - keep_releases[type.to_sym]
+					if destroy_count > 0
+						print "Deleting older #{type} backups ... "
+						destroy_count.times do
+							FileUtils.rm_r(File.join(Dir.pwd, backup_versions.shift))
+						end
+						puts "done"
+					end
+				end
+			end
+
+			def backups_path
+				"#{SmartMachine.config.user_home_path}/.smartmachine/grids/mysql/backups"
+			end
+
+			def container_name
+				"mysql"
+			end
 		end
 	end
 end

data/lib/smart_machine/grids/scheduler.rb

@@ -0,0 +1,188 @@
+# The main SmartMachine Grids Scheduler driver
+module SmartMachine
+	class Grids
+		class Scheduler < SmartMachine::Base
+
+			def initialize
+			end
+
+			def install
+				puts "-----> Installing Scheduler"
+
+				ssh = SmartMachine::SSH.new
+				commands = ["smartmachine runner scheduler create"]
+				ssh.run commands
+
+				puts "-----> Scheduler Installation Complete"
+			end
+
+			def uninstall
+				puts "-----> Uninstalling Scheduler"
+
+				ssh = SmartMachine::SSH.new
+				commands = ["smartmachine runner scheduler destroy"]
+				ssh.run commands
+
+				puts "-----> Scheduler Uninstallation Complete"
+			end
+
+			def update
+				uninstall
+				install
+			end
+
+			def create
+				unless system("docker image inspect #{scheduler_image_name}", [:out, :err] => File::NULL)
+					print "-----> Creating image #{scheduler_image_name} ... "
+					if system("docker image build -t #{scheduler_image_name} \
+									--build-arg SMARTMACHINE_VERSION=#{SmartMachine.version} \
+									#{SmartMachine.config.root_path}/lib/smart_machine/grids/scheduler", out: File::NULL)
+						puts "done"
+
+						up
+					end
+				end
+			end
+
+			def destroy
+				down
+
+				if system("docker image inspect #{scheduler_image_name}", [:out, :err] => File::NULL)
+					print "-----> Removing image #{scheduler_image_name} ... "
+					if system("docker image rm #{scheduler_image_name}", out: File::NULL)
+						puts "done"
+					end
+				end
+			end
+
+			def up
+				if SmartMachine::Docker.running?
+					if system("docker image inspect #{scheduler_image_name}", [:out, :err] => File::NULL)
+						print "-----> Creating container scheduler with image #{scheduler_image_name} ... "
+						if system("docker create \
+							--name='scheduler' \
+							--env MAILTO=#{SmartMachine.config.sysadmin_email} \
+							--user `id -u` \
+							--workdir /home/`id -un`/.smartmachine \
+							--volume='#{SmartMachine.config.user_home_path}/.smartmachine/config:#{SmartMachine.config.user_home_path}/.smartmachine/config' \
+							--volume='#{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler:#{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler' \
+							--volume='#{SmartMachine.config.user_home_path}/.smartmachine/bin/smartmachine.sh:/usr/local/bundle/bin/smartmachine' \
+							--volume='/var/run/docker.sock:/var/run/docker.sock:ro' \
+							--restart='always' \
+							#{scheduler_image_name}", out: File::NULL)
+							puts "done"
+
+							print "-----> Starting container scheduler with image #{scheduler_image_name} ... "
+							if system("docker start scheduler", out: File::NULL)
+								puts "done"
+
+								restore_crontabs
+							else
+								puts "error"
+							end
+						end
+					end
+				end
+			end
+	
+			def down
+				if SmartMachine::Docker.running?
+					# Stopping & Removing containers - in reverse order
+					if system("docker inspect -f '{{.State.Running}}' 'scheduler'", [:out, :err] => File::NULL)
+						print "-----> Stopping container scheduler with image #{scheduler_image_name} ... "
+						if system("docker stop 'scheduler'", out: File::NULL)
+							puts "done"
+
+							print "-----> Removing container scheduler with image #{scheduler_image_name} ... "
+							if system("docker rm 'scheduler'", out: File::NULL)
+								puts "done"
+							end
+						end
+					else
+						puts "-----> Container 'scheduler' is currently not running."
+					end
+				end
+			end
+
+
+			def start(*args)
+				args.flatten!
+				type = args.empty? ? '--all' : args.shift
+
+				self.class.running!
+
+				if type == '--mysql'
+					system("docker exec -i scheduler sh -c 'exec scheduler start --mysql'")
+				end
+			end
+
+			def stop(*args)
+				args.flatten!
+				type = args.empty? ? '--all' : args.shift
+
+				self.class.running!
+
+				if type == '--mysql'
+					system("docker exec -i scheduler sh -c 'exec scheduler stop --mysql'")
+				end
+			end
+
+			def mysql(*args)
+				args.flatten!
+				action = args.empty? ? '' : args.shift
+
+				return unless ['start', 'stop'].include? action
+
+				command = "whenever --set 'output=#{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler/crontabs/crontabs.log' --load-file #{SmartMachine.config.user_home_path}/.smartmachine/config/mysql/schedule.rb"
+
+				if action == 'start'
+					command += " --update-crontab"
+					action_text = "Starting"
+				elsif action == 'stop'
+					command += " --clear-crontab"
+					action_text = "Stopping"
+				end
+
+				print "-----> #{action_text} automatic backup schedule for mysql ... "
+				if system(command, out: File::NULL)
+					puts "done"
+				else
+					puts "error"
+				end
+			ensure
+				backup_crontabs
+			end
+
+			def scheduler_image_name
+				"smartmachine/scheduler:#{SmartMachine.version}"
+			end
+
+			def restore_crontabs
+				if system("docker exec scheduler sh -c 'exec test -f #{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler/crontabs/`id -un`'")
+					print "-----> Restoring latest crontabs ... "
+
+					if system("docker exec scheduler sh -c 'exec crontab - < #{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler/crontabs/`id -un`'")
+						puts "done"
+					else
+						puts "error"
+					end
+				end
+			end
+
+			def backup_crontabs
+				print "-----> Backing up latest crontabs ... "
+				if system("crontab -l > #{SmartMachine.config.user_home_path}/.smartmachine/grids/scheduler/crontabs/`id -un`", out: File::NULL)
+					puts "done"
+				else
+					puts "error"
+				end
+			end
+
+			def self.running!
+				unless system("docker inspect -f '{{.State.Running}}' 'scheduler'", [:out, :err] => File::NULL)
+					raise "Scheduler is not running. Please start scheduler before scheduling"
+				end
+			end
+		end
+	end
+end

data/lib/smart_machine/grids/scheduler/Dockerfile

@@ -0,0 +1,14 @@
+ARG SMARTMACHINE_VERSION
+
+FROM smartmachine:$SMARTMACHINE_VERSION
+LABEL maintainer="Timeboard <hello@timeboard.me>"
+
+# Scheduler Essentials
+RUN apk add --update busybox-suid && \
+	apk add --update bash && \
+    rm -rf /var/cache/apk/*
+
+# Gems
+RUN gem install whenever -v 1.0.0
+
+CMD ["crond", "-f", "-l", "0", "-L", "/dev/stdout"]

data/lib/smart_machine/machine.rb

@@ -64,6 +64,11 @@ module SmartMachine
 				prereceiver.public_send(action)
 			end
 
+			if args.empty? || args.include?("scheduler")
+				scheduler = SmartMachine::Grids::Scheduler.new
+				scheduler.public_send(action)
+			end
+
 			if args.empty? || args.include?("elasticsearch")
 				elasticsearch = SmartMachine::Grids::Elasticsearch.new
 				elasticsearch.public_send(action)

data/lib/smart_machine/sync.rb

@@ -43,11 +43,15 @@ module SmartMachine
 				'grids/minio/data/***',
 
 				'grids/mysql',
+				'grids/mysql/backups/***',
 				'grids/mysql/data/***',
 
 				'grids/nginx',
 				'grids/nginx/certificates/***',
 
+				'grids/scheduler',
+				'grids/scheduler/crontabs/***',
+
 				'grids/solr',
 				'grids/solr/solr/***',
 			]
@@ -63,6 +67,8 @@ module SmartMachine
 				'apps/repositories/.keep',
 
 				'config',
+				'config/mysql',
+				'config/mysql/schedule.rb',
 				'config/credentials.yml.enc',
 				'config/environment.rb',
 
@@ -79,6 +85,8 @@ module SmartMachine
 				'grids/minio/data/.keep',
 
 				'grids/mysql',
+				'grids/mysql/backups',
+				'grids/mysql/backups/.keep',
 				'grids/mysql/data',
 				'grids/mysql/data/.keep',
 
@@ -96,6 +104,10 @@ module SmartMachine
 				'grids/redis/data',
 				'grids/redis/data/.keep',
 
+				'grids/scheduler',
+				'grids/scheduler/crontabs',
+				'grids/scheduler/crontabs/.keep',
+
 				'grids/solr',
 				'grids/solr/solr',
 				'grids/solr/solr/.keep',

data/lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb

@@ -0,0 +1,19 @@
+# Recommended defaults have been setup.
+# This file should be changed only when you know what you are doing.
+
+# Flush logs used to create consistent binlogs to be used for incremental backups
+every :day, at: '12:00 pm' do
+	command "smartmachine runner grids mysql flushlogs"
+end
+
+# Create daily backup.
+# This also flushes the logs before backup
+every :day, at: '12:00 am' do
+	command "smartmachine runner grids mysql backup --daily"
+end
+
+# Promote currently latest daily backup to weekly backup.
+# This is only possible when a daily backup creation has already been completed.
+every :monday, at: '3:00 am' do
+	command "smartmachine runner grids mysql backup --promote-to-weekly"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: smartmachine
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Timeboard
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-05-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: net-ssh
@@ -58,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: whenever
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: SmartMachine is a full-stack deployment framework for rails optimized
   for admin programmer happiness and peaceful administration. It encourages natural
   simplicity by favoring convention over configuration.
@@ -65,6 +79,7 @@ email: hello@timeboard.me
 executables:
 - buildpacker
 - prereceiver
+- scheduler
 - smartmachine
 - smartrunner
 extensions: []
@@ -76,6 +91,7 @@ files:
 - README.rdoc
 - bin/buildpacker
 - bin/prereceiver
+- bin/scheduler
 - bin/smartmachine
 - bin/smartrunner
 - lib/smart_machine.rb
@@ -112,6 +128,8 @@ files:
 - lib/smart_machine/grids/prereceiver/fcgiwrap/packages/main/x86_64/fcgiwrap-openrc-1.1.1-r4.apk
 - lib/smart_machine/grids/redis.rb
 - lib/smart_machine/grids/redis/.keep
+- lib/smart_machine/grids/scheduler.rb
+- lib/smart_machine/grids/scheduler/Dockerfile
 - lib/smart_machine/grids/solr.rb
 - lib/smart_machine/grids/solr/config/.keep
 - lib/smart_machine/grids/solr/config/README.txt
@@ -283,10 +301,12 @@ files:
 - lib/smart_machine/templates/dotsmartmachine/apps/containers/.keep
 - lib/smart_machine/templates/dotsmartmachine/apps/repositories/.keep
 - lib/smart_machine/templates/dotsmartmachine/config/environment.rb
+- lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb
 - lib/smart_machine/templates/dotsmartmachine/config/users.yml
 - lib/smart_machine/templates/dotsmartmachine/grids/elasticsearch/data/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/elasticsearch/logs/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/minio/data/.keep
+- lib/smart_machine/templates/dotsmartmachine/grids/mysql/backups/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/mysql/data/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/nginx/certificates/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/nginx/fastcgi.conf
@@ -294,6 +314,7 @@ files:
 - lib/smart_machine/templates/dotsmartmachine/grids/nginx/nginx.tmpl
 - lib/smart_machine/templates/dotsmartmachine/grids/prereceiver/pre-receive
 - lib/smart_machine/templates/dotsmartmachine/grids/redis/data/.keep
+- lib/smart_machine/templates/dotsmartmachine/grids/scheduler/crontabs/.keep
 - lib/smart_machine/templates/dotsmartmachine/grids/solr/solr/.keep
 - lib/smart_machine/templates/dotsmartmachine/tmp/.keep
 - lib/smart_machine/user.rb
@@ -303,8 +324,8 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/timeboarders/smartbytes/issues
-  changelog_uri: https://github.com/timeboarders/smartbytes/blob/v0.7.0/smartmachine/CHANGELOG.rdoc
-  source_code_uri: https://github.com/timeboarders/smartbytes/tree/v0.7.0/smartmachine
+  changelog_uri: https://github.com/timeboarders/smartbytes/blob/v0.8.0/smartmachine/CHANGELOG.rdoc
+  source_code_uri: https://github.com/timeboarders/smartbytes/tree/v0.8.0/smartmachine
 post_install_message: 
 rdoc_options:
 - "--main"