smartcloud 0.2.0.beta2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4051b840648e71abfa206f4086f9b88a0b32a3ffa298552051b01e7d400a406e
-  data.tar.gz: 31797b7b8e04c51cad59cfe18edf3006b8faa2b8bd8149b51a35d7e6bc0508c3
+  metadata.gz: 77ed71de3cc780a8cd8672f942b0f31691309159dc941b88bef8be0fbfc36561
+  data.tar.gz: db7906edf628c1d762415148c1932bd86feb01dfee647c7ca8b075f095689432
 SHA512:
-  metadata.gz: c14a5a273256e5c9addf84b93ba11370038f7f574baba0b6113958626188718ff51f2aaac1482b23c7d935c8f05e09d5d90b0e69085c3bade2b92f45a084b6aa
-  data.tar.gz: d3461f67d5b2b7c22d890d3fc78d3ef96ac7b03f55cb3f553cb523e0320022823fac6ea86b946357ff4c19d9ce28927a5b05d653c1859749248aea542cdfb0a1
+  metadata.gz: 07f54bcb600bdeaef2405a741b9a05b7522264d517d7999a5b7dbbec0214d5552deba01567862b458e1c94fea0acd97e1aef2b574ff71d9e5c9b1af0e8490d05
+  data.tar.gz: a29bcfb7e3c84b5de7038ce9148a1d17772b26535aac05665b0c56b610c2d06c2a663fcbf023150e990fe20c6081d738c74934896c277e61855670118b78f69b

data/README.md

@@ -8,7 +8,7 @@ Deploy your Rails apps to your own server with - git push production master
 After you run the below commands, you get.
 1. Setup of basic best practices of setting up and securing a VPS server.
 2. Setup and installation of Docker.
-3. Setup and installation of docker based Mysql, Solr, Nginx, App Runner.
+3. Setup and installation of docker based Mysql, Solr, Nginx, App Prereceiver.
 4. Deployment of your Rails apps to your own server with - git push production master
 
 ## Setup a New Machine - Ubuntu 18.04 LTS
@@ -67,9 +67,9 @@ $ smartcloud grids solr up
 ```
 $ smartcloud grids nginx up
 ```
-4. Start runner grid:
+4. Start prereceiver grid:
 ```
-$ smartcloud grids runner up
+$ smartcloud grids prereceiver up
 ```
 
 ## TODO - Creating New App

data/bin/buildpacker

@@ -2,7 +2,7 @@
 
 require 'smartcloud'
 
-if ARGV[0] == 'start'
-	buildpacker = Smartcloud::Grids::Buildpacker.new
+if ARGV[0] == 'pack'
+	buildpacker = Smartcloud::Buildpacker.new
 	buildpacker.pack
 end

data/bin/prereceiver

@@ -0,0 +1,8 @@
+#!/usr/bin/env ruby
+
+require 'smartcloud'
+
+if ARGV[0] == 'prereceive'
+	raise "Please provide appname, username, oldrev, newrev and refname" unless ARGV[1] && ARGV[2] && ARGV[3] && ARGV[4] && ARGV[5]
+	Smartcloud::Grids::Prereceiver.prereceive(ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5])
+end

data/bin/smartcloud

@@ -4,32 +4,77 @@ require 'smartcloud'
 
 command = ARGV.shift
 
-unless command == 'new'
-	unless Smartcloud::Machine.smartcloud_dir?
-		raise "This is not a smartcloud directory. Are you in the correct directory?"
-	end
+if command == '--version'
+	puts "Smartcloud #{Smartcloud.version}"
+	return
 end
 
-if command == 'new'
-	raise "This is already a smartcloud directory. Are you in the correct directory?" if Smartcloud::Machine.smartcloud_dir?
-	raise "Please specify a machine name" if ARGV.empty?
-	machine = Smartcloud::Machine.new
-	machine.create ARGV
-elsif command == 'start'
-	machine = Smartcloud::Machine.new
-	machine.start
-elsif command == 'stop'
-	machine = Smartcloud::Machine.new
-	machine.stop
-elsif command == 'grid'
-	machine = Smartcloud::Machine.new
-	machine.grid ARGV
-elsif command == 'ssh'
-	machine = Smartcloud::Machine.new
-	machine.ssh
-elsif command == 'sync'
-	machine = Smartcloud::Machine.new
-	machine.sync
+if Smartcloud::Machine.smartcloud_local?
+	if command == 'start'
+		machine = Smartcloud::Machine.new
+		machine.start
+	elsif command == 'stop'
+		machine = Smartcloud::Machine.new
+		machine.stop
+	elsif command == 'ssh'
+		machine = Smartcloud::Machine.new
+		machine.ssh
+	elsif command == 'sync'
+		machine = Smartcloud::Machine.new
+		machine.sync
+	elsif command == 'grid'
+		machine = Smartcloud::Machine.new
+		machine.grid ARGV
+	elsif command == 'app'
+		machine = Smartcloud::Machine.new
+		machine.app ARGV
+	elsif command == 'ps'
+		ssh = Smartcloud::SSH.new
+		ssh.run "smartcloud ps #{ARGV.join(' ')}"
+	elsif command == 'logs'
+		ssh = Smartcloud::SSH.new
+		ssh.run "smartcloud logs #{ARGV.join(' ')}"
+	elsif command == 'credentials:edit'
+		credentials = Smartcloud::Credentials.new
+		credentials.edit
+	else
+		puts "Specify a valid local command to execute."
+	end
+elsif Smartcloud::Machine.smartcloud_server?
+	if command == 'run'
+		machine = Smartcloud::Machine.new
+		machine.run ARGV
+	elsif command == 'buildpacker'
+		buildpacker = Smartcloud::Buildpacker.new
+		action = ARGV.shift
+		if action == "install"
+			buildpacker.install
+		elsif action == "uninstall"
+			buildpacker.uninstall
+		end
+	elsif command == 'prereceiver'
+		prereceiver = Smartcloud::Grids::Prereceiver.new
+		action = ARGV.shift
+		if action == "install"
+			prereceiver.install
+		elsif action == "uninstall"
+			prereceiver.uninstall
+		end
+	elsif command == 'ps'
+		exec "docker ps #{ARGV.join(' ')}"
+	elsif command == 'logs'
+		exec "docker logs #{ARGV.join(' ')}"
+	elsif command == 'shell'
+		exec '/bin/sh -c sh'
+	else
+		puts "Specify a valid server command to execute."
+	end
 else
-	puts "Specify a command to execute."
+	if command == 'new'
+		raise "Please specify a machine name" if ARGV.empty?
+		machine = Smartcloud::Machine.new
+		machine.create ARGV
+	else
+		puts "Specify a valid smartcloud command to execute. Are you in the correct directory to run this command?"
+	end
 end

data/lib/smartcloud.rb

@@ -8,18 +8,12 @@ module Smartcloud
 	end
 
 	def self.credentials
-		@@credentials ||= OpenStruct.new(Smartcloud.transform_keys_to_symbols(YAML.load_file("#{Dir.pwd}/config/credentials.yml")))
-	end
-
-	private
-
-	def self.transform_keys_to_symbols(value)
-		return value if not value.is_a?(Hash)
-		hash = value.inject({}){|memo,(k,v)| memo[k.to_sym] = self.transform_keys_to_symbols(v); memo}
-		return hash
+		@@credentials ||= OpenStruct.new(Smartcloud::Credentials.new.config)
 	end
 end
 
+require 'smartcloud/version'
+
 Smartcloud.config.root_path = File.expand_path('../..', __FILE__)
 Smartcloud.config.user_home_path = File.expand_path('~')
 if File.exist?("#{Smartcloud.config.user_home_path}/.smartcloud/config/environment.rb")

data/lib/smartcloud/apps/app.rb

@@ -0,0 +1,182 @@
+# The main Smartcloud App driver
+module Smartcloud
+	module Apps
+		class App < Smartcloud::Base
+			def initialize
+			end
+
+			# Creating App!
+			#
+			# Example:
+			#   >> Smartcloud::Apps::Rails.create
+			#   => Creation Complete
+			#
+			# Arguments:
+			#   appname => (String)
+			# 	username => (String)
+			def self.create(*args)
+				args.flatten!
+				appname = args.empty? ? '' : args.shift
+				username = args.empty? ? '' : args.shift
+
+				raise "Please provide appname and username" if appname.empty? || username.empty?
+
+				if Smartcloud::Docker.running?
+					repository_path = "#{Smartcloud.config.user_home_path}/.smartcloud/apps/repositories/#{appname}.git"
+					container_path = "#{Smartcloud.config.user_home_path}/.smartcloud/apps/containers/#{appname}"
+					print "-----> Creating Application ... "
+
+					# Checking if app with given name already exists
+					if Dir.exist?(repository_path)
+						puts "failed. App with name '#{appname}' already exists."
+						exit
+					end
+
+					# Creating Directories
+					FileUtils.mkdir_p(repository_path)
+					FileUtils.mkdir_p(container_path)
+
+					# Initializing bare repo and pre-receive
+					Dir.chdir(repository_path) do
+						%x[git init --bare]
+						%x[chmod +x #{Smartcloud.config.user_home_path}/.smartcloud/grids/grid-prereceiver/pre-receive]
+						%x[ln -s #{Smartcloud.config.user_home_path}/.smartcloud/grids/grid-prereceiver/pre-receive #{repository_path}/hooks/pre-receive]
+						puts "done"
+					end
+
+					# Creating Environment File
+					if File.exist?("#{Smartcloud.config.user_home_path}/.smartcloud/config/environment.rb")
+						require "#{Smartcloud.config.user_home_path}/.smartcloud/config/environment"
+					end
+					unless File.exist? "#{container_path}/env"
+						print "-----> Creating App Environment ... "
+						page = <<~HEREDOC
+							## System
+							USERNAME=#{username}
+							KEEP_RELEASES=3
+
+							## Docker
+							VIRTUAL_HOST=#{appname}.#{Smartcloud.config.apps_domain}
+							LETSENCRYPT_HOST=#{appname}.#{Smartcloud.config.apps_domain}
+							LETSENCRYPT_EMAIL=#{Smartcloud.config.sysadmin_email}
+							LETSENCRYPT_TEST=false
+						HEREDOC
+						puts "done" if system("echo '#{page}' > #{container_path}/env")
+					end
+				end
+			end
+
+			def self.destroy(*args)
+				args.flatten!
+				appname = args.empty? ? '' : args.shift
+
+				raise "Please provide appname" if appname.empty?
+
+				if Smartcloud::Docker.running?
+					# Stopping & Removing old container
+					self.stop(appname)
+
+					# Destroying Directories
+					print "-----> Deleting App #{appname} ... "
+					repository_path = "#{Smartcloud.config.user_home_path}/.smartcloud/apps/repositories/#{appname}.git"
+					container_path = "#{Smartcloud.config.user_home_path}/.smartcloud/apps/containers/#{appname}"
+					FileUtils.rm_r(repository_path)
+					FileUtils.rm_r(container_path)
+					puts "done"
+				end
+			end
+			
+			def self.start(*args)
+				args.flatten!
+				appname = args.empty? ? '' : args.shift
+				app_version = args.empty? ? 0 : args.shift.to_i
+
+				raise "Please provide appname" if appname.empty?
+
+				logger.formatter = proc do |severity, datetime, progname, message|
+					severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+						"ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+					}
+					"\t\t\t\t#{severity_text[severity]} #{message}\n"
+				end
+
+				if Smartcloud::Docker.running?
+					container_path = "#{Smartcloud.config.user_home_path}/.smartcloud/apps/containers/#{appname}"
+
+					Dir.chdir("#{container_path}/releases") do
+						# Getting App Version
+						if app_version == 0
+							app_versions = Dir.glob('*').select { |f| File.directory? f }.sort
+							app_version = app_versions.last
+						end
+						container_path_with_version = "#{container_path}/releases/#{app_version}"
+
+						logger.info "Launching Application ..."
+
+						app = Smartcloud::Apps::Rails.new
+						app.start(appname, container_path, container_path_with_version)
+					end
+				end
+
+				logger.formatter = nil
+			end
+
+			def self.stop(*args)
+				args.flatten!
+				appname = args.empty? ? '' : args.shift
+
+				raise "Please provide appname" if appname.empty?
+
+				container_name = appname
+
+				if Smartcloud::Docker.running?
+					container_id = `docker ps -a -q --filter='name=^#{container_name}$'`.chomp
+					unless container_id.empty?
+						logger.debug "Stopping & Removing container #{container_name} ..."
+						if system("docker stop #{container_name} && docker rm #{container_name}", out: File::NULL)
+							logger.debug "Stopped & Removed container #{container_name} ..."
+						end
+					else
+						logger.debug "Container '#{container_name}' does not exist to stop."
+					end
+				end
+			end
+
+			def self.clean_up(container_path)
+				env_vars = Smartcloud::Apps::App.get_env_vars(container_path)
+				return unless env_vars
+
+				logger.info "Cleaning up ..."
+
+				# Clean up very old versions
+				Dir.chdir("#{container_path}/releases") do
+					app_versions = Dir.glob('*').select { |f| File.directory? f }.sort
+					destroy_count = app_versions.count - env_vars['KEEP_RELEASES'].to_i
+					if destroy_count > 0
+						logger.debug "Deleting older application releases ..."
+						destroy_count.times do
+							FileUtils.rm_r(File.join(Dir.pwd, app_versions.shift))
+						end
+					end
+				end
+			end
+
+			def self.get_env_vars(container_path)
+				unless File.exist? "#{container_path}/env"
+					logger.fatal "Environment could not be loaded ... Failed."
+					return false
+				end
+
+				env_vars = {}
+				File.open("#{container_path}/env").each_line do |line|
+					line.chomp!
+					next if line.empty? || line.start_with?('#')
+				    key, value = line.split "="
+				    env_vars[key] = value
+				end
+
+				env_vars
+			end
+		end
+	end
+end

data/lib/smartcloud/apps/rails.rb

@@ -0,0 +1,240 @@
+# The main Smartcloud Apps Rails driver
+module Smartcloud
+	module Apps
+		class Rails < Smartcloud::Base
+			def initialize
+			end
+
+			def start(appname, container_path, container_path_with_version)
+				return unless File.exist? "#{container_path_with_version}/bin/rails"
+
+				logger.formatter = proc do |severity, datetime, progname, message|
+					severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+						"ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+					}
+					"\t\t\t\t#{severity_text[severity]} #{message}\n"
+				end
+
+				logger.info "Ruby on Rails application detected."
+
+				# Setup rails env
+				env_path = "#{container_path}/env"
+				system("grep -q '^## Rails' #{env_path} || echo '## Rails' >> #{env_path}")
+				system("grep -q '^MALLOC_ARENA_MAX=' #{env_path} || echo '# MALLOC_ARENA_MAX=2' >> #{env_path}")
+				system("grep -q '^RAILS_ENV=' #{env_path} || echo 'RAILS_ENV=production' >> #{env_path}")
+				system("grep -q '^RACK_ENV=' #{env_path} || echo 'RACK_ENV=production' >> #{env_path}")
+				system("grep -q '^RAILS_LOG_TO_STDOUT=' #{env_path} || echo 'RAILS_LOG_TO_STDOUT=enabled' >> #{env_path}")
+				system("grep -q '^RAILS_SERVE_STATIC_FILES=' #{env_path} || echo 'RAILS_SERVE_STATIC_FILES=enabled' >> #{env_path}")
+				system("grep -q '^LANG=' #{env_path} || echo 'LANG=en_US.UTF-8' >> #{env_path}")
+				system("grep -q '^RAILS_MASTER_KEY=' #{env_path} || echo 'RAILS_MASTER_KEY=yourmasterkey' >> #{env_path}")
+				logger.warn "Please set your RAILS_MASTER_KEY env var for this rails app." if system("grep -q '^RAILS_MASTER_KEY=yourmasterkey' #{env_path}")
+
+				# Setup app folders needed for volumes. If this is not created then docker will create it while running the container,
+				# but the folder will have root user assigned instead of the current user.
+				FileUtils.mkdir_p("#{container_path}/app/vendor/bundle")
+				FileUtils.mkdir_p("#{container_path}/app/public/assets")
+				FileUtils.mkdir_p("#{container_path}/app/public/packs")
+				FileUtils.mkdir_p("#{container_path}/app/node_modules")
+				FileUtils.mkdir_p("#{container_path_with_version}/vendor/bundle")
+				FileUtils.mkdir_p("#{container_path_with_version}/public/assets")
+				FileUtils.mkdir_p("#{container_path_with_version}/public/packs")
+				FileUtils.mkdir_p("#{container_path_with_version}/node_modules")
+
+				# Creating & Starting container
+				container_id = `docker ps -a -q --filter='name=^#{appname}_1$' --filter='status=running'`.chomp
+				new_container = container_id.empty? ? "#{appname}_1" : "#{appname}_2"
+				old_container = container_id.empty? ? "#{appname}_2" : "#{appname}_1"
+
+				Smartcloud::Apps::App.stop("#{new_container}")
+				if system("docker create \
+					--name='#{new_container}' \
+					--env-file='#{container_path}/env' \
+					--user `id -u`:`id -g` \
+					--workdir /app \
+					--expose='3000' \
+					--volume='#{Smartcloud.config.user_home_path}/.smartcloud/config:#{Smartcloud.config.user_home_path}/.smartcloud/config' \
+					--volume='#{container_path_with_version}:/app' \
+					--volume='#{container_path}/app/vendor/bundle:/app/vendor/bundle' \
+					--volume='#{container_path}/app/public:/app/public/assets' \
+					--volume='#{container_path}/app/public:/app/public/packs' \
+					--volume='#{container_path}/app/node_modules:/app/node_modules' \
+					--restart='always' \
+					--init \
+					--network='nginx-network' \
+					smartcloud/buildpacks/rails", out: File::NULL)
+
+					# system("docker network connect solr-network #{new_container}")
+					system("docker network connect mysql-network #{new_container}")
+
+					if system("docker start --attach #{new_container}")
+						logger.debug "Starting Web Server ..."
+						if system("docker start #{new_container}", out: File::NULL)
+							sleep 7
+							logger.info "Web Server started successfully."
+							Smartcloud::Apps::App.stop(old_container)
+							Smartcloud::Apps::App.clean_up(container_path)
+							logger.info "Launched Application ... Success."
+							exit 10
+						end
+					else
+						Smartcloud::Apps::App.stop("#{new_container}")
+					end
+				end
+
+				logger.formatter = nil
+			end
+
+			def pack
+				set_logger_formatter_arrow
+
+				if File.exist? "tmp/smartcloud/packed"
+					begin
+						pid = File.read('tmp/smartcloud/packed').to_i
+						Process.kill('QUIT', pid)
+					rescue Errno::ESRCH # No such process
+					end
+					exec "bundle", "exec", "puma", "--config", "config/puma.rb"
+				else
+					if initial_setup? && bundle_install? && precompile_assets? && db_migrate? && test_web_server?
+						logger.formatter = nil
+
+						exit 0
+					else
+						logger.error "Could not continue ... Launch Failed."
+						logger.formatter = nil
+
+						exit 1
+					end
+				end
+			end
+
+			private
+
+			# Perform initial_setup
+			def initial_setup?
+				logger.info "Performing initial setup ..."
+
+				exit_status = nil
+
+				# Fix for mysql2 gem to support sha256_password, until it is fixed in main mysql2 gem.
+				# https://github.com/brianmario/mysql2/issues/1023
+				exit_status = system("mkdir ./lib/mariadb && ln -s /usr/lib/mariadb/plugin ./lib/mariadb/plugin")
+
+				if exit_status
+					return true
+				else
+					logger.error "Could not complete initial setup."
+					return false
+				end
+			end
+
+			# Perform bundle install
+			def bundle_install?
+				logger.info "Performing bundle install ..."
+
+				set_logger_formatter_tabs
+				exit_status = nil
+				Open3.popen2e("bundle", "install", "--deployment", "--clean") do |stdin, stdout_and_stderr, wait_thr|
+					stdout_and_stderr.each { |line| logger.info "#{line}" }
+					exit_status = wait_thr.value.success?
+				end
+				set_logger_formatter_arrow
+
+				if exit_status
+					return true
+				else
+					logger.error "Could not complete bundle install."
+					return false
+				end
+			end
+
+			# Perform pre-compiling of assets
+			def precompile_assets?
+				logger.info "Installing Javascript dependencies & pre-compiling assets ..."
+
+				set_logger_formatter_tabs
+				exit_status = nil
+				Open3.popen2e("bundle", "exec", "rails", "assets:precompile") do |stdin, stdout_and_stderr, wait_thr|
+					stdout_and_stderr.each { |line| logger.info "#{line}" }
+					exit_status = wait_thr.value.success?
+				end
+				set_logger_formatter_arrow
+
+				if exit_status
+					return true
+				else
+					logger.error "Could not install Javascript dependencies or pre-compile assets."
+					return false
+				end
+			end
+
+			# Perform db_migrate
+			def db_migrate?
+				return true # remove this line when you want to start using db_migrate?
+
+				logger.info "Performing database migrations ..."
+
+				set_logger_formatter_tabs
+				exit_status = nil
+				Open3.popen2e("bundle", "exec", "rails", "db:migrate") do |stdin, stdout_and_stderr, wait_thr|
+					stdout_and_stderr.each { |line| logger.info "#{line}" }
+					exit_status = wait_thr.value.success?
+				end
+				set_logger_formatter_arrow
+
+				if exit_status
+					return true
+				else
+					logger.error "Could not complete database migrations."
+					return false
+				end
+			end
+
+			# Perform testing of web server
+			def test_web_server?
+				logger.info "Setting up Web Server ..."
+
+				# tmp folders
+				FileUtils.mkdir_p("tmp/pids")
+				FileUtils.mkdir_p("tmp/smartcloud")
+				FileUtils.rm_f("tmp/smartcloud/packed")
+
+				# Spawn Process
+				pid = Process.spawn("bundle", "exec", "puma", "--config", "config/puma.rb", out: File::NULL)
+				Process.detach(pid)
+
+				# Sleep
+				sleep 5
+
+				# Check PID running
+				status = nil
+				begin
+					Process.kill(0, pid)
+					system("echo '#{pid}' > tmp/smartcloud/packed")
+					status = true
+				rescue Errno::ESRCH # No such process
+					logger.info "Web Server could not start"
+					status = false
+				end
+
+				# Return status
+				return status
+			end
+
+			def set_logger_formatter_arrow
+				logger.formatter = proc do |severity, datetime, progname, message|
+					severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+						"ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+					}
+					"\t\t\t\t#{severity_text[severity]} #{message}\n"
+				end
+			end
+
+			def set_logger_formatter_tabs
+				logger.formatter = proc do |severity, datetime, progname, message|
+					"\t\t\t\t       #{message}"
+				end
+			end
+		end
+	end
+end