smartmachine 0.8.0 → 1.0.0

data/lib/smart_machine/apps/manager.rb

@@ -0,0 +1,182 @@
+module SmartMachine
+  module Apps
+    class Manager < SmartMachine::Base
+      attr_accessor :appname, :appversion
+
+      def initialize(appname:, appversion: nil)
+        @appname = appname
+        @appversion = appversion
+
+        @home_dir = File.expand_path('~')
+        @container_path = "#{@home_dir}/machine/apps/containers/#{@appname}"
+
+        # Checking if given appname exists
+        repository_path = "#{@home_dir}/machine/apps/repositories/#{@appname}.git"
+        raise "App with name '#{@appname}' does not exist. Please provide a valid appname." unless Dir.exist?(repository_path)
+      end
+
+      def deploy
+        logger.formatter = proc do |severity, datetime, progname, message|
+          severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+                           "ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+                          }
+          "\t\t\t\t#{severity_text[severity]} #{message}\n"
+        end
+
+        Dir.chdir("#{@container_path}/releases") do
+          # Setting app version to latest if not set.
+          unless @appversion
+            versions = Dir.glob('*').select { |f| File.directory? f }.sort
+            @appversion = versions.last
+          end
+
+          buildpacker = SmartMachine::Buildpackers::Rails.new(appname: @appname, appversion: @appversion)
+          if buildpacker.package
+            if release
+              sleep 7
+              clean
+              logger.info "Launched Application ... Success."
+              exit 10
+            else
+              raise "Error: Could not release the application."
+            end
+          else
+            raise "Error: Could not package the application."
+          end
+        end
+
+        logger.formatter = nil
+      end
+
+      def ps_scaler(formation:)
+        Dir.chdir("#{@container_path}/releases") do
+          # Setting app version to latest if not set.
+          unless @appversion
+            versions = Dir.glob('*').select { |f| File.directory? f }.sort
+            @appversion = versions.last
+          end
+
+          formation.each do |proc_name, final_count|
+            config = processes.dig(proc_name.to_sym)
+            raise "no config found for the #{proc_name} process." unless config.present?
+
+            current_count = `docker ps -aq --filter name="#{@appname}-#{@appversion}-#{proc_name}-" | wc -l`
+
+            final_count = final_count.to_i
+            current_count = current_count.to_i
+
+            if final_count > current_count
+              ((current_count + 1)..final_count).each do |index|
+                containerize_process!(name: "#{@appname}-#{@appversion}-#{proc_name}-#{index}", config: config)
+              end
+            elsif final_count < current_count
+              ((final_count + 1)..current_count).each do |index|
+                container = SmartMachine::Apps::Container.new(name: "#{@appname}-#{@appversion}-#{proc_name}-#{index}", appname: @appname, appversion: @appversion)
+                container.stop!
+              end
+            else
+              # No Operation. Don't do anything.
+            end
+          end
+        end
+      end
+
+      def env_vars
+        @env_vars ||= load_env_vars
+      end
+
+      private
+
+      def release
+        logger.info "Releasing Application ..."
+
+        if processes.present?
+          processes.each do |name, config|
+            containerize_process!(name: "#{@appname}-#{@appversion}-#{name}-1", config: config)
+          end
+        else
+          logger.fatal "No smartmachine.yml file found. Proceeding with default settings."
+          return false
+
+          # Use the below code when you want to provide a default when smartmachine.yml is not present.
+          # containerize_process!(name: "#{@appname}-#{@appversion}-web-1", config: { command: "bundle exec puma --config config/puma.rb", networks: "nginx-network" })
+        end
+      end
+
+      def containerize_process!(name:, config:)
+        container = SmartMachine::Apps::Container.new(name: name, appname: @appname, appversion: @appversion)
+        if container.create!(using_command: config.dig(:command))
+          networks = config.dig(:networks)&.split(" ")
+          networks.each do |network|
+            container.connect_to_network!(network_name: network)
+          end
+
+          unless container.start!
+            container.stop!
+          end
+        else
+          container.stop!
+        end
+      end
+
+      def clean
+        return unless env_vars.present?
+
+        logger.info "Cleaning up ..."
+
+        # Clean up very old versions
+        Dir.chdir("#{@container_path}/releases") do
+          versions = Dir.glob('*').select { |f| File.directory? f }.sort
+          destroy_count = versions.count - env_vars['KEEP_RELEASES'].to_i
+          if destroy_count > 0
+            logger.debug "Deleting older application releases ..."
+            destroy_count.times do
+              version = versions.shift
+              FileUtils.rm_r(File.join(Dir.pwd, version))
+
+              # Remove corresponding docker containers & images, if they exist.
+              system("docker ps -aq --filter ancestor=smartmachine/apps/#{@appname}:#{version} | xargs --no-run-if-empty docker stop | xargs --no-run-if-empty docker rm", out: File::NULL)
+              system("docker image ls -aq --filter reference=smartmachine/apps/#{@appname}:#{version} | xargs --no-run-if-empty docker image rm", out: File::NULL)
+            end
+          end
+
+          versions = Dir.glob('*').select { |f| File.directory? f }.sort
+          versions_count = versions.count
+          if versions_count > 0
+            versions_count.times do
+              version = versions.shift
+              unless version == @appversion
+                system("docker ps -aq --filter ancestor=smartmachine/apps/#{@appname}:#{version} | xargs --no-run-if-empty docker stop", out: File::NULL)
+              end
+            end
+          end
+        end
+      end
+
+      def load_env_vars
+        unless File.exist? "#{@container_path}/env"
+          logger.fatal "Environment could not be loaded ... Failed."
+          return false
+        end
+
+        env_vars = {}
+        File.open("#{@container_path}/env").each_line do |line|
+          line.chomp!
+          next if line.empty? || line.start_with?('#')
+          key, value = line.split "="
+          env_vars[key] = value
+        end
+
+        env_vars
+      end
+
+      def processes
+        @processes ||= deserialize(IO.binread("#{@appversion}/smartmachine.yml")).deep_symbolize_keys
+      end
+
+      def deserialize(config)
+        YAML.load(ERB.new(config).result).presence || {}
+      end
+    end
+  end
+end

data/lib/smart_machine/base.rb

@@ -2,10 +2,28 @@ require 'smart_machine/logger'
 require "active_support/inflector"
 
 module SmartMachine
-	class Base
-		include SmartMachine::Logger
+  class Base
+    include SmartMachine::Logger
 
-		def initialize
-		end		
-	end
-end
+    def initialize
+    end
+
+    def platform_on_machine?(os:, distro_name: nil)
+      case os
+      when "linux"
+        command = "(uname | grep -q 'Linux')"
+        command += " && (cat /etc/os-release | grep -q 'NAME=\"Debian GNU/Linux\"')" if distro_name == "debian"
+      when "mac"
+        command = "(uname | grep -q 'Darwin')"
+      end
+
+      machine = SmartMachine::Machine.new
+      command ? machine.run_on_machine(commands: command) : false
+    end
+
+    def machine_has_engine_installed?
+      machine = SmartMachine::Machine.new
+      machine.run_on_machine(commands: ["which smartengine | grep -q '/smartengine'"])
+    end
+  end
+end

data/lib/smart_machine/buildpackers/buildpacker.rb

@@ -0,0 +1,95 @@
+module SmartMachine
+  module Buildpackers
+    class Buildpacker < SmartMachine::Base
+      def initialize(packname:)
+        @packname = packname
+      end
+
+      def installer
+        if @packname == "rails"
+          unless system("docker image inspect #{rails_image_name}", [:out, :err] => File::NULL)
+            print "-----> Creating image #{rails_image_name} ... "
+            command = [
+              "docker image build -t #{rails_image_name}",
+              "--build-arg SMARTMACHINE_VERSION=#{SmartMachine.version}",
+              "--build-arg USER_UID=`id -u`",
+              "--build-arg USER_NAME=`id -un`",
+              "#{SmartMachine.config.gem_dir}/lib/smart_machine/buildpackers/rails"
+            ]
+            if system(command.join(" "), out: File::NULL)
+              puts "done"
+            end
+          end
+        else
+          raise "Error: Pack with name #{name} not supported."
+        end
+      end
+
+      def uninstaller
+        if @packname == "rails"
+          if system("docker image inspect #{rails_image_name}", [:out, :err] => File::NULL)
+            print "-----> Removing image #{rails_image_name} ... "
+            if system("docker image rm #{rails_image_name}", out: File::NULL)
+              puts "done"
+            end
+          end
+        else
+          raise "Error: Pack with name #{name} not supported."
+        end
+      end
+
+      def packer
+        if @packname == "rails" && File.exist?("bin/rails")
+          rails = SmartMachine::Buildpackers::Rails.new(appname: nil, appversion: nil)
+          rails.packer
+        else
+          raise "Error: Pack with name #{@packname} not supported."
+        end
+      end
+
+      private
+
+      def rails_image_name
+        "smartmachine/buildpackers/rails:#{SmartMachine.version}"
+      end
+
+      # These swapfile methods can be used (after required modification), when you need to make swapfile for more memory.
+      # def self.create_swapfile
+      # 	# Creating swapfile for bundler to work properly
+      # 	unless system("sudo swapon -s | grep -ci '/swapfile'", out: File::NULL)
+      # 		print "-----> Creating swap swapfile ... "
+      # 		system("sudo install -o root -g root -m 0600 /dev/null /swapfile", out: File::NULL)
+      # 		system("sudo dd if=/dev/zero of=/swapfile bs=1k count=2048k", [:out, :err] => File::NULL)
+      # 		system("sudo mkswap /swapfile", out: File::NULL)
+      # 		system("sudo sh -c 'echo \"/swapfile       none    swap    sw      0       0\" >> /etc/fstab'", out: File::NULL)
+      # 		system("echo 10 | sudo tee /proc/sys/vm/swappiness", out: File::NULL)
+      # 		system("sudo sed -i '/^vm.swappiness = /d' /etc/sysctl.conf", out: File::NULL)
+      # 		system("echo vm.swappiness = 10 | sudo tee -a /etc/sysctl.conf", out: File::NULL)
+      # 		puts "done"
+      #
+      # 		print "-----> Starting swap swapfile ... "
+      # 		if system("sudo swapon /swapfile", out: File::NULL)
+      # 			puts "done"
+      # 		end
+      # 	end
+      # end
+      #
+      # def self.destroy_swapfile
+      # 	if system("sudo swapon -s | grep -ci '/swapfile'", out: File::NULL)
+      # 		print "-----> Stopping swap swapfile ... "
+      # 		if system("sudo swapoff /swapfile", out: File::NULL)
+      # 			system("sudo sed -i '/^vm.swappiness = /d' /etc/sysctl.conf", out: File::NULL)
+      # 			system("echo 60 | sudo tee /proc/sys/vm/swappiness", out: File::NULL)
+      # 			puts "done"
+      #
+      # 			print "-----> Removing swap swapfile ... "
+      # 			system("sudo sed -i '/^\\/swapfile/d' /etc/fstab", out: File::NULL)
+      # 			if system("sudo rm /swapfile", out: File::NULL)
+      # 				puts "done"
+      # 			end
+      # 		end
+      # 	end
+      # end
+    end
+  end
+end

data/lib/smart_machine/{engine/buildpacks → buildpackers}/rails/Dockerfile

@@ -1,7 +1,7 @@
 ARG SMARTMACHINE_VERSION
 
-FROM smartmachine:$SMARTMACHINE_VERSION
-LABEL maintainer="Timeboard <hello@timeboard.me>"
+FROM smartmachine/smartengine:$SMARTMACHINE_VERSION
+LABEL maintainer="plainsource <plainsource@humanmind.me>"
 
 # Ruby on Rails Essentials
 RUN apk add --update build-base && \
@@ -18,4 +18,4 @@ RUN apk add --update build-base && \
 # Gems
 RUN gem install bundler -v 2.1.4
 
-CMD ["buildpacker", "pack"]
+CMD ["smartmachine", "buildpacker", "packer", "rails"]

data/lib/smart_machine/buildpackers/rails.rb

@@ -0,0 +1,221 @@
+require 'open3'
+
+module SmartMachine
+  module Buildpackers
+    class Rails < SmartMachine::Base
+      def initialize(appname:, appversion:)
+        @home_dir = File.expand_path('~')
+        @appname = appname
+        @appversion = appversion
+        @container_path = "#{@home_dir}/machine/apps/containers/#{@appname}"
+      end
+
+      def package
+        return unless File.exist? "#{@container_path}/releases/#{@appversion}/bin/rails"
+
+        logger.formatter = proc do |severity, datetime, progname, message|
+          severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+                           "ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+                          }
+          "\t\t\t\t#{severity_text[severity]} #{message}\n"
+        end
+
+        logger.info "Ruby on Rails application detected."
+        logger.info "Packaging Application ..."
+
+        # Setup rails env
+        env_path = "#{@container_path}/env"
+        system("grep -q '^## Rails' #{env_path} || echo '## Rails' >> #{env_path}")
+        system("grep -q '^MALLOC_ARENA_MAX=' #{env_path} || echo '# MALLOC_ARENA_MAX=2' >> #{env_path}")
+        system("grep -q '^RAILS_ENV=' #{env_path} || echo 'RAILS_ENV=production' >> #{env_path}")
+        system("grep -q '^RACK_ENV=' #{env_path} || echo 'RACK_ENV=production' >> #{env_path}")
+        system("grep -q '^RAILS_LOG_TO_STDOUT=' #{env_path} || echo 'RAILS_LOG_TO_STDOUT=enabled' >> #{env_path}")
+        system("grep -q '^RAILS_SERVE_STATIC_FILES=' #{env_path} || echo 'RAILS_SERVE_STATIC_FILES=enabled' >> #{env_path}")
+        system("grep -q '^LANG=' #{env_path} || echo 'LANG=en_US.UTF-8' >> #{env_path}")
+        system("grep -q '^RAILS_MASTER_KEY=' #{env_path} || echo 'RAILS_MASTER_KEY=yourmasterkey' >> #{env_path}")
+        logger.warn "Please set your RAILS_MASTER_KEY env var for this rails app." if system("grep -q '^RAILS_MASTER_KEY=yourmasterkey' #{env_path}")
+
+        # Setup app folders needed for volumes. If this is not created then docker will create it while running the container,
+        # but the folder will have root user assigned instead of the current user.
+        FileUtils.mkdir_p("#{@container_path}/app/vendor/bundle")
+        FileUtils.mkdir_p("#{@container_path}/app/public/assets")
+        FileUtils.mkdir_p("#{@container_path}/app/public/packs")
+        FileUtils.mkdir_p("#{@container_path}/app/node_modules")
+        FileUtils.mkdir_p("#{@container_path}/app/storage")
+        FileUtils.mkdir_p("#{@container_path}/releases/#{@appversion}/vendor/bundle")
+        FileUtils.mkdir_p("#{@container_path}/releases/#{@appversion}/public/assets")
+        FileUtils.mkdir_p("#{@container_path}/releases/#{@appversion}/public/packs")
+        FileUtils.mkdir_p("#{@container_path}/releases/#{@appversion}/node_modules")
+        FileUtils.mkdir_p("#{@container_path}/releases/#{@appversion}/storage")
+
+        # Creating a valid docker app image.
+        container = SmartMachine::Apps::Container.new(name: "#{@appname}-#{@appversion}-packed", appname: @appname, appversion: @appversion)
+        if container.commit_app_image!
+          logger.formatter = nil
+          return true
+        end
+
+        logger.formatter = nil
+        return false
+      end
+
+      def packer
+        set_logger_formatter_arrow
+
+        if File.exist? "tmp/smartmachine/packed"
+          begin
+            pid = File.read('tmp/smartmachine/packed').to_i
+            Process.kill('QUIT', pid)
+          rescue Errno::ESRCH # No such process
+          end
+          exec "bundle", "exec", "puma", "--config", "config/puma.rb"
+        else
+          if initial_setup? && bundle_install? && precompile_assets? && db_migrate? && test_web_server?
+            logger.formatter = nil
+
+            exit 0
+          else
+            logger.error "Could not continue ... Launch Failed."
+            logger.formatter = nil
+
+            exit 1
+          end
+        end
+      end
+
+      private
+
+      # Perform initial_setup
+      def initial_setup?
+        logger.info "Performing initial setup ..."
+
+        exit_status = nil
+
+        # Fix for mysql2 gem to support sha256_password, until it is fixed in main mysql2 gem.
+        # https://github.com/brianmario/mysql2/issues/1023
+        exit_status = system("mkdir -p ./lib/mariadb && ln -s /usr/lib/mariadb/plugin ./lib/mariadb/plugin")
+
+        if exit_status
+          return true
+        else
+          logger.error "Could not complete initial setup."
+          return false
+        end
+      end
+
+      # Perform bundle install
+      def bundle_install?
+        logger.info "Performing bundle install ..."
+
+        set_logger_formatter_tabs
+
+        unless system("bundle config set deployment 'true' && bundle config set clean 'true'")
+          logger.error "Could not complete bundle config setting."
+          return false
+        end
+
+        exit_status = nil
+        Open3.popen2e("bundle", "install") do |stdin, stdout_and_stderr, wait_thr|
+          stdout_and_stderr.each { |line| logger.info "#{line}" }
+          exit_status = wait_thr.value.success?
+        end
+        set_logger_formatter_arrow
+
+        if exit_status
+          return true
+        else
+          logger.error "Could not complete bundle install."
+          return false
+        end
+      end
+
+      # Perform pre-compiling of assets
+      def precompile_assets?
+        logger.info "Installing Javascript dependencies & pre-compiling assets ..."
+
+        set_logger_formatter_tabs
+        exit_status = nil
+        Open3.popen2e("bundle", "exec", "rails", "assets:precompile") do |stdin, stdout_and_stderr, wait_thr|
+          stdout_and_stderr.each { |line| logger.info "#{line}" }
+          exit_status = wait_thr.value.success?
+        end
+        set_logger_formatter_arrow
+
+        if exit_status
+          return true
+        else
+          logger.error "Could not install Javascript dependencies or pre-compile assets."
+          return false
+        end
+      end
+
+      # Perform db_migrate
+      def db_migrate?
+        return true # remove this line when you want to start using db_migrate?
+
+        logger.info "Performing database migrations ..."
+
+        set_logger_formatter_tabs
+        exit_status = nil
+        Open3.popen2e("bundle", "exec", "rails", "db:migrate") do |stdin, stdout_and_stderr, wait_thr|
+          stdout_and_stderr.each { |line| logger.info "#{line}" }
+          exit_status = wait_thr.value.success?
+        end
+        set_logger_formatter_arrow
+
+        if exit_status
+          return true
+        else
+          logger.error "Could not complete database migrations."
+          return false
+        end
+      end
+
+      # Perform testing of web server
+      def test_web_server?
+        logger.info "Setting up Web Server ..."
+
+        # tmp folders
+        FileUtils.mkdir_p("tmp/pids")
+        FileUtils.mkdir_p("tmp/smartmachine")
+        FileUtils.rm_f("tmp/smartmachine/packed")
+
+        # Spawn Process
+        pid = Process.spawn("bundle", "exec", "puma", "--config", "config/puma.rb", out: File::NULL)
+        Process.detach(pid)
+
+        # Sleep
+        sleep 5
+
+        # Check PID running
+        status = nil
+        begin
+          Process.kill(0, pid)
+          system("echo '#{pid}' > tmp/smartmachine/packed")
+          status = true
+        rescue Errno::ESRCH # No such process
+          logger.info "Web Server could not start"
+          status = false
+        end
+
+        # Return status
+        return status
+      end
+
+      def set_logger_formatter_arrow
+        logger.formatter = proc do |severity, datetime, progname, message|
+          severity_text = { "DEBUG" => "\u{1f527} #{severity}:", "INFO" => " \u{276f}", "WARN" => "\u{2757} #{severity}:",
+                           "ERROR" => "\u{274c} #{severity}:", "FATAL" => "\u{2b55} #{severity}:", "UNKNOWN" => "\u{2753} #{severity}:"
+                          }
+          "\t\t\t\t#{severity_text[severity]} #{message}\n"
+        end
+      end
+
+      def set_logger_formatter_tabs
+        logger.formatter = proc do |severity, datetime, progname, message|
+          "\t\t\t\t       #{message}"
+        end
+      end
+    end
+  end
+end

data/lib/smart_machine/commands/app.rb

@@ -0,0 +1,112 @@
+module SmartMachine
+  module Commands
+    class App < Thor
+      include Utilities
+
+      desc "create [APPNAME] [APPDOMAIN] [USERNAME]", "Create an App"
+      def create(appname, appdomain, username)
+        inside_machine_dir do
+          with_docker_running do
+            machine = SmartMachine::Machine.new
+            machine.run_on_machine commands: "smartengine app creater #{appname} #{appdomain} #{username}"
+          end
+        end
+      end
+
+      desc "destroy [APPNAME]", "Destroy an App"
+      def destroy(appname)
+        inside_machine_dir do
+          with_docker_running do
+            machine = SmartMachine::Machine.new
+            machine.run_on_machine commands: "smartengine app destroyer #{appname}"
+          end
+        end
+      end
+
+      desc "ps:scale [APPNAME]", "Scale the app processes"
+      map "ps:scale" => "ps_scale"
+      option :formation, type: :hash, required: true
+      option :version, type: :numeric, default: nil
+      def ps_scale(appname)
+        inside_machine_dir do
+          with_docker_running do
+            machine = SmartMachine::Machine.new
+            command = [
+              "smartengine app ps:scaler #{appname}"
+            ]
+            formation = options[:formation].map { |proctype, count| "#{proctype}:#{count}" }.join(' ')
+            command.push("--formation=#{formation}") if options[:formation]
+            command.push("--version=#{options[:version]}") if options[:version]
+            machine.run_on_machine commands: command.join(" ")
+          end
+        end
+      end
+
+      desc "up [APPNAME]", "Take UP the app"
+      option :version, type: :numeric, default: 0
+      def up(appname)
+        inside_machine_dir do
+          with_docker_running do
+            machine = SmartMachine::Machine.new
+            machine.run_on_machine commands: "smartengine app uper #{appname} --version=#{options[:version]}"
+          end
+        end
+      end
+
+      desc "down [APPNAME]", "Take DOWN the app"
+      def down(appname)
+        inside_machine_dir do
+          with_docker_running do
+            machine = SmartMachine::Machine.new
+            machine.run_on_machine commands: "smartengine app downer #{appname}"
+          end
+        end
+      end
+
+      desc "creater", "App creator", hide: true
+      def creater(appname, appdomain, username)
+        inside_engine_machine_dir do
+          app = SmartMachine::Apps::App.new(appname: appname, username: username)
+          prereceiver_name, prereceiver_config = SmartMachine.config.grids.prereceiver.first
+          app.creater(appdomain: appdomain, prereceiver_name: prereceiver_name)
+        end
+      end
+
+      desc "destroyer", "App destroyer", hide: true
+      def destroyer(appname)
+        inside_engine_machine_dir do
+          app = SmartMachine::Apps::App.new(appname: appname)
+          app.destroyer
+        end
+      end
+
+      desc "ps:scaler [APPNAME]", "Scale the app processes", hide: true
+      map "ps:scaler" => "ps_scaler"
+      option :formation, type: :hash, required: true
+      option :version, type: :numeric, default: nil
+      def ps_scaler(appname)
+        inside_engine_machine_dir do
+          manager = SmartMachine::Apps::Manager.new(appname: appname, appversion: options[:version])
+          manager.ps_scaler(formation: options[:formation])
+        end
+      end
+
+      desc "uper [APPNAME]", "App uper", hide: true
+      option :version, type: :numeric, default: 0
+      def uper(appname)
+        inside_engine_machine_dir do
+          app = SmartMachine::Apps::App.new(appname: appname)
+          app.uper(version: options[:version])
+        end
+      end
+
+      desc "downer [APPNAME]", "App downer", hide: true
+      def downer(appname)
+        inside_engine_machine_dir do
+          app = SmartMachine::Apps::App.new(appname: appname)
+          app.downer
+        end
+      end
+    end
+  end
+end