smartmachine 0.8.0 → 0.9.0

data/lib/smart_machine/machine.rb

@@ -1,197 +1,131 @@
 require "net/ssh"
 
-# The main SmartMachine Machine driver
 module SmartMachine
-	class Machine < SmartMachine::Base
-		def initialize
-		end
-
-		def create(*args)
-			args.flatten!
-
-			raise "Please specify a machine name" if args.empty?
-
-			name = args.shift
-			pathname = File.expand_path "./#{name}"
-
-			self.setup_dotsmartmachine(pathname)
-
-			puts "New machine #{name} has been created."
-		end
-
-		def init_local(*args)
-			args.flatten!
-
-			pathname = File.expand_path "~/.smartmachine"
-
-			if args.delete("--force")
-				puts "Removing all the data to reinitialize."
-				FileUtils.rmtree(pathname)
-			end
-
-			if Dir.exist?(pathname)
-				puts "SmartMachine Local already initialized. If you want to delete all the data and reinitialize, please use the --force option."
-				return
-			end
-
-			self.setup_dotsmartmachine(pathname)
-
-			puts "SmartMachine Local Initialised."
-		end
-
-		def installer(*args)
-			args.flatten!
-
-			action = args.shift
-
-			if args.empty? || args.include?("docker")
-				docker = SmartMachine::Docker.new
-				docker.public_send(action)
-			end
-
-			if args.empty? || args.include?("engine")
-				engine = SmartMachine::Engine.new
-				engine.public_send(action)
-			end
-
-			if args.empty? || args.include?("buildpacker")
-				buildpacker = SmartMachine::Buildpacker.new
-				buildpacker.public_send(action)
-			end
-
-			if args.empty? || args.include?("prereceiver")
-				prereceiver = SmartMachine::Grids::Prereceiver.new
-				prereceiver.public_send(action)
-			end
-
-			if args.empty? || args.include?("scheduler")
-				scheduler = SmartMachine::Grids::Scheduler.new
-				scheduler.public_send(action)
-			end
-
-			if args.empty? || args.include?("elasticsearch")
-				elasticsearch = SmartMachine::Grids::Elasticsearch.new
-				elasticsearch.public_send(action)
-			end
-		end
-
-		def grids(*args)
-			args.flatten!
-
-			if args.delete("--local")
-				exec "smartmachine runner grids #{args.join(" ")}"
-			else
-				ssh = SmartMachine::SSH.new
-				ssh.run "smartmachine runner grids #{args.join(" ")}"
-			end
-		end
-
-		def apps(*args)
-			args.flatten!
-
-			if args.delete("--local")
-				exec "smartmachine runner apps #{args.join(" ")}"
-			else
-				ssh = SmartMachine::SSH.new
-				ssh.run "smartmachine runner apps #{args.join(" ")}"
-			end
-		end
-
-		def ps(*args)
-			args.flatten!
-
-			if args.delete("--local")
-				exec "docker ps #{args.join(' ')}"
-			else
-				ssh = SmartMachine::SSH.new
-				ssh.run "docker ps #{args.join(' ')}"
-			end
-		end
-
-		def logs(*args)
-			args.flatten!
-
-			if args.delete("--local")
-				exec "docker logs #{args.join(' ')}"
-			else
-				ssh = SmartMachine::SSH.new
-				ssh.run "docker logs #{args.join(' ')}"
-			end
-		end
-
-		def ssh
-			ssh = SmartMachine::SSH.new
-			ssh.login
-		end
-
-		def getting_started
-			# puts 'You may be prompted to make a menu selection when the Grub package is updated on Ubuntu. If prompted, select keep the local version currently installed.'
-
-			# apt-get update && apt-get upgrade
-
-			# hostnamectl set-hostname example_hostname
-
-			# /etc/hosts
-			# 127.0.0.1 localhost.localdomain localhost
-			# 203.0.113.10 hostname.example.com hostname
-			# 2600:3c01::a123:b456:c789:d012 hostname.example.com hostname
-			# Add DNS records for IPv4 and IPv6 for ip addresses and their fully qualified domain names FQDN
-
-			# dpkg-reconfigure tzdata
-			# date
-		end
-
-		def securing_your_server
-			# sudo apt install unattended-upgrades
-
-			# sudo nano /etc/apt/apt.conf.d/20auto-upgrades
-			# APT::Periodic::Update-Package-Lists "1";
-			# APT::Periodic::Download-Upgradeable-Packages "1";
-			# APT::Periodic::AutocleanInterval "7";
-			# APT::Periodic::Unattended-Upgrade "1";
-
-			# sudo apt install apticron
-			# /etc/apticron/apticron.conf
-			# EMAIL="root@example.com"
-
-			# adduser example_user
-			# adduser example_user sudo
-
-			# mkdir -p ~/.ssh && sudo chmod -R 700 ~/.ssh/
-			# scp ~/.ssh/id_rsa.pub example_user@203.0.113.10:~/.ssh/authorized_keys
-			# sudo chmod -R 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys
-
-			# sudo nano /etc/ssh/sshd_config
-			# PermitRootLogin no
-			# PasswordAuthentication no
-			# echo 'AddressFamily inet' | sudo tee -a /etc/ssh/sshd_config
-			# sudo systemctl restart sshd
-
-			# sudo apt-get update && sudo apt-get upgrade -y
-			# sudo apt-get install fail2ban
-			# sudo apt-get install sendmail
-			# sudo ufw allow ssh
-			# sudo ufw enable
-			# sudo cp /etc/fail2ban/fail2ban.conf /etc/fail2ban/fail2ban.local
-			# sudo cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
-			# Change destmail, sendername, sender
-			# Change action = %(action_mwl)s
-			# sudo fail2ban-client reload
-			# sudo fail2ban-client status
-		end
-
-		def in_local_machine_dir?
-			File.file?("./config/master.key")
-		end
-
-		def setup_dotsmartmachine(pathname)
-			FileUtils.mkdir pathname
-			FileUtils.cp_r "#{SmartMachine.config.root_path}/lib/smart_machine/templates/dotsmartmachine/.", pathname
-			FileUtils.chdir pathname do
-				credentials = SmartMachine::Credentials.new
-				credentials.create
-				system("git init && git add . && git commit -m 'initial commit'")
-			end	
-		end
-	end
-end
+  class Machine < SmartMachine::Base
+    def initialize
+    end
+
+    # Create a new smartmachine
+    #
+    # Example:
+    #   >> Machine.create("qw21334q")
+    #   => "New machine qw21334q has been created."
+    #
+    # Arguments:
+    #   name: (String)
+    #   dev: (Boolean)
+    def create(name:, dev:)
+      raise "Please specify a machine name" if name.blank?
+
+      pathname = File.expand_path "./#{name}"
+
+      if Dir.exist?(pathname)
+        puts "A machine with this name already exists. Please use a different name."
+        return
+      end
+
+      FileUtils.mkdir pathname
+      FileUtils.cp_r "#{SmartMachine.config.gem_dir}/lib/smart_machine/templates/dotsmartmachine/.", pathname
+      FileUtils.chdir pathname do
+        credentials = SmartMachine::Credentials.new
+        credentials.create
+
+        File.write("Gemfile", File.open("Gemfile",&:read).gsub("replace_ruby_version", "#{SmartMachine.ruby_version}"))
+        File.write(".ruby-version", SmartMachine.ruby_version)
+        if dev
+          File.write("Gemfile", File.open("Gemfile",&:read).gsub("\"~> replace_smartmachine_version\"", "path: \"../\""))
+        else
+          File.write("Gemfile", File.open("Gemfile",&:read).gsub("replace_smartmachine_version", "#{SmartMachine.version}"))
+        end
+        system("mv gitignore-template .gitignore")
+
+        # Here BUNDLE_GEMFILE is needed as it may be already set due to usage of bundle exec (which may not be correct in this case)
+        bundle_gemfile = "#{pathname}/Gemfile"
+        system("BUNDLE_GEMFILE='#{bundle_gemfile}' bundle install && BUNDLE_GEMFILE='#{bundle_gemfile}' bundle binstubs smartmachine")
+
+        system("git init && git add . && git commit -m 'initial commit by SmartMachine #{SmartMachine.version}'")
+      end
+
+      puts "New machine #{name} has been created."
+    end
+
+    def initial_setup
+      getting_started
+      securing_your_server
+    end
+
+    def run_on_machine(commands:)
+      commands = Array(commands).flatten
+      ssh = SmartMachine::SSH.new
+      status = ssh.run commands
+
+      status[:exit_code] == 0
+    end
+
+    private
+
+    def getting_started
+      # apt install locales-all
+
+      # puts 'You may be prompted to make a menu selection when the Grub package is updated on Ubuntu. If prompted, select keep the local version currently installed.'
+      # apt update && apt upgrade
+
+      # hostnamectl set-hostname SmartMachine.credentials.machine[:name]
+
+      # The value you assign as your system’s FQDN should have an “A” record in DNS pointing to your Linode’s IPv4 address. For IPv6, you should also set up a DNS “AAAA” record pointing to your Linode’s IPv6 address.
+      # Add DNS records for IPv4 and IPv6 for ip addresses and their fully qualified domain names FQDN
+      # /etc/hosts
+      # 203.0.113.10 SmartMachine.credentials.machine[:name].example.com SmartMachine.credentials.machine[:name]
+      # 2600:3c01::a123:b456:c789:d012 SmartMachine.credentials.machine[:name].example.com SmartMachine.credentials.machine[:name]
+
+      # dpkg-reconfigure tzdata
+      # date
+    end
+
+    def securing_your_server
+      # apt install unattended-upgrades
+      # dpkg-reconfigure --priority=low unattended-upgrades
+
+      # nano /etc/apt/apt.conf.d/20auto-upgrades
+      # APT::Periodic::Update-Package-Lists "1";
+      # APT::Periodic::Download-Upgradeable-Packages "1";
+      # APT::Periodic::AutocleanInterval "7";
+      # APT::Periodic::Unattended-Upgrade "1";
+
+      # apt install apticron
+      # /usr/lib/apticron/apticron.conf
+      # EMAIL="root@example.com"
+
+      # adduser example_user
+      # adduser example_user sudo
+
+      # mkdir -p ~/.ssh && sudo chmod -R 700 ~/.ssh/
+      # scp ~/.ssh/id_rsa.pub example_user@203.0.113.10:~/.ssh/authorized_keys
+      # sudo chmod -R 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys
+
+      # sudo nano /etc/ssh/sshd_config
+      # PermitRootLogin no
+      # PasswordAuthentication no
+      # echo 'AddressFamily inet' | sudo tee -a /etc/ssh/sshd_config
+      # sudo systemctl restart sshd
+
+      # sudo apt update && sudo apt upgrade -y
+
+      # sudo apt install ufw
+      # sudo ufw default allow outgoing
+      # sudo ufw default deny incoming
+      # sudo ufw allow SmartMachine.credentials.machine[:port]/tcp
+      # sudo ufw enable
+      # sudo ufw logging on
+
+      # sudo apt install fail2ban
+      # sudo apt install sendmail
+      # sudo cp /etc/fail2ban/fail2ban.conf /etc/fail2ban/fail2ban.local
+      # sudo cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
+      # Change destmail, sendername, sender
+      # Change action = %(action_mwl)s
+      # sudo fail2ban-client reload
+      # sudo fail2ban-client status
+    end
+  end
+end

data/lib/smart_machine/scp.rb

@@ -0,0 +1,15 @@
+module SmartMachine
+  class SCP < SmartMachine::Base
+    def initialize
+      @address  = SmartMachine.credentials.machine[:address]
+      @port     = SmartMachine.credentials.machine[:port]
+      @username = SmartMachine.credentials.machine[:username]
+      @password = SmartMachine.credentials.machine[:password]
+    end
+
+    def upload!(local_path:, remote_path:)
+      system("scp -q -P #{@port} #{local_path} #{@username}@#{@address}:#{remote_path}")
+      $?
+    end
+  end
+end

data/lib/smart_machine/ssh.rb

@@ -1,43 +1,72 @@
 require "net/ssh"
 
-# The main SmartMachine SSH driver
 module SmartMachine
-	class SSH < SmartMachine::Base
-		def initialize
-		end
-
-		def run(*commands)
-			commands.flatten!
-			Net::SSH.start(SmartMachine.credentials.machine[:address], SmartMachine.credentials.machine[:username], { port: SmartMachine.credentials.machine[:port], password: SmartMachine.credentials.machine[:password] }) do |ssh|
-				channel = ssh.open_channel do |channel, success|
-					channel.request_pty do |channel, success|
-						channel.exec commands.join(';') do |channel, success|
-							raise "Could not execute command" unless success
-
-							channel.on_data do |channel, data|
-								$stdout.print data
-
-								if data =~ /^\[sudo\] password for /
-									channel.send_data "#{SmartMachine.credentials.machine[:password]}\n"
-								end
-							end
-
-							channel.on_extended_data do |channel, type, data|
-								$stderr.print data
-							end
-
-							channel.on_close do |channel|
-								# puts "done!"
-							end
-						end
-					end
-				end
-				channel.wait
-			end
-		end
-
-		def login
-			exec "ssh #{SmartMachine.credentials.machine[:username]}@#{SmartMachine.credentials.machine[:address]}"
-		end
-	end
-end
+  class SSH < SmartMachine::Base
+    def initialize
+    end
+
+    def run(*commands)
+      commands.flatten!
+      status = {}
+
+      Net::SSH.start(SmartMachine.credentials.machine[:address], SmartMachine.credentials.machine[:username], { port: SmartMachine.credentials.machine[:port], password: SmartMachine.credentials.machine[:password] }) do |ssh|
+        commands.each do |command|
+
+          puts "\e[1m" + "$ " + command + "\e[0m"
+
+          channel = ssh.open_channel do |channel|
+
+            channel.on_open_failed do |channel, code, description|
+              raise "could not open channel (#{description}, ##{code})"
+            end
+
+            ssh.listen_to(STDIN) do |stdin|
+              input = stdin.readpartial(1024)
+              channel.send_data(input) unless input.empty?
+            end
+
+            channel.request_pty :modes => { Net::SSH::Connection::Term::ECHO => 0 } do |channel, success|
+              raise "could not obtain pty" unless success
+
+              channel.exec command do |channel, success|
+                raise "could not execute command: #{command.inspect}" unless success
+
+                if status
+                  channel.on_request("exit-status") do |ch2, data|
+                    status[:exit_code] = data.read_long
+                  end
+
+                  channel.on_request("exit-signal") do |ch2, data|
+                    status[:exit_signal] = data.read_long
+                  end
+                end
+
+                channel.on_data do |ch2, data|
+                  $stdout.print data
+
+                  if (data[/\[sudo\]|Password/i])
+                    channel.send_data "#{SmartMachine.credentials.machine[:password]}\n"
+                  end
+                end
+
+                channel.on_extended_data do |ch2, type, data|
+                  $stderr.print data
+                end
+              end
+            end
+          end
+
+          channel.wait
+
+          break if status[:exit_code] != 0
+        end
+      end
+
+      status
+    end
+
+    def login
+      exec "ssh -p #{SmartMachine.credentials.machine[:port]} #{SmartMachine.credentials.machine[:username]}@#{SmartMachine.credentials.machine[:address]}"
+    end
+  end
+end

data/lib/smart_machine/syncer.rb

@@ -0,0 +1,133 @@
+module SmartMachine
+  class Syncer < SmartMachine::Base
+    def initialize
+    end
+
+    def sync(initial: false)
+      puts "-----> Syncing SmartMachine"
+
+      push if initial
+
+      pull
+      push
+
+      puts "-----> Syncing SmartMachine Complete"
+    end
+
+    def rsync(*args)
+      exec "rsync #{args.join(' ')}"
+    end
+
+    private
+
+    def pull
+      puts "-----> Syncer pulling ... "
+      if system("#{rsync_command(pull_files_list)} #{SmartMachine.credentials.machine[:username]}@#{SmartMachine.credentials.machine[:address]}:~/machine/ .")
+        puts "done"
+      else
+        raise "Syncer error while pulling..."
+      end
+    end
+
+    def push
+      puts "-----> Syncer pushing ... "
+      if system("#{rsync_command(push_files_list)} ./ #{SmartMachine.credentials.machine[:username]}@#{SmartMachine.credentials.machine[:address]}:~/machine")
+        puts "done"
+      else
+        raise "Syncer error while pushing..."
+      end
+    end
+
+    def rsync_command(files_list)
+      command = [
+        "rsync -azumv",
+        "-e 'ssh -p #{SmartMachine.credentials.machine[:port]}'",
+        "--rsync-path='smartengine syncer rsync'",
+        "--delete",
+        "--include={#{files_list}}",
+        "--exclude=*"
+      ]
+
+      command.join(" ")
+    end
+
+    def pull_files_list
+      files = [
+        'apps/***',
+
+        'bin/***',
+
+        'grids',
+
+        'grids/elasticsearch',
+        'grids/elasticsearch/***',
+
+        'grids/minio',
+        'grids/minio/***',
+
+        'grids/mysql',
+        'grids/mysql/***',
+
+        'grids/nginx',
+        'grids/nginx/certificates/***',
+
+        'grids/prereceiver',
+        'grids/prereceiver/***',
+
+        'grids/redis',
+        'grids/redis/***',
+
+        'grids/scheduler',
+        'grids/scheduler/crontabs/***',
+
+        'grids/solr',
+        'grids/solr/solr/***',
+      ]
+      files.join(',')
+    end
+
+    def push_files_list
+      files = [
+        'apps',
+        'apps/containers',
+        'apps/containers/.keep',
+        'apps/repositories',
+        'apps/repositories/.keep',
+
+        'bin',
+        'bin/smartmachine',
+
+        'config',
+        'config/mysql',
+        'config/mysql/schedule.rb',
+        'config/credentials.yml.enc',
+        'config/environment.rb',
+        'config/elasticsearch.yml',
+        'config/minio.yml',
+        'config/mysql.yml',
+        'config/prereceiver.yml',
+        'config/redis.yml',
+
+        'grids',
+
+        'grids/nginx',
+        'grids/nginx/certificates',
+        'grids/nginx/certificates/.keep',
+        'grids/nginx/htpasswd/***',
+        'grids/nginx/fastcgi.conf',
+        'grids/nginx/nginx.tmpl',
+
+        'grids/scheduler',
+        'grids/scheduler/crontabs',
+        'grids/scheduler/crontabs/.keep',
+
+        'grids/solr',
+        'grids/solr/solr',
+        'grids/solr/solr/.keep',
+
+        'tmp/***',
+      ]
+      files.join(',')
+    end
+  end
+end

data/lib/smart_machine/templates/dotsmartmachine/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+ruby 'replace_ruby_version'
+
+# SmartMachine
+gem 'smartmachine', "~> replace_smartmachine_version"

data/lib/smart_machine/templates/dotsmartmachine/config/elasticsearch.yml

@@ -0,0 +1,5 @@
+elasticsearchone:
+  port: 9200
+
+# elasticsearchtwo:
+#   port: 9201

data/lib/smart_machine/templates/dotsmartmachine/config/environment.rb

@@ -1,18 +1,9 @@
 # => NOTE: Ensure that the specified top-level domains are pointing to this server ip address using DNS records.
 # => Be sure to restart your server when you modify this file.
 
-# Top-level naked domain to be used for subdomains of apps.
-SmartMachine.config.apps_domain = "yourdomain.com"
-
-# domain to be used for git prereceiver
-SmartMachine.config.git_domain = "git.yourdomain.com"
-
 # Sysadmin email id.
 SmartMachine.config.sysadmin_email = "admin@yourdomain.com"
 
-# letsencrypt test boolean to be used
-SmartMachine.config.letsencrypt_test = false
-
 # logger level
 # DEBUG, INFO, WARN, ERROR, FATAL, UNKNOWN
 SmartMachine.config.logger_level = "INFO"

data/lib/smart_machine/templates/dotsmartmachine/config/minio.yml

@@ -0,0 +1,13 @@
+minioone:
+  host: "yourminioone.yourdomain.com"
+  access_key: <%= SmartMachine.credentials.dig(:minioone, :access_key) %>
+  secret_key: <%= SmartMachine.credentials.dig(:minioone, :secret_key) %>
+  browser: on
+  worm: off
+
+# miniotwo:
+#   host: "yourminiotwo.yourdomain.com"
+#   access_key: <%= SmartMachine.credentials.dig(:miniotwo, :access_key) %>
+#   secret_key: <%= SmartMachine.credentials.dig(:miniotwo, :secret_key) %>
+#   browser: on
+#   worm: off

data/lib/smart_machine/templates/dotsmartmachine/config/mysql/schedule.rb

@@ -3,17 +3,17 @@
 
 # Flush logs used to create consistent binlogs to be used for incremental backups
 every :day, at: '12:00 pm' do
-	command "smartmachine runner grids mysql flushlogs"
+  command "smartmachine runner grids mysql flushlogs"
 end
 
 # Create daily backup.
 # This also flushes the logs before backup
 every :day, at: '12:00 am' do
-	command "smartmachine runner grids mysql backup --daily"
+  command "smartmachine runner grids mysql backup --daily"
 end
 
 # Promote currently latest daily backup to weekly backup.
 # This is only possible when a daily backup creation has already been completed.
 every :monday, at: '3:00 am' do
-	command "smartmachine runner grids mysql backup --promote-to-weekly"
+  command "smartmachine runner grids mysql backup --promote-to-weekly"
 end