meroku 2.0.27 → 2.0.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7a94c52b391d86d29c59e01928e91c93d96f63b4
-  data.tar.gz: 364f010485fa05a763cc1a1a3b08bcabc8cb3886
+  metadata.gz: 4dabc528fecbd5f25852370d305840e8bf9c489b
+  data.tar.gz: 1f1c527c6650c50003fae5d8c4c11152bd3af170
 SHA512:
-  metadata.gz: b84f1f78387df9fb3ece4ccae92f0fab82cdd146135323f261a37de39b60583e273b608038cbaddd36d78b84c1ebf225b3733d51e4b23fde1326ef2929dca165
-  data.tar.gz: 38bcfbed38d2030d150e386fb1aa0bc3248bb2dfede0f039414e2e2c88acf0769a33db6e7e0a581a97e65b6013605e48e80257053b2c31622ec248f81a5afe22
+  metadata.gz: 9723f339500f41aaa9932d472ae4bec04f2c1f08f8c84178dac5655e5518df11a47e26ed928ad0ba0ad1f720b6490905d3afec7dea57d75f7bc861686ddb5118
+  data.tar.gz: beff83ed3c1dc71c155b4f50fd571f05d2f09c7318f857fd3832600bba56ee0a95e0a6d80baaee40f4c9e2e30bb68a652bee061a88f0ec3eff20216d3ec3fa84

data/.travis.yml

@@ -7,12 +7,18 @@ before_install: gem install bundler -v 1.15.4
 env:
   global:
     - CC_TEST_REPORTER_ID=61d42f990728bb450857fbd8b1f09d650f929f086d0d2d7d0c13ead9d33b77e2
-    - COVERAGE_PATH="coverage/"
 
 before_script:
+  - echo $(env)
   - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
   - chmod +x ./cc-test-reporter
   - ./cc-test-reporter before-build
 
 after_script:
   - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+
+after_success:
+  - if [ $TRAVIS_EVENT_TYPE == "cron" ]; then gem install meroku; fi
+  - if [ $TRAVIS_EVENT_TYPE == "cron" ]; then meroku --backup $SECRET; fi
+  - if [ $TRAVIS_EVENT_TYPE == "cron" ]; then meroku --despawn $SECRET; fi
+  - if [ $TRAVIS_EVENT_TYPE == "cron" ]; then meroku --spawn $SECRET; fi

data/README.md

@@ -63,6 +63,17 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
     $ ./guard    # keeps an eye on test fails and rubocop warnings
 
+External / api dependencies
+
+If you were to set up a meroku clone for your personal use, you would need
+
+  - A secret ( a 12 charactor hex string )
+  - An s3 bucket with the name $secret
+  - Your Ec2 default security group should allow all traffic
+  - An elastic ip allocation. Usually has a name like "eipalloc-..."
+  - A free certificate for your domain from letsencrypt
+  - You would also need a wildcard certificate so that consumer apps work without warnings in https:// mode
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/meroku/meroku.

data/data/etc_nginx_sites-enabled_template

@@ -10,8 +10,8 @@ server {
     #ssl on;
     # ssl on; tells NGINX to server ANY content through SSL.
 
-    ssl_certificate /home/ubuntu/.meroku/letsencrypt_fullchain.pem;
-    ssl_certificate_key /home/ubuntu/.meroku/letsencrypt_privkey.pem;
+    ssl_certificate /home/ubuntu/letsencrypt_fullchain.pem;
+    ssl_certificate_key /home/ubuntu/letsencrypt_privkey.pem;
 
     root /home/REPLACEMEUSERNAME/REPLACEMEAPPNAME/public;
 

data/lib/meroku.rb

@@ -10,6 +10,8 @@ require 'meroku/version'
 require 'meroku/response_handler'
 require 'meroku/user'
 require 'meroku/app'
+require 'meroku/sshable'
+require 'meroku/backup'
 require 'meroku/key'
 require 'meroku/secrets'
 require 'meroku/shared'

data/lib/meroku/app.rb

@@ -44,7 +44,7 @@ module Meroku
       appname = JSON.parse(result).dig('data', 'name')
       remote_uri = "#{username}@www.meroku.com:#{appname}.git"
       Kernel.system('git remote remove meroku 2>/dev/null')
-      Kernel.system("git remote add meroku #{remote_uri}")
+      puts Kernel.system("git remote add meroku #{remote_uri}")
     end
   end
 end

data/lib/meroku/aws.rb

@@ -33,6 +33,11 @@ module Meroku
       Meroku::Shared.ec2_client.associate_address(
         allocation_id: allocation_id, instance_id: instance_id
       )
+      # TODO: stub this in tests
+      print '*'
+      TCPSocket.new '34.239.241.218', 22
+      # Socket.tcp("www.meroku.com", 22, connect_timeout: 60) {}
+      puts '*'
     end
 
     # Private S3 Bucket

data/lib/meroku/backup.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Meroku
+  # Logic related to backup of production server
+  class Backup
+    include Meroku::Sshable
+
+    def initialize
+      @connection = Net::SSH.start('34.239.241.218', 'ubuntu', sshopts)
+      execute_script <<~HEREDOC
+        SECRET=#{Meroku::Shared.secrets.meroku_secret} ~/backend_api/script/backup_db
+        SECRET=#{Meroku::Shared.secrets.meroku_secret} ~/backend_api/script/backup_etc
+        SECRET=#{Meroku::Shared.secrets.meroku_secret} ~/backend_api/script/backup_nginxconfs
+        SECRET=#{Meroku::Shared.secrets.meroku_secret} ~/backend_api/script/backup_homedirs
+      HEREDOC
+      puts 'OK'
+    end
+  end
+end

data/lib/meroku/cli.rb

@@ -49,8 +49,12 @@ module Meroku
       Meroku::Shared.secrets.meroku_secret = @options[:meroku_secret] \
         if @options[:meroku_secret]
 
-      Node.new if @options[:spawn]
+      if @options[:spawn]
+        Node.new
+        puts 'Node created'
+      end
       Meroku::Aws.terminate_all(tag: 'node') if @options[:despawn]
+      Meroku::Backup.new if @options[:backup]
     end
 
     def act_on_user_options

data/lib/meroku/node.rb

@@ -5,51 +5,28 @@ module Meroku
   class Node
     include Meroku::Shared
     include Meroku::Aws
-    attr_reader :instance_id, :connection
+    include Meroku::Sshable
+    attr_reader :instance_id
 
     def initialize
       make_instance && associate_address
-      start_ssh && configure_keys
-      install_packages
+      start_ssh && user_creations && configure_keys
+      install_packages && homedir_creations
       database_inits && git_clone
-      nginx_configs && start_rails && close_ssh
-      puts 'Node created'
+      nginx_configs
+      start_rails
+      close_ssh
     end
 
-    def execute_script(script)
-      script.each_line do |line|
-        puts line
-        execute(line)
-      end
-    end
+    private
 
     def start_ssh
       @connection = Net::SSH.start(ip_address, 'ubuntu', sshopts)
     end
 
-    def close_ssh
-      @connection.close
-    end
-
-    def execute(command)
-      @connection.exec(command) { |_ch, _stream, data| puts data }
-      @connection.loop
-    end
-
-    private
-
-    def sshopts
-      {
-        verify_host_key: false,
-        key_data: Meroku::Shared.secrets.private_key
-      }
-    end
-
     def configure_keys
+      configure_host_keys
       execute_script <<~HEREDOC
-        sudo curl -s -o /etc/ssh/ssh_host_ecdsa_key #{bucket}ssh_host_ecdsa_key
-        sudo curl -s -o /etc/ssh/ssh_host_ecdsa_key.pub #{bucket}ssh_host_ecdsa_key.pub
-        sudo service ssh restart
         curl -s -o /home/ubuntu/letsencrypt_fullchain.pem #{bucket}letsencrypt_fullchain.pem
         curl -s -o /home/ubuntu/letsencrypt_privkey.pem #{bucket}letsencrypt_privkey.pem
         curl -s https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
@@ -57,6 +34,12 @@ module Meroku
       HEREDOC
     end
 
+    def configure_host_keys
+      execute_script <<~HEREDOC
+        sudo sh -c 'curl -s #{bucket}ssh_host_.tar | tar xvf - -C /etc/ssh/'
+      HEREDOC
+    end
+
     def install_packages
       ubuntu_site = 'http://archive.ubuntu.com/ubuntu/'
       postgres_site = 'http://apt.postgresql.org/pub/repos/apt/'
@@ -68,7 +51,22 @@ module Meroku
         echo "deb #{postgres_site} trusty-pgdg main" | sudo tee -a /etc/apt/sources.list
         DEBIAN_FRONTEND=noninteractive sudo apt-get update -qq
         DEBIAN_FRONTEND=noninteractive sudo apt-get upgrade -yqq --force-yes 2>/dev/null >/dev/null
-        DEBIAN_PRIORITY=critical DEBIAN_FRONTEND=noninteractive sudo apt-get install -q -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" autoconf bind9-host bison build-essential coreutils curl daemontools dnsutils ed git imagemagick iputils-tracepath language-pack-en libbz2-dev libcurl4-openssl-dev libev-dev libevent-dev libglib2.0-dev libjpeg-dev libmagickwand-dev libmysqlclient-dev libncurses5-dev librdkafka-dev libreadline6-dev libssl-dev libuv-dev libxml2-dev libxslt-dev netcat-openbsd openjdk-7-jdk openjdk-7-jre-headless openssh-client openssh-server python python-dev socat stunnel syslinux tar telnet zip zlib1g-dev ruby2.4 ruby2.4-dev nginx libsqlite3-dev nodejs libpq-dev postgresql-9.6 2>/dev/null >/dev/null
+        DEBIAN_PRIORITY=critical DEBIAN_FRONTEND=noninteractive sudo apt-get install -q -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" autoconf bind9-host bison build-essential coreutils curl daemontools dnsutils ed git imagemagick iputils-tracepath language-pack-en libbz2-dev libcurl4-openssl-dev libev-dev libevent-dev libglib2.0-dev libjpeg-dev libmagickwand-dev libmysqlclient-dev libncurses5-dev librdkafka-dev libreadline6-dev libssl-dev libuv-dev libxml2-dev libxslt-dev netcat-openbsd openjdk-7-jdk openjdk-7-jre-headless openssh-client openssh-server python python-dev socat stunnel syslinux tar telnet zip zlib1g-dev ruby2.4 ruby2.4-dev nginx libsqlite3-dev nodejs libpq-dev postgresql-9.6 awscli 2>/dev/null >/dev/null
+      HEREDOC
+    end
+
+    def homedir_creations
+      execute_script <<~HEREDOC
+        curl -s -o /tmp/latest #{bucket}homedir_backups/latest
+        tar -tvf /tmp/latest | tail
+        sudo tar -xvf /tmp/latest -C /home/
+        sudo rm -f /tmp/latest
+      HEREDOC
+    end
+
+    def user_creations
+      execute_script <<~HEREDOC
+        sudo sh -c 'curl -s #{bucket}etc_backups/latest | tar xvf - -C /'
       HEREDOC
     end
 
@@ -76,7 +74,10 @@ module Meroku
       execute_script <<~HEREDOC
         sudo -u postgres createuser -e meroku
         sudo -u postgres createdb meroku
-        sudo -u postgres psql -c "ALTER ROLE meroku WITH PASSWORD '#{Meroku::Shared.secrets.meroku_secret}';"
+        #sudo -u postgres psql -c "ALTER ROLE meroku WITH PASSWORD '#{Meroku::Shared.secrets.meroku_secret}';"
+        curl -s -o /tmp/latest #{bucket}db_backups/latest
+        sudo -u postgres psql -f /tmp/latest
+        sudo rm -f /tmp/latest
         sudo perl -pi -e "s#local.*all.*all.*peer#local   all             all                                     md5#" /etc/postgresql/9.6/main/pg_hba.conf
         sudo service postgresql restart
       HEREDOC
@@ -93,21 +94,31 @@ module Meroku
       execute_script <<~HEREDOC
         sudo rm -f /etc/nginx/sites-enabled/* /etc/nginx/sites-available/*
         # Place vhost config for backend_api
-        sudo cp ~/backend_api/config/etc_nginx_sites-enabled_default /etc/nginx/sites-enabled/default
+        sudo cp ~/backend_api/config/etc_nginx_sites-enabled_default /etc/nginx/conf.d/backend_api.conf
+
         # Place vhost config template for end-user apps
         sudo mkdir /world_readable && sudo chmod 777 /world_readable
         sudo cp /home/ubuntu/meroku/data/etc_nginx_sites-enabled_template /world_readable
         sudo chmod a+r /world_readable/etc_nginx_sites-enabled_template
+
+        # consumer apps nginx config folder
+        sudo mkdir /owner_writable/
+        sudo chmod 777 /owner_writable/
+        sudo sh -c 'echo "include /owner_writable/*.conf;" > /etc/nginx/conf.d/include_owner_writable.conf'
+        sudo curl -s -o /tmp/latest #{bucket}nginxconfs_backups/latest
+        sudo tar -xvz -f /tmp/latest -C /owner_writable/
+        sudo chmod a+w /owner_writable/*  # TODO fix this
+        sudo rm -f /tmp/latest
       HEREDOC
     end
 
     def start_rails
       execute_script <<~HEREDOC
-        cd ~/backend_api; sudo gem install bundler; bundle;
-        cd ~/backend_api; SECRET=#{Meroku::Shared.secrets.meroku_secret} RAILS_ENV=production bundle exec rake db:migrate db:seed
+        cd ~/backend_api; sudo gem install bundler; NOKOGIRI_USE_SYSTEM_LIBRARIES=true bundle;
+        cd ~/backend_api; SECRET=#{Meroku::Shared.secrets.meroku_secret} RAILS_ENV=production bundle exec rake db:migrate
         cd ~/backend_api; SECRET=#{Meroku::Shared.secrets.meroku_secret} SECRET_KEY_BASE=#{Meroku::Shared.secrets.meroku_secret} RAILS_ENV=production bundle exec unicorn -D -l unix:///home/ubuntu/backend_api/tmp/backend_api.sock -c config/unicorn.rb
-        sudo /usr/sbin/nginx -s reload
       HEREDOC
+      execute_script 'sudo /home/ubuntu/backend_api/script/start_user_apps'
     end
   end
 end

data/lib/meroku/options.rb

@@ -22,6 +22,7 @@ module Meroku
         add_key_options(opts)
         add_app_options(opts)
         opts.separator ''
+        add_maintainer_spawn_options(opts)
         add_maintainer_options(opts)
       end
     end
@@ -76,7 +77,7 @@ module Meroku
       end
     end
 
-    def add_maintainer_options(opts)
+    def add_maintainer_spawn_options(opts)
       option(opts, '--spawn MEROKU_SECRET') do |meroku_secret|
         @options[:spawn] = true
         @options[:meroku_secret] = meroku_secret
@@ -87,6 +88,13 @@ module Meroku
       end
     end
 
+    def add_maintainer_options(opts)
+      option(opts, '--backup MEROKU_SECRET') do |meroku_secret|
+        @options[:backup] = true
+        @options[:meroku_secret] = meroku_secret
+      end
+    end
+
     # Sets a value in the @options hash, based on the given long option and its
     # value, in addition to calling the block if a block is given.
     def option(opts, *args)
@@ -112,7 +120,8 @@ module Meroku
       version:               'Display version.',
       spawn:                 'Spawn infrastructure',
       keys:                  'List already uploaded keys',
-      create:                'Create an app'
+      create:                'Create an app',
+      backup:                'Take a backup, before despawing'
     }.freeze
   end
 end

data/lib/meroku/sshable.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Meroku
+  # Mix in to objects that can be sshed in to
+  module Sshable
+    attr_reader :connection
+
+    def execute(command)
+      @connection.exec(command) { |_ch, _stream, data| puts data }
+      @connection.loop
+    end
+
+    def sshopts
+      { verify_host_key: false, key_data: Meroku::Shared.secrets.private_key, \
+        timeout: 60 }
+    end
+
+    def close_ssh
+      @connection.close
+    end
+
+    def execute_script(script)
+      script.each_line do |line|
+        puts line
+        execute(line)
+      end
+    end
+  end
+end

data/lib/meroku/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Meroku
-  VERSION = '2.0.27'
+  VERSION = '2.0.28'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: meroku
 version: !ruby/object:Gem::Version
-  version: 2.0.27
+  version: 2.0.28
 platform: ruby
 authors:
 - Meroku System
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-08 00:00:00.000000000 Z
+date: 2017-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: byebug
@@ -180,6 +180,7 @@ files:
 - lib/meroku.rb
 - lib/meroku/app.rb
 - lib/meroku/aws.rb
+- lib/meroku/backup.rb
 - lib/meroku/cli.rb
 - lib/meroku/error.rb
 - lib/meroku/key.rb
@@ -188,6 +189,7 @@ files:
 - lib/meroku/response_handler.rb
 - lib/meroku/secrets.rb
 - lib/meroku/shared.rb
+- lib/meroku/sshable.rb
 - lib/meroku/user.rb
 - lib/meroku/version.rb
 - meroku.gemspec