shiprails 0.1.0

data/lib/shiprails/ship/exec.rb

@@ -0,0 +1,140 @@
+require "active_support/all"
+require "aws-sdk"
+require "thor/group"
+
+module Shiprails
+  class Ship < Thor
+    class Exec < Thor::Group
+      include Thor::Actions
+      class_option "path",
+        aliases: ["-p"],
+        default: ".",
+        desc: "Specify a configuration path"
+      class_option "environment",
+        default: "production",
+        desc: "Specify the environment"
+      class_option "region",
+        default: "us-west-2",
+        desc: "Specify the region"
+      class_option "service",
+        default: "app",
+        desc: "Specify the service name"
+      class_option "private-key",
+        default: "~/.ssh/aws.pem",
+        desc: "Specify the AWS SSH private key path"
+
+      def run_command
+        region = options['region']
+        service = options['service']
+        cluster_name = "#{project_name}_#{options['environment']}"
+        command_string = args.join ' '
+        ssh_private_key_path = options['private-key']
+        ecs_exec(region, cluster_name, service, command_string, ssh_private_key_path)
+      end
+
+      private
+
+      def configuration
+        YAML.load(File.read("#{options[:path]}/.shiprails.yml")).deep_symbolize_keys
+      end
+
+      def aws_access_key_id
+        @aws_access_key_id ||= ask "AWS Access Key ID", default: ENV.fetch("AWS_ACCESS_KEY_ID")
+      end
+
+      def aws_access_key_secret
+        @aws_access_key_secret ||= ask "AWS Access Key Secret", default: ENV.fetch("AWS_SECRET_ACCESS_KEY")
+      end
+
+      def project_name
+        configuration[:project_name]
+      end
+
+      def ecs_exec(region, cluster, service, command, ssh_private_key_path, ssh_user: 'ec2-user')
+        # we'll need to use both the ecs and ec2 apis
+        ecs = Aws::ECS::Client.new(region: region)
+        ec2 = Aws::EC2::Client.new(region: region)
+
+        # first we get the ARN of the task managed by the service
+        tasks_list = ecs.list_tasks({cluster: cluster, desired_status: 'RUNNING', service_name: service})
+        task_arn = tasks_list.task_arns.first
+
+        # using the ARN of the task, we can get the ARN of the container instance where its being deployed
+        task_descriptions = ecs.describe_tasks({cluster: cluster, tasks: [task_arn]})
+        task_definition_arn = task_descriptions.tasks.first.task_definition_arn
+        task_definition_name = task_definition_arn.split('/').last
+        container_instance_arn = task_descriptions.tasks.first.container_instance_arn
+        task_definition_description = ecs.describe_task_definition({task_definition: task_definition_name})
+
+        say "Setting up EC2 instance for SSH..."
+        # with the instance ARN let's grab the intance id
+        ec2_instance_id = ecs.describe_container_instances({cluster: cluster, container_instances: [container_instance_arn]}).container_instances.first.ec2_instance_id
+        ec2_instance = ec2.describe_instances({instance_ids: [ec2_instance_id]}).reservations.first.instances.first
+
+        # get its current security groups to restory later
+        security_group_ids = ec2_instance.security_groups.map(&:group_id)
+
+        # create new public ip
+        elastic_ip = ec2.allocate_address({ domain: "vpc" })
+        # link ip to ec2 instance
+        associate_address_response = ec2.associate_address({
+          allocation_id: elastic_ip.allocation_id,
+          instance_id: ec2_instance_id
+        })
+        # create security group for us
+        security_group_response = ec2.create_security_group({
+          group_name: "shiprails-exec-#{cluster}-#{Time.now.to_i}",
+          description: "SSH access to run interactive command (created by #{`whoami`.rstrip} via shiprails)",
+          vpc_id: ec2_instance.vpc_id
+        })
+        # get our public ip
+        my_ip_address = open('http://whatismyip.akamai.com').read
+        # authorize SSH access from our public ip
+        ec2.authorize_security_group_ingress({
+          group_id: security_group_response.group_id,
+          ip_protocol: "tcp",
+          from_port: 22,
+          to_port: 22,
+          cidr_ip: "#{my_ip_address}/32"
+        })
+        # add ec2 instance to our new security group
+        ec2.modify_instance_attribute({
+          instance_id: ec2_instance_id,
+          groups: security_group_ids + [security_group_response.group_id]
+        })
+
+        # build the command we'll run on the instance
+        command_array = ["docker run -it --rm"]
+        task_definition_description.task_definition.container_definitions.first.environment.each do |env|
+          command_array << "-e #{env.name}='#{env.value}'"
+        end
+        command_array << task_definition_description.task_definition.container_definitions.first.image
+        command_array << command
+        command_string = command_array.join ' '
+
+        say "Waiting for AWS to setup networking..."
+        sleep 5 # AWS just needs a little bit to setup networking
+        say "Connecting #{ssh_user}@#{elastic_ip.public_ip}..."
+        say "Executing: $ #{command_string}"
+        system "ssh -o ConnectTimeout=15 -o 'StrictHostKeyChecking no' -t -i #{ssh_private_key_path} #{ssh_user}@#{elastic_ip.public_ip} '#{command_string}'"
+      rescue => e
+        say "Error: #{e.message}", :red
+      ensure
+        say "Cleaning up SSH access..."
+        # restore original security groups
+        ec2.modify_instance_attribute({
+          instance_id: ec2_instance_id,
+          groups: security_group_ids
+        }) rescue nil
+        # remove our access security group
+        ec2.delete_security_group({ group_id: security_group_response.group_id }) rescue nil
+        # unlink ec2 instance from public ip
+        ec2.disassociate_address({ association_id: associate_address_response.association_id }) rescue nil
+        # release public ip address
+        ec2.release_address({ allocation_id: elastic_ip.allocation_id }) rescue nil
+        say "Done.", :green
+      end
+
+    end
+  end
+end

data/lib/shiprails/ship/install/.env.erb

@@ -0,0 +1,23 @@
+# https://github.com/ddollar/forego
+
+COMPOSE_PROJECT_NAME=<%= project_name %>
+APPLICATION_HOST=<%= application_host %>
+ASSET_HOST=<%= application_host %>
+APPLICATION_HOST=<%= application_host %>
+
+PORT=3000
+RACK_ENV=development
+SECRET_KEY_BASE=development_secret
+EXECJS_RUNTIME=Node
+
+SMTP_ADDRESS=smtp.example.com
+SMTP_DOMAIN=example.com
+SMTP_PASSWORD=password
+SMTP_USERNAME=username
+
+RACK_MINI_PROFILER=1
+WEB_CONCURRENCY=1
+MAX_THREADS=4
+
+FACEBOOK_APP_ID=FILL_ME_IN
+FACEBOOK_SECRET=FILL_ME_IN

data/lib/shiprails/ship/install/Dockerfile.erb

@@ -0,0 +1,28 @@
+FROM ruby:<%= ruby_version %>-slim
+
+RUN apt-get update -qq && apt-get install -y build-essential git-core
+
+# for postgres
+RUN apt-get install -y libpq-dev
+
+# for nokogiri
+RUN apt-get install -y libxml2-dev libxslt1-dev
+
+# for capybara-webkit
+RUN apt-get install -y libqt4-webkit libqt4-dev xvfb
+
+# for a JS runtime
+RUN apt-get install -y nodejs
+
+# lighten up the image size
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN gem install bundler --no-document
+
+ENV APP_HOME /app
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+ENV BUNDLE_PATH /bundle
+
+ADD . $APP_HOME

data/lib/shiprails/ship/install/Dockerfile.production.erb

@@ -0,0 +1,29 @@
+FROM ruby:<%= ruby_version %>-slim
+
+RUN apt-get update -qq && apt-get install -y build-essential git-core
+
+# for postgres
+RUN apt-get install -y libpq-dev
+
+# for nokogiri
+RUN apt-get install -y libxml2-dev libxslt1-dev
+
+# for capybara-webkit
+RUN apt-get install -y libqt4-webkit libqt4-dev xvfb
+
+# for a JS runtime
+RUN apt-get install -y nodejs
+
+# lighten up the image size
+RUN rm -rf /var/lib/apt/lists/*
+
+RUN gem install bundler --no-document
+
+ENV APP_HOME /app
+RUN mkdir $APP_HOME
+WORKDIR $APP_HOME
+
+ADD Gemfile* $APP_HOME/
+RUN bundle install
+
+ADD . $APP_HOME

data/lib/shiprails/ship/install/docker-compose.yml.erb

@@ -0,0 +1,81 @@
+version: '2'
+
+services:
+  memcached:
+    image: memcached
+  postgres:
+    environment:
+      LC_ALL: C.UTF-8
+    image: postgres
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+  redis:
+    command: redis-server --appendonly yes
+    image: redis
+    volumes:
+      - redis-data:/var/lib/redis
+  app: &app_base
+    build: .
+    command: ./bin/start
+    depends_on:
+      - memcached
+      - postgres
+      - redis
+    environment: &app_environment
+      # PostgreSQL Development Database:
+      DATABASE_URL: postgres://postgres:@postgres:5432/development?pool=25&encoding=unicode&schema_search_path=public
+      # memcached Development Cache:
+      MEMCACHED_URL: memcached:11211
+      # Redis Database:
+      REDIS_URL: redis://redis:6379
+      DOCKERIZED: 1
+      POOL_SIZE: 5
+
+      # Sidekiq configuration:
+      SIDEKIQ_CONCURRENCY: 5
+      SIDEKIQ_TIMEOUT: 10
+
+      # Enable the byebug debugging server - this can be overriden
+      # from the command line:
+      ENABLE_DEBUG_SERVER: 0
+
+      # Run the app in the 'development' environment:
+      RACK_ENV: development
+    env_file: .env
+    ports:
+      - "3000:3000"
+    stdin_open: true
+    tmpfs: /app/tmp
+    volumes:
+      - ".:/app"
+      - gems-data:/bundle
+  worker:
+    <<: *app_base
+    command: bundle exec sidekiq -C config/sidekiq.yml
+    ports: []
+  cable:
+    <<: *app_base
+    command: bundle exec puma --bind tcp://0.0.0.0:28080 cable/config.ru
+    ports:
+      - 28080:28080
+  # App Guard: Keeps running tests on a separate process:
+  test:
+    <<: *app_base # We copy from &app_base, and override:
+    command: bundle exec guard start --no-bundler-warning --no-interactions
+    environment:
+      <<: *app_environment
+      # PostgreSQL Test Database:
+      DATABASE_URL: postgres://postgres:@postgres:5432/test?pool=25&encoding=unicode&schema_search_path=public
+
+      # Run the app in the 'test' environment, instead of the default 'developent'
+      RACK_ENV: test
+      RAILS_ENV: test
+    ports: []
+
+volumes:
+  gems-data:
+    driver: local
+  postgres-data:
+    driver: local
+  redis-data:
+    driver: local

data/lib/shiprails/ship/install/shiprails.yml.erb

@@ -0,0 +1,29 @@
+config_s3_bucket: '<%= config_s3_bucket %>'
+project_name: '<%= project_name %>'
+services:
+<%- services.each do |service| -%>
+  <%= service[:name] %>:
+    command: '<%= service[:command] %>'
+    image: '<%= service[:image] %>'
+<%- if service[:ports].any? -%>
+    ports:
+<%- service[:ports].each do |port| -%>
+      - <%= port %>
+<%- end -%>
+<%- end -%>
+    regions:
+<%- service[:regions].each do |region, data| -%>
+      <%= region %>:
+        environments:
+<%- environments.each do |environment| -%>
+          - <%= environment[:name] %>
+<%- end -%>
+<%- data.each do |k,v| -%>
+        <%= k %>: '<%= v %>'
+<%- end -%>
+<%- end -%>
+    resources:
+<%- service[:resources].each do |k,v| -%>
+      <%= k %>: <%= v %>
+<%- end -%>
+<%- end -%>

data/lib/shiprails/ship/install.rb

@@ -0,0 +1,159 @@
+require "active_support/all"
+require "aws-sdk"
+require "thor/group"
+
+module Shiprails
+  class Ship < Thor
+    class Install < Thor::Group
+      include Thor::Actions
+
+      class_option "path",
+        aliases: ["-p"],
+        default: ".",
+        desc: "Specify a configuration path"
+
+      def self.source_root
+        File.expand_path("../install", __FILE__)
+      end
+
+      def application_host
+        "#{project_name}.dev"
+      end
+
+      no_commands {
+        def aws_access_key_id
+          @aws_access_key_id ||= ask "AWS Access Key ID", default: ENV.fetch("AWS_ACCESS_KEY_ID")
+        end
+
+        def aws_access_key_secret
+          @aws_access_key_secret ||= ask "AWS Access Key Secret", default: ENV.fetch("AWS_SECRET_ACCESS_KEY")
+        end
+      }
+
+      def config_s3_bucket
+        return @bucket_name unless @bucket_name.nil?
+        @_s3_client ||= Aws::S3::Client.new(region: ENV.fetch("AWS_REGION", "us-west-2"), access_key_id: aws_access_key_id, secret_access_key: aws_access_key_secret)
+        begin
+          bucket_name = "#{project_name}-config"
+          bucket_name = ask "S3 bucket name for configuration store", default: bucket_name
+          resp = @_s3_client.create_bucket({
+            bucket: bucket_name
+          })
+        rescue Aws::S3::Errors::BucketAlreadyExists
+          error "'#{bucket_name}' already exists"
+          retry
+        rescue Aws::S3::Errors::BucketAlreadyOwnedByYou
+        end
+        @bucket_name = bucket_name
+      end
+
+      def environments
+        environments = Dir.entries("#{Dir.getwd}/config/environments").grep(/\.rb$/).map { |fname| fname.chomp!(".rb") }.select{ |e| !['development', 'test'].include? e } rescue ['production']
+        environments ||= ['production']
+        @regions ||= ask("Which regions?", default: 'us-west-2').split(',')
+        environments.map do |environment|
+          {
+            name: environment,
+            regions: @regions
+          }
+        end
+      end
+
+      no_commands {
+      def services
+        return @services unless @services.nil?
+        docker_compose = YAML.load(File.read("#{options[:path]}/docker-compose.yml")).deep_symbolize_keys
+        image_for_build = {}
+        regions_for_build = {}
+        @services = docker_compose[:services].map do |service_name, service|
+          next if [:test].include? service_name
+          if service[:image].nil?
+            build_name = service[:build]
+            build_name = "Dockerfile" if build_name == '.'
+            unless image = image_for_build[build_name]
+              image = service_name
+              image_for_build[build_name] = image
+            end
+            unless regions = regions_for_build[build_name]
+              regions = @regions.map do |region|
+                [region, {
+                  repository_url: repository_url_for_region(region, service_name)
+                }]
+              end
+              regions_for_build[build_name] = regions
+            end
+            {
+              command: service[:command],
+              image: image,
+              name: service_name.to_s,
+              ports: (service[:ports] || []).map{ |port| port.split(":").last },
+              regions: regions,
+              resources: {
+                cpu_units: 256,
+                memory_units: 256
+              }
+            }
+          end
+        end.compact
+      end
+
+      def project_name
+        @project_name ||= ask "What's your project called?", default: File.basename(Dir.getwd)
+      end
+
+      def ruby_version
+        "#{RUBY_VERSION}"
+      end
+      }
+
+      def create_dockerfile
+        template("Dockerfile.erb", "#{options[:path]}/Dockerfile")
+        template("Dockerfile.production.erb", "#{options[:path]}/Dockerfile.production")
+      end
+
+      def create_dot_env
+        template(".env.erb", "#{options[:path]}/.env")
+        template(".env.erb", "#{options[:path]}/.env.example")
+      end
+
+      def ignore_dot_env
+        if File.exists?(".gitignore")
+          append_to_file(".gitignore", <<-EOF)
+
+# Ignore Docker ENV
+/.env
+EOF
+        end
+      end
+
+      def create_docker_compose
+        template("docker-compose.yml.erb", "#{options[:path]}/docker-compose.yml")
+      end
+
+      def create_configuration
+        template("shiprails.yml.erb", "#{options[:path]}/.shiprails.yml")
+      end
+
+      private
+
+      def repository_url_for_region(region, service_name)
+        @_ecr_client ||= Aws::ECR::Client.new(region: region, access_key_id: aws_access_key_id, secret_access_key: aws_access_key_secret)
+        resp = @_ecr_client.describe_repositories({}).to_h
+        say "Amazon EC2 Container Registry (ECR) for #{project_name}_#{service_name}?"
+        choices = ["CREATE NEW REGISTRY"] + resp[:repositories].map{|r| "#{r[:repository_name]} (#{r[:repository_uri]})" }
+        choices = choices.map.with_index{ |a, i| [i+1, *a]}
+        print_table choices
+        selection = ask("Pick one:").to_i
+        if selection == 1
+          resp = @_ecr_client.create_repository({
+            repository_name: "#{project_name}/#{service_name}",
+          }).to_h
+          resp[:repository][:repository_uri]
+        else
+          resp[:repositories][selection - 2][:repository_uri]
+        end
+      end
+
+    end
+  end
+end

data/lib/shiprails/ship/scale.rb

@@ -0,0 +1,81 @@
+require "active_support/all"
+require "aws-sdk"
+require "git"
+require "thor/group"
+
+module Shiprails
+  class Ship < Thor
+    class Scale < Thor::Group
+      include Thor::Actions
+
+      argument :method_name, type: :string
+      argument :environment, type: :string
+      argument :service, type: :string
+      argument :scale, type: :string
+
+      class_option "region",
+        desc: "Specify region"
+      class_option "path",
+        aliases: ["-p"],
+        default: ".",
+        desc: "Specify a configuration path"
+
+      def update_ecs_services
+        say "Setting ECS service #{service} scale=#{scale} in #{environment}..."
+        configuration[:services].each do |service_name, service|
+          next unless service_name.to_s == self.service
+          image_name = "#{project_name}_#{service_name}"
+          service[:regions].each do |region_name, region|
+            next unless options["region"].nil? or options["region"] == region_name.to_s
+            ecs = Aws::ECS::Client.new(region: region_name.to_s)
+            region[:environments].each do |environment_name|
+              next unless environment_name == self.environment
+              cluster_name = "#{project_name}_#{environment_name}"
+              task_name = "#{image_name}_#{environment_name}"
+              begin
+                task_definition_description = ecs.describe_task_definition({task_definition: task_name})
+                task_definition = task_definition_description.task_definition.to_hash
+              rescue Aws::ECS::Errors::ClientException => e
+                say "Missing ECS task for #{task_name}!", :red
+                say "Run `ship setup`", :red
+                exit
+              end
+              begin
+                service_response = ecs.update_service({
+                  cluster: cluster_name,
+                  service: service_name,
+                  desired_count: scale
+                })
+                say "Set ECS service #{service_name} scale=#{scale} in #{environment} (#{region_name})...", :green
+              rescue Aws::ECS::Errors::ServiceNotFoundException, Aws::ECS::Errors::ServiceNotActiveException => e
+                say "Missing ECS service for #{task_name}!", :red
+                say "Run `ship setup`", :red
+                exit
+              end
+            end
+          end
+        end
+        say "ECS service updated.", :green
+      end
+
+      private
+
+      def aws_access_key_id
+        @aws_access_key_id ||= ask "AWS Access Key ID", default: ENV.fetch("AWS_ACCESS_KEY_ID")
+      end
+
+      def aws_access_key_secret
+        @aws_access_key_secret ||= ask "AWS Access Key Secret", default: ENV.fetch("AWS_SECRET_ACCESS_KEY")
+      end
+
+      def configuration
+        YAML.load(File.read("#{options[:path]}/.shiprails.yml")).deep_symbolize_keys
+      end
+
+      def project_name
+        configuration[:project_name]
+      end
+
+    end
+  end
+end