gaptool-server 0.5.19 → 0.6.0.beta1

data/lib/helpers/redis.rb

@@ -0,0 +1,19 @@
+require 'redis'
+
+# docker links support
+unless ENV['REDIS_PORT_6379_TCP_ADDR'].nil?
+  ENV['REDIS_HOST'] = ENV['REDIS_PORT_6379_TCP_ADDR']
+  ENV['REDIS_PORT'] = ENV['REDIS_PORT_6379_TCP_PORT']
+  ENV.delete('REDIS_PASS')
+end
+
+ENV['REDIS_HOST'] = 'localhost' unless ENV['REDIS_HOST']
+ENV['REDIS_PORT'] = '6379' unless ENV['REDIS_PORT']
+ENV['REDIS_PASS'] = nil unless ENV['REDIS_PASS']
+ENV['REDIS_DB'] = '0' unless ENV['REDIS_DB']
+
+$redis = Redis.new(:host => ENV['REDIS_HOST'],
+                   :port => ENV['REDIS_PORT'].to_i,
+                   :password => ENV['REDIS_PASS'],
+                   :db => ENV['REDIS_DB'].to_i)
+

data/lib/helpers/rehash.rb

@@ -1,63 +1,38 @@
-module RehashHelpers
-  def rehash()
-    @layout = {}
-    @roles = []
-    $redis.keys('app:*').each do |app|
-      role = $redis.hget(app,'role')
-      unless role == 'nil'
-        @roles << role
+module Gaptool
+  module EC2
+    def self.rehash()
+      servers.each do |inst|
+        rmserver inst
       end
-    end
-    $redis.keys('host:*').each do |key|
-      @roles << key.gsub('host:','').gsub(/:.*:i-.*/,'')
-    end
-    @roles.uniq!
-    @roles.each do |role|
-      apps = []
-      $redis.keys('app:*').each do |app|
-        if $redis.hget(app,'role') == role
-          apps << app.gsub('app:','')
-        end
-      end
-      @layout[role] = apps
-    end
-
-    $redis.keys("host:*").each do |rediskey|
-      $redis.del rediskey
-    end
-
-    $redis.keys("instance:*").each do |rediskey|
-      $redis.del rediskey
-    end
-
-    @zones = $redis.hgetall('amis').keys
-    @zones.each do |zone|
-      @ec2 = AWS::EC2.new(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{zone}.amazonaws.com")
-      @instances = []
-      @ec2.instances.each do |instance|
-        if instance.tags['gaptool'] == 'yes' && instance.status == :running
-          @instances << instance
+      layout = roles()
+      puts layout
+      zones.each do |zone|
+        @ec2 = AWS::EC2.new(:access_key_id => $redis.hget('config', 'aws_id'),
+                            :secret_access_key => $redis.hget('config', 'aws_secret'),
+                            :ec2_endpoint => "ec2.#{zone}.amazonaws.com")
+        ilist = []
+        @ec2.instances.each do |instance|
+          if instance.tags['gaptool'] == 'yes' && instance.status == :running
+            ilist << instance
+          end
         end
-      end
-      @instances.each do |instance|
-        @name = instance.tags['Name'].split('-')
-        @rediskey = "host:#{@name[0]}:#{@name[1]}:i-#{@name.last}"
-        @host = {
-          "zone"=> instance.availability_zone,
-          "itype"=> instance.instance_type,
-          "role"=> @name.first,
-          "environment"=> @name[1],
-          "secret"=>"NA",
-          "capacity"=>"6",
-          "hostname"=> instance.public_dns_name,
-          "apps" => @layout[@name[0]].to_s,
-          "instance"=> instance.instance_id
-        }
-        @host.keys.each do |key|
-          $redis.hset(@rediskey, key, @host[key])
+        ilist.each do |instance|
+          puts " - #{instance.tags['Name']}"
+          role, environment, iid = instance.tags['Name'].split('-')
+          data = {
+            "zone"=> instance.availability_zone,
+            "itype"=> instance.instance_type,
+            "role"=> role,
+            "environment"=> environment,
+            "hostname"=> instance.public_dns_name,
+            "apps" => layout[role].to_s,
+            "instance"=> instance.instance_id
+          }
+          puts data
+          addserver(data['instance'], data, nil)
         end
       end
+      return {"action" => "complete"}
     end
-    return {"action" => "complete"}
   end
 end

data/lib/routes.rb

@@ -0,0 +1,172 @@
+# encoding: utf-8
+require 'securerandom'
+require 'set'
+require_relative 'exceptions'
+class GaptoolServer < Sinatra::Application
+
+  def require_parameters(required_keys, data)
+    data = data.delete_if { |k,v| v.nil? }
+    required_keys = [*required_keys].to_set unless required_keys.is_a? Set
+    keys = data.keys.to_set
+    unless keys >= required_keys
+      raise BadRequest, "Missing required_parameters: #{(required_keys - keys).to_a.join(" ")}"
+    end
+    data
+  end
+
+  get '/' do
+    "You must be lost. Read the instructions."
+  end
+
+  get '/ping' do
+    "PONG"
+  end
+
+  post '/init' do
+    data = require_parameters(%w(role environment zone itype),
+                              JSON.parse(request.body.read))
+
+    secret = SecureRandom.hex(12)
+    security_group = data['security_group'] || Gaptool::Data::get_role_data(data['role'])["security_group"]
+    sgid = Gaptool::EC2::get_or_create_securitygroup(data['role'], data['environment'], data['zone'], security_group)
+    image_id = data['ami'] || Gaptool::Data::get_ami_for_role(data['role'], data['zone'].chop)
+    data['terminable'] = data['terminable'].nil? ? true : !!data['terminable']
+
+    id = Gaptool::EC2::create_ec2_instance(
+    {
+      :image_id => image_id,
+      :availability_zone => data['zone'],
+      :instance_type => data['itype'],
+      :key_name => "gaptool",
+      :security_group_ids => sgid,
+      :user_data => "#!/bin/bash\ncurl --silent -H 'X-GAPTOOL-USER: #{env['HTTP_X_GAPTOOL_USER']}' -H 'X-GAPTOOL-KEY: #{env['HTTP_X_GAPTOOL_KEY']}' #{$redis.hget('config', 'url')}/register -X PUT --data '#{data.to_json}' | bash"
+     }, {
+       role: data['role'],
+       env: data['environment'],
+       zone: data['zone']
+     }
+    )
+    # Add host tag
+    Gaptool::Data::addserver(id, data, secret)
+    json({instance: id,
+          ami: image_id,
+          role: data['role'],
+          environment: data['environment'],
+          secret: secret,
+          terminable: data['terminable'],
+          security_group: sgid})
+  end
+
+  post '/terminate' do
+    data = require_parameters("id",
+                              JSON.parse(request.body.read))
+    host_data = Gaptool::Data::get_server_data data['id']
+    raise NotFound, "No such instance: #{data['id']}" if host_data.nil?
+    raise Conflict, "Instance #{data['id']} cannot be terminated" if host_data['terminable'] == false
+
+    Gaptool::EC2::terminate_ec2_instance(data['zone'], data['id'])
+    Gaptool::Data::rmserver(data['id'])
+    json data['id'] => {'status' => 'terminated'}
+  end
+
+  put '/register' do
+    data = JSON.parse request.body.read
+    data = require_parameters(%w(role zone environment secret), data)
+    instance_id = Gaptool::Data::register_server data['role'], data['environment'], data['secret']
+
+    raise Forbidden, "Can't register instance: wrong secret or missing role/environment" unless instance_id
+
+    hostname = Gaptool::EC2::get_ec2_instance_data(data['zone'].chop, instance_id)[:hostname]
+    host_data = Gaptool::Data::get_server_data instance_id, initkey: true, force_runlist: true
+
+    chef_repo = host_data['chef_repo']
+    chef_branch = host_data['chef_branch']
+    # FIXME: remove init key from redis
+    initkey = host_data['init_key']
+    run_list = host_data['chef_runlist'].to_json
+
+    jdata = {
+      'hostname' => hostname,
+      'recipe' => 'init',
+      'number' => instance_id,
+      'instance' => instance_id,
+      'run_list' => run_list,
+      'role' => data['role'],
+      'environment' => data['environment'],
+      'chefrepo' => chef_repo,
+      'chefbranch' => chef_branch,
+      'identity' => initkey,
+      'appuser' => Gaptool::Data::get_config('appuser'),
+      'apps' => host_data['apps']
+    }.to_json
+
+    erb :init, locals: {
+      initkey: initkey,
+      chef_branch: chef_branch,
+      chef_repo: chef_repo,
+      json: jdata
+    }
+  end
+
+  get '/hosts' do
+    servers = Gaptool::Data::servers.map do |inst|
+      Gaptool::Data::get_server_data inst
+    end
+    json servers
+  end
+
+  get '/apps' do
+    out = {}
+    Gaptool::Data.apps.each do |app|
+      out[app] = Gaptool::Data::get_app_data(app)
+    end
+    json out
+  end
+
+  get '/hosts/:role' do
+    servers =  Gaptool::Data::servers_in_role(params[:role]).map do |inst|
+      Gaptool::Data::get_server_data inst
+    end
+    json servers
+  end
+
+  get '/instance/:id' do
+    json Gaptool::Data::get_server_data(params[:id])
+  end
+
+  get '/hosts/:role/:environment' do
+    if params[:role] == 'ALL'
+      list = Gaptool::Data::servers_in_env params[:environment]
+    else
+      list = Gaptool::Data::servers_in_role_env params[:role], params[:environment]
+    end
+    servers = list.map do |inst|
+      Gaptool::Data::get_server_data inst
+    end
+
+    json servers
+  end
+
+  get '/host/:role/:environment/:instance' do
+    json Gaptool::Data::get_server_data params[:instance]
+  end
+
+  get '/ssh/:role/:environment/:instance' do
+    data = Gaptool::Data::get_server_data params[:instance]
+    host = data['hostname']
+    key, pubkey = Gaptool::EC2::putkey(host)
+    json hostname: host, key: key, pubkey: pubkey
+  end
+
+  get '/version' do
+    version = File.read(File.realpath(
+      File.join(File.dirname(__FILE__), "..", 'VERSION')
+    )).strip
+    json server_version: version, api: {v0: "/"}
+  end
+
+  post '/rehash' do
+    json Gaptool::Rehash::rehash()
+  end
+
+end

data/lib/views/init.erb

@@ -1,14 +1,12 @@
 apt-get update
 apt-get install -ymq zsh git libssl-dev ruby1.9.1-full build-essential
-curl -LO https://www.getchef.com/chef/install.sh
-bash install.sh -v <%= @chef_version %>
-
+gem install --bindir /usr/local/bin --no-ri --no-rdoc chef
 cat << 'EOFKEY' > /root/.ssh/id_rsa
-<%= @initkey %>
+<%= initkey %>
 EOFKEY
 chmod 600 /root/.ssh/id_rsa
 echo 'StrictHostKeyChecking no' > /root/.ssh/config
-git clone -b <%= @chef_branch %> <%= @chef_repo %> /root/ops
-echo '<%= @json %>' > /root/init.json
-chef-solo -c /root/ops/cookbooks/init.rb -j /root/init.json -E <%= @chef_environment %> && \
-    (rm /root/.ssh/id_rsa; rm /root/install.sh; userdel -r ubuntu; rm -rf /root/.ssh; rm -rf /root/ops)
+git clone -b <%= chef_branch %> <%= chef_repo %> /root/ops
+echo '<%= json %>' > /root/init.json
+chef-solo -c /root/ops/cookbooks/init.rb -j /root/init.json && \
+    (rm /root/.ssh/id_rsa; userdel -r ubuntu; rm -rf /root/.ssh; rm -rf /root/ops)

data/tasks/app.rb

@@ -0,0 +1,28 @@
+namespace :app do
+  desc "Add an application"
+  task :add, [:app, :role] do |t, args|
+    abort("Missing parameters") if args[:app].nil? || args[:app].empty? || args[:role].nil? || args[:role].empty?
+    ex = DH.get_app_data(args[:app])
+    unless ex.nil?
+      puts "Updating application #{args[:app]}" unless ex.nil?
+      DH.remove_app(args[:app])
+    end
+    DH.add_app(args[:app], args[:role])
+    puts "Added app #{args[:app]} in role #{args[:role]}"
+  end
+
+  desc "Remove an application"
+  task :delete, [:app] do |t, args|
+    DH.remove_app(args[:app])
+    puts "Removed app #{args[:app]}"
+  end
+end
+
+desc "List apps"
+task :app do
+  puts "Apps:"
+  DH.apps.each do |app|
+    info = DH.get_app_data(app)
+    puts " * #{app} => #{info["role"]}"
+  end
+end

data/tasks/docker.rb

@@ -0,0 +1,84 @@
+def printimages
+  puts %x[docker images | grep '^gild/gaptool']
+end
+
+unless File.exists?('/.dockerenv')
+  namespace :docker do
+    namespace :build do
+      task :image do
+        sys(%w(./scripts/build_docker_images.sh))
+        printimages
+      end
+
+      desc "Build the release image"
+      task :release do
+        sys(%w(./scripts/build_docker_images.sh -t release))
+        printimages
+      end
+
+      task :all => [:image]
+    end
+
+    namespace :push do
+      task :release do
+        sys(%w(docker push gild/gaptool:release))
+      end
+
+      desc 'Push all tags to the Docker Hub'
+      task :all do
+        sys(%w(docker push gild/gaptool))
+      end
+    end
+
+    desc 'Build the docker image'
+    task :build => 'build:image'
+
+    desc "Push the release image to the Docker Hub"
+    task :push => 'push:release'
+
+    task :up => [:build, :recreate]
+
+    desc "Run tests w/ docker"
+    task :test => :build do
+      sys(%w(fig run --rm gaptool rake test))
+    end
+
+    desc "Stop docker containers"
+    task :stop do
+      sys(%w(fig stop))
+    end
+
+    desc "Start docker containers"
+    task :start do
+      sys(%w(fig start))
+    end
+
+    desc "Restart docker containers"
+    task :restart => [:stop, :start]
+
+    desc "Stop and remove docker containers (alias 'rm')"
+    task :remove => :stop do
+      sys(%w(fig rm --force))
+    end
+
+    task :rm => :remove
+
+    desc "Recreate docker containers without building"
+    task :recreate do
+      sys(%w(fig up -d))
+    end
+
+    desc "Run a command in the docker container"
+    task :run do
+      exit sys(%W(fig run --rm gaptool #{ARGV[1..-1].shelljoin}))
+    end
+
+    desc "Run a rake task inside the docker container"
+    task :rake do
+      exit sys(%W(fig run --rm gaptool rake #{ARGV[1..-1].shelljoin}))
+    end
+  end
+
+  desc "Bring up docker containers"
+  task :docker => 'docker:up'
+end

data/tasks/gem.rb

@@ -0,0 +1,27 @@
+namespace :gem do
+
+  desc "Build the gem"
+  task :build => :clean do
+    sys(%w(gem build gaptool-server.gemspec))
+  end
+
+  desc "Clean built file"
+  task :clean do
+    sys(%w(rm -vf *.gem))
+  end
+
+  desc "Bump the version"
+  task :bump do
+    version = File.read('VERSION').strip
+    nver = version.next
+    f = File.open('VERSION', 'w')
+    f.write(nver)
+    f.close
+    puts "Bumped #{version} => #{nver}"
+  end
+
+  desc "Push"
+  task :push => :build do
+    sys(%w(gem push gaptool-server*.gem))
+  end
+end

data/tasks/user.rb

@@ -0,0 +1,32 @@
+namespace :user do
+  desc "Add a new user. rake user:create <username>"
+  task :create, [:username] do |t, args|
+    puts DH.useradd(args[:username])[:key]
+  end
+
+  desc "Rename a user. rake user:rename <oldname> <newname>"
+  task :rename, [:oldname, :newname] do |t, args|
+    user = DH.user(args[:oldname])
+    abort("Unknown user #{args[:oldname]}") if user.nil?
+    DH.userdel(user[:username])
+    DH.useradd(args[:newname], user[:key])
+    puts "User #{args[:oldname]} renamed to #{args[:newname]}"
+  end
+
+  desc "Delete a user. rake user:delete <username>"
+  task :delete, [:username] do |t, args|
+    DH.userdel(args[:username])
+  end
+
+  desc "Set user key. rake user:setkey <username> <key>"
+  task :setkey, [:username, :key] do |t, args|
+    user = DH.user(args[:username])
+    abort("Unknown user #{args[:username]}") if user.nil?
+    puts DH.useradd(args[:username], args[:key])
+  end
+end
+
+desc "List users"
+task :user do
+  puts DH.users.keys.sort.join("\n")
+end