gaptool-server 0.5.19 → 0.6.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dc933d12128d570ffc1b9bfc876a9d36dda2bfa2
-  data.tar.gz: 681bbe145ee819be3519ae5910a7f243c9e52ff1
+  metadata.gz: f07c5a5a5853295749abfb9e33430dd54ad1678b
+  data.tar.gz: 1b6082b74ddbe7bcd38efaa24a73b75ecc58f2e5
 SHA512:
-  metadata.gz: 8a0c3c12f5080a09d7f75fb1d0d898f7b9e33af6351225d19f70653b8b7d4fe168ce85e9435516f03a4557f73bff5e2ee7625928c41cd56982d5817a232eab81
-  data.tar.gz: 3db3357844481c1ee51e415c13b7620283b5568a6fbb198ec20f17ef274e695b67ab58d2fde0a25bee3a592a7ce1c4666d488c30e2c07a3999fe1125f1f97145
+  metadata.gz: 9e86ef3243630f8e5633721d7c69c29471b887285c3156bc6984118f843aa9aff5ebd13bc9d92fedd2c0896500ca9944fff58a3849ab369fd04dd0e15b9c9051
+  data.tar.gz: 44d38d87ccfc61820177391152663335f6a2f2c30ba8d268852ca7d7b2810001e248f5090069482e2c158d031bb07beffb4543858c4eedca4f986450ac39904a

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Rakefile

@@ -0,0 +1,43 @@
+# encoding: utf-8
+Dir.chdir(File.dirname(__FILE__))
+
+require 'shellwords'
+require 'rspec/core/rake_task'
+require_relative 'lib/helpers/redis'
+require_relative 'lib/helpers/init'
+
+DH = Gaptool::Data
+$stdout.sync = true
+
+def sys(cmd)
+  IO.popen(cmd) do |f|
+    until f.eof?
+      puts f.gets
+    end
+  end
+  $?.to_i
+end
+
+Dir.glob('tasks/*.rb').each { |r| load r}
+
+desc "Start the shell"
+task :shell do
+  exec "racksh #{Shellwords.join(ARGV[1..-1])}"
+end
+task :sh => :shell
+
+desc "Start the HTTP server"
+task :server do
+  exec "unicorn #{Shellwords.join(ARGV[1..-1])}"
+end
+
+task :help do
+  puts "Available tasks"
+  exec "rake -T"
+end
+
+RSpec::Core::RakeTask.new :test do |task|
+  task.pattern = Dir['test/*_test.rb']
+end
+
+task :default => :help

data/VERSION

@@ -1 +1 @@
-0.5.19
+0.6.0.beta1

data/bin/gaptool-server

@@ -1,33 +1,11 @@
 #!/usr/bin/env ruby
 require 'rubygems'
-require 'thin'
-#require 'gaptool-server/app.rb'
 
-# Hijack the thin module to set default options to run in the right place
-module Thin
-  class GaptoolRunner < Thin::Runner
-    def initialize(argv)
-      @argv = argv
-
-      # Default options values
-      @options = {
-        :chdir                => File.expand_path('..', File.dirname(__FILE__)),
-        :environment          => ENV['RACK_ENV'] || 'development',
-        :address              => '0.0.0.0',
-        :port                 => Server::DEFAULT_PORT,
-        :timeout              => Server::DEFAULT_TIMEOUT,
-        :log                  => 'log/thin.log',
-        :pid                  => 'tmp/pids/thin.pid',
-        :max_conns            => Server::DEFAULT_MAXIMUM_CONNECTIONS,
-        :max_persistent_conns => Server::DEFAULT_MAXIMUM_PERSISTENT_CONNECTIONS,
-        :require              => [],
-        :wait                 => Controllers::Cluster::DEFAULT_WAIT_TIME
-      }
-
-      parse!
-    end
-  end
+begin
+  require 'gaptool-server'
+  root = Gem.loaded_specs['gaptool-server'].full_gem_path
+rescue LoadError
+  root = File.realpath(File.join(File.dirname(__FILE__), ".."))
 end
-
-# Run with thin, with thin args
-Thin::GaptoolRunner.new(ARGV).run!
+Dir.chdir(root)
+exec "unicorn #{ARGV.join(" ")}"

data/bin/gaptool-shell

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+
+begin
+  require 'gaptool-server'
+  root = Gem.loaded_specs['gaptool-server'].full_gem_path
+rescue LoadError
+  root = File.realpath(File.join(File.dirname(__FILE__), ".."))
+end
+Dir.chdir(root)
+
+exec "racksh #{ARGV.join(" ")}"

data/bin/gaptool_server_migrate_0.5-0.6

@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+#
+# convert the old data model for the gaptool server
+# to the new struct
+
+libpath = File.expand_path(File.join(File.dirname(__FILE__), "..", "lib"))
+$:.unshift(libpath)
+require "#{libpath}/helpers/redis"
+require "#{libpath}/helpers/data"
+DH = Gaptool::Data
+
+puts "Removing cruft"
+# First, remove cruft. Like all servers created but never registered, etc
+['instance:*', '_*', 'apikey:*', 'service:*', 'running', 'ec2*',
+ 'failover', 'capacity', 'sg:*'].each do |pattern|
+  $redis.keys(pattern).each do |key|
+    $redis.del key
+  end
+end
+
+puts "Migrating roles..."
+$redis.keys('role:*').each do |key|
+  puts " - #{key}"
+  data = $redis.hgetall(key)
+  role = key.sub('role:', '')
+  unless data['ami'].nil?
+    # We didn't have per-regin ami in roles, so
+    # default to us-west-2
+    data['amis'] = {'us-west-2' => data['ami']}
+    role.delete('ami')
+  end
+  DH.save_role_data(role, data)
+end
+
+puts "Migrating ami info to role..."
+$redis.keys('amis:*').each do |key|
+  role = key.sub("amis:", '')
+  puts " - #{key}"
+  amis = $redis.hgetall(key)
+  roledata = DH.get_role_data(role)
+  roledata['amis'] = amis.merge(roledata['amis'] || {})
+  DH.save_role_data(role, roledata)
+  $redis.del(key)
+end
+
+puts "Migrating apps..."
+$redis.keys('app:*').each do |key|
+  puts " - #{key}"
+  appname = key.sub('app:', '')
+  role = $redis.hget(key, "role")
+  if role.nil? || appname.nil?
+    puts "Missing info for app: #{appname} role: #{role}"
+  end
+  $redis.del(key)
+  DH.add_app(appname, role) unless role.nil?
+end
+
+puts "Migrating instances..."
+$redis.keys('host:*').each do |key|
+  puts " - #{key}"
+  pfx, role, env, iid = key.split(':')
+  data = $redis.hgetall(key)
+  data['role'] = role
+  data['environment'] = env
+  DH.addserver(iid, data, nil)
+  $redis.del(key)
+  # remove stale keys from hsh
+  $redis.hdel "instance:#{iid}", ["capacity", "mirror", "secret", 'instance']
+end
+
+puts "Removing unneeded info from instance data..."
+$redis.smembers('instances').each do |inst|
+  $redis.hdel "instance:#{inst}", ["capacity", "mirror", "secret", "instance"]
+  if $redis.hget("instance:#{inst}", 'chef_runlist') == "recipe[init]"
+    $redis.hdel "instance:#{inst}", "chef_runlist"
+  end
+  term = $redis.hget("instance:#{inst}", "terminate")
+  unless term.nil?
+    unless term == 'true'
+      $redis.hset("instance:#{inst}", "terminable", term)
+    end
+    $redis.hdel("instance:#{inst}", "terminate")
+  end
+end
+
+puts "Rename old config keys"
+$redis.hset('config', 'chef_repo', $redis.hget('config', 'chefrepo'))
+$redis.hset('config', 'chef_branch', $redis.hget('config', 'chefbranch'))
+
+puts "Removing old config keys"
+['remoteredis:host', 'remoteredis:port', 'aws_id', 'aws_secret',
+ 'chefrepo', 'chefbranch'].each do |key|
+  $redis.hdel 'config', key
+end
+
+$redis.save

data/config.ru

@@ -1,19 +1,10 @@
 #!/usr/bin/env ruby
 
-require 'redis'
-
-ENV['REDIS_HOST'] = 'localhost' unless ENV['REDIS_HOST']
-ENV['REDIS_PORT'] = '6379' unless ENV['REDIS_PORT']
-ENV['REDIS_PASS'] = nil unless ENV['REDIS_PASS']
-ENV['REDIS_DB'] = '0' unless ENV['REDIS_DB']
-$redis = Redis.new(:host => ENV['REDIS_HOST'],
-                   :port => ENV['REDIS_PORT'],
-                   :password => ENV['REDIS_PASS'],
-                   :db => ENV['REDIS_DB'])
+ENV['DRYRUN'] = nil unless ENV['DRYRUN'] == 'true'
 
 libpath = File.expand_path(File.join(File.dirname(__FILE__), "lib"))
 $:.unshift(libpath)
+require "#{libpath}/helpers/redis"
 require "#{libpath}/app.rb"
 
-instance = GaptoolServer.new
-run instance
+run GaptoolServer

data/lib/app.rb

@@ -1,5 +1,6 @@
 # encoding: utf-8
 require 'sinatra'
+require 'sinatra/json'
 require 'json'
 require 'yaml'
 require 'erb'
@@ -8,11 +9,30 @@ require 'openssl'
 require 'net/ssh'
 require 'peach'
 require 'airbrake'
+require_relative 'exceptions'
 
 class GaptoolServer < Sinatra::Application
+  helpers Sinatra::JSON
+
+  def error_response msg=''
+    message = "#{env['sinatra.error'].message}"
+    message = "#{msg} #{message}" unless msg.empty?
+    json result: 'error', message: message
+  end
+
+  error JSON::ParserError do
+    status 400
+    error_response "Invalid data."
+  end
+
+  error HTTPError do
+    status env['sinatra.error'].code
+    error_response
+  end
 
   error do
-    {:result => 'error', :message => env['sinatra.error']}.to_json
+    status 500
+    error_response
   end
 
   configure do
@@ -24,20 +44,17 @@ class GaptoolServer < Sinatra::Application
     end
     disable :sessions
     enable  :dump_errors
+    disable :show_exceptions
   end
 
   before do
-    if request.path_info != '/ping'
-      error 401 unless $redis.hget('users', env['HTTP_X_GAPTOOL_USER']) == env['HTTP_X_GAPTOOL_KEY']
-      error 401 unless env['HTTP_X_GAPTOOL_USER'] && env['HTTP_X_GAPTOOL_KEY']
+    if request.path_info != '/ping' && ENV['GAPTOOL_DISABLE_AUTH'].nil?
+      user = Gaptool::Data.user(env['HTTP_X_GAPTOOL_USER'])
+      raise Unauthenticated if user.nil?
+      raise Unauthenticated unless user[:key] == env['HTTP_X_GAPTOOL_KEY']
     end
   end
-
-  helpers do
-    include Rack::Utils
-    alias_method :h, :escape_html
-  end
 end
 
 require_relative 'helpers/init'
-require_relative 'routes/init'
+require_relative 'routes'

data/lib/exceptions.rb

@@ -0,0 +1,35 @@
+class HTTPError < StandardError
+  @code = 500
+
+  def self.code
+    @code
+  end
+
+ def code
+    self.class.code
+ end
+end
+
+class ClientError < HTTPError
+  @code = 400
+end
+
+class BadRequest < ClientError
+  @code = 400
+end
+
+class Unauthenticated < ClientError
+  @code = 401
+end
+
+class Forbidden < ClientError
+  @code = 403
+end
+
+class NotFound < ClientError
+  @code = 404
+end
+
+class Conflict < ClientError
+  @code = 409
+end

data/lib/helpers/data.rb

@@ -0,0 +1,257 @@
+require 'json'
+require 'securerandom'
+
+module Gaptool
+  module Data
+
+    def self.init_recipe
+      'recipe[init]'
+    end
+
+    def self.default_runlist
+      [init_recipe]
+    end
+
+    def self.addserver(instance, data, secret)
+      role = data['role']
+      environment = data['environment']
+      if role.nil? || environment.nil?
+        raise ArgumentError, "Missing role or environment"
+      end
+
+      if instance.nil? || instance.empty?
+        raise ArgumentError, "Missing instance"
+      end
+
+      $redis.sadd("instances", instance)
+      $redis.sadd("role:#{role}:instances", instance)
+      $redis.sadd("environment:#{environment}:instances", instance)
+      unless secret.nil?
+        data['registered'] = false
+        $redis.sadd('instances:unregistered', instance)
+        $redis.set("instances:secrets:#{data['role']}:#{data['environment']}:#{secret}", instance)
+        data['secret'] = secret
+      end
+      save_server_data instance, data
+    end
+
+    def self.overwrite_hash(key, data)
+      if !key.nil? && !data.nil? && !data.empty?
+        $redis.multi do
+          $redis.del(key)
+          $redis.hmset(key, *data.select{ |k,v| !v.nil? && ((v.is_a?(String) && !v.empty?) || !!v == v)}.flatten)
+        end
+      end
+    end
+
+    def self.save_server_data(instance, data)
+
+      unless data['chef_runlist'].nil?
+        data['chef_runlist'] = [*data['chef_runlist']]
+        unless data['chef_runlist'].include? init_recipe
+          data['chef_runlist'].unshift(init_recipe)
+        end
+
+        if data['chef_runlist'] == default_runlist
+          data.delete('chef_runlist')
+        else
+          data['chef_runlist'] = data['chef_runlist'].to_json
+        end
+      end
+      overwrite_hash("instance:#{instance}", data)
+    end
+
+    def self.register_server(role, environment, secret)
+      key = "instances:secrets:#{role}:#{environment}:#{secret}"
+      instance = nil
+      $redis.watch(key) do
+        instance = $redis.get(key)
+        if !instance.nil? && !instance.empty?
+          $redis.multi do |m|
+            m.hdel("instance:#{instance}", "secret")
+            m.hdel("instance:#{instance}", "registered")
+            m.srem('instances:unregistered', instance)
+            m.del(key)
+          end
+        else
+          $redis.unwatch
+          instance = nil
+        end
+      end
+      instance
+    end
+
+    def self.rmserver(instance)
+      data = get_server_data instance
+      return if data.nil?
+      $redis.multi do |m|
+        m.srem("instances", instance)
+        m.srem("role:#{data['role']}:instances", instance)
+        m.srem("environment:#{data['environment']}:instances", instance)
+        m.del("instance:#{instance}")
+      end
+      instance
+    end
+
+    def self.get_config(key)
+      $redis.hget('config', key)
+    end
+
+    def self.set_config(key, value)
+      $redis.hset('config', key, value)
+    end
+
+    def self.get_server_data(instance, opts={})
+      rs = $redis.hgetall("instance:#{instance}")
+      return nil if rs.nil? || rs.empty?
+      rs['instance'] = instance
+      if !rs['chef_runlist'].nil? && !rs['chef_runlist'].empty?
+        rs['chef_runlist'] = JSON.parse rs['chef_runlist']
+      else
+        rs['chef_runlist'] = get_runlist_for_role rs['role']
+      end
+
+      %w(chef_repo chef_branch).each do |v|
+        if rs[v].nil? || rs[v].empty?
+          rs[v] = get_config(v)
+        end
+      end
+
+      if opts[:initkey]
+        rs['initkey'] = get_config('initkey')
+      end
+
+      if !rs['terminable'].nil? && rs['terminable'] == "false"
+        rs['terminable'] = false
+      else
+        rs.delete('terminable')
+      end
+
+      if opts[:force_runlist] && rs['chef_runlist'].nil?
+        rs['chef_runlist'] = default_runlist
+      end
+
+      rs['apps'] = apps_in_role(rs['role'])
+      rs.delete_if {|k,v| v.nil? || (!!v != v && v.empty?)}
+    end
+
+    def self.save_role_data(role, data)
+      return if role.nil? || data.nil?
+      if data['amis']
+        amis = data.delete("amis") || {}
+        overwrite_hash("role:#{role}:amis", amis)
+      end
+      if data['apps']
+        apps = data.delete("apps") || []
+        apps.each {|a| add_app(a, role)}
+      end
+      overwrite_hash("role:#{role}", data)
+      $redis.sadd("roles", role)
+    end
+
+    def self.get_role_data(role)
+      res = $redis.hgetall("role:#{role}")
+      res['apps'] = apps_in_role(role)
+      res['amis'] = $redis.hgetall("role:#{role}:amis")
+      res
+    end
+
+    def self.get_ami_for_role(role, region=nil)
+
+      $redis.hget("role:#{role}:amis", region) || $redis.hget("amis", region)
+    end
+
+    def self.get_runlist_for_role(role)
+      rl = $redis.hget("role:#{role}", "chef_runlist")
+      return nil if rl.nil?
+      rl = JSON.parse rl
+      return nil if rl == default_runlist
+      rl
+    end
+
+    def self.set_amis(amis)
+      overwrite_hash("amis", amis)
+    end
+
+    def self.zones
+      $redis.hgetall('amis').keys
+    end
+
+    def self.servers_in_role(role)
+      $redis.smembers("role:#{role}:instances")
+    end
+
+    def self.servers_in_env(env)
+      $redis.smembers("environment:#{env}:instances")
+    end
+
+    def self.servers_in_role_env(role, env)
+      $redis.sinter("role:#{role}:instances", "environment:#{env}:instances")
+    end
+
+    def self.servers
+      $redis.smembers("instances")
+    end
+
+    def self.roles
+      Hash[$redis.smembers("roles").map {|r| [r, get_role_data(r)] }]
+    end
+
+    def self.add_app(name, role)
+      return if name.nil? || role.nil?
+      $redis.multi do
+        $redis.sadd("apps", name)
+        $redis.sadd("role:#{role}:apps", name)
+        $redis.set("app:#{name}", role)
+      end
+    end
+
+    def self.remove_app(name)
+      return if name.nil?
+      key = "app:#{name}"
+      $redis.watch(key) do
+        role = $redis.get(key)
+        if !role.nil?
+          $redis.multi do |m|
+            m.del(key)
+            m.srem("apps", name)
+            m.srem("role:#{role}")
+          end
+        else
+          $redis.unwatch
+        end
+      end
+    end
+
+    def self.get_app_data(app)
+      {"role" => $redis.get("app:#{app}")}
+    end
+
+    def self.apps
+      $redis.smembers("apps")
+    end
+
+    def self.apps_in_role(role)
+      $redis.smembers("role:#{role}:apps")
+    end
+
+    def self.useradd(user, key=nil)
+      key = SecureRandom.hex(64) unless key
+      $redis.hset('users', user, key)
+      {username: user, key: key}
+    end
+
+    def self.userdel(user)
+      $redis.hdel('users', user)
+    end
+
+    def self.users
+      $redis.hgetall('users')
+    end
+
+    def self.user(user)
+      userdesc = {username: user, key: $redis.hget('users', user)}
+      return userdesc[:key].nil? ? nil : userdesc
+    end
+  end
+end

data/lib/helpers/ec2.rb

@@ -0,0 +1,80 @@
+require 'securerandom'
+
+# encoding: utf-8
+module Gaptool
+  module EC2
+
+    def self.configure_ec2 zone
+      return if ENV['DRYRUN']
+      id = ENV['AWS_ACCESS_KEY_ID']
+      secret = ENV['AWS_SECRET_ACCESS_KEY']
+      AWS.config(access_key_id: id, secret_access_key: secret,
+                 ec2_endpoint: "ec2.#{zone}.amazonaws.com")
+    end
+
+    def self.putkey(host)
+      return "FAKEKEY", "FAKEPUB" if ENV['DRYRUN']
+      key = OpenSSL::PKey::RSA.new 2048
+      pubkey = "#{key.ssh_type} #{[key.to_blob].pack('m0')} GAPTOOL_GENERATED_KEY"
+      ENV['SSH_AUTH_SOCK'] = ''
+      Net::SSH.start(host, 'admin',
+                     :key_data => [$redis.hget('config', 'gaptoolkey')],
+                     :config => false, :keys_only => true,
+                     :paranoid => false) do |ssh|
+        ssh.exec! "grep -v GAPTOOL_GENERATED_KEY ~/.ssh/authorized_keys > /tmp/pubkeys"
+        ssh.exec! "echo #{pubkey} >> /tmp/pubkeys"
+        ssh.exec! "mv /tmp/pubkeys ~/.ssh/authorized_keys"
+      end
+      return key.to_pem, pubkey
+    end
+
+    def self.get_or_create_securitygroup(role, environment, zone, groupname=nil)
+      return "sg-test#{SecureRandom.hex(2)}" if ENV['DRYRUN']
+      configure_ec2 zone.chop
+      ec2 = AWS::EC2.new
+      groupname = groupname || "#{role}-#{environment}"
+      ec2.security_groups.each do |group|
+        if group.name == groupname
+          return group.id
+        end
+      end
+      internet = ['0.0.0.0/0']
+      sg = ec2.security_groups.create(groupname)
+      sg.authorize_ingress :tcp, 22, *internet
+      return sg.id
+    end
+
+    def self.create_ec2_instance(ec2opts, data)
+      return "i-test#{SecureRandom.hex(2)}" if ENV['DRYRUN']
+      configure_ec2 data['zone'].chop
+      ec2 = AWS::EC2.new
+
+      instance = ec2.instances.create(opts)
+      instance.add_tag('Name', value: "#{data[:role]}-#{data[:env]}-#{instance.id}")
+      instance.add_tag('gaptool', :value => "yes")
+      instance.id
+    end
+
+    def self.terminate_ec2_instance(zone, id)
+      return if ENV['DRYRUN']
+      configure_ec2 zone
+      ec2 = AWS::EC2.new
+      instance = ec2.instances[id]
+      instance.terminate
+    end
+
+    def self.get_ec2_instance_data(zone, id)
+      if ENV['DRYRUN']
+        return {
+          hostname: 'fake.hostname.gild.com'
+        }
+      end
+      configure_ec2 zone
+      ec2 = AWS::EC2.new
+      instance = ec2.instances[id]
+      return {
+        dns_name: instance.dns_name
+      }
+    end
+  end
+end

data/lib/helpers/init.rb

@@ -1,6 +1,4 @@
 # encoding: utf-8
-require_relative 'gaptool-base'
-GaptoolServer.helpers GaptoolBaseHelpers
-
+require_relative 'ec2'
 require_relative 'rehash'
-GaptoolServer.helpers RehashHelpers
+require_relative 'data'