gaptool-server 0.1.1 → 0.2.0

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.2.0

data/bin/gaptool-server

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 require 'rubygems'
 require 'thin'
-require 'gaptool-server/app.rb'
+#require 'gaptool-server/app.rb'
 
 # Hijack the thin module to set default options to run in the right place
 module Thin

data/config.ru

@@ -1,17 +1,24 @@
 #!/usr/bin/env ruby
 
-require 'sinatra/base'
-require 'sinatra'
-require 'json'
 require 'redis'
-require 'yaml'
-require 'erb'
-require 'aws-sdk'
-require 'openssl'
-require 'net/ssh'
 
-$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "lib")))
+ENV['REDIS_HOST'] = 'localhost' unless ENV['REDIS_HOST']
+ENV['REDIS_PORT'] = '6379' unless ENV['REDIS_PORT']
+ENV['REDIS_PASS'] = nil unless ENV['REDIS_PASS']
+$redis = Redis.new(:host => ENV['REDIS_HOST'], :port => ENV['REDIS_PORT'], :password => ENV['REDIS_PASS'])
 
-require 'gaptool-server/app.rb'
+libpath = File.expand_path(File.join(File.dirname(__FILE__), "lib"))
 
-run GaptoolServer
+$:.unshift(libpath)
+
+require "#{libpath}/app.rb"
+#Dir["#{ENV['HOME']}/.gaptool-server-plugins/*.rb"].each {|file| require file }
+#Dir["#{libpath}/plugins/*.rb"].each {|file| require file }
+
+instance = GaptoolServer.new
+#$redis.smembers("plugins").each do |plugin|
+##  puts "Loading Plugin #{plugin}"
+#  instance.extend(Object.instance_eval{remove_const(plugin)})
+#end
+
+run instance

data/gaptool-server.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "gaptool-server"
-  s.version = "0.1.1"
+  s.version = "0.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Matt Bailey"]
-  s.date = "2012-12-17"
+  s.date = "2013-01-05"
   s.description = "gaptool-server for managing cloud resources"
   s.email = "m@mdb.io"
   s.executables = ["gaptool-server"]
@@ -29,9 +29,19 @@ Gem::Specification.new do |s|
     "bin/gaptool-server",
     "config.ru",
     "gaptool-server.gemspec",
-    "lib/gaptool-server/app.rb",
-    "lib/gaptool-server/views/hosts.erb",
-    "lib/gaptool-server/views/init.erb",
+    "lib/app.rb",
+    "lib/helpers/gaptool-base.rb",
+    "lib/helpers/init.rb",
+    "lib/helpers/nicebytes.rb",
+    "lib/helpers/partials.rb",
+    "lib/models/init.rb",
+    "lib/models/user.rb",
+    "lib/public/css/common.css",
+    "lib/public/js/manifest.txt",
+    "lib/routes/init.rb",
+    "lib/routes/main.rb",
+    "lib/views/hosts.erb",
+    "lib/views/init.erb",
     "setup.rb",
     "test/helper.rb",
     "test/test_gaptool-server.rb"

data/lib/app.rb

@@ -0,0 +1,29 @@
+# encoding: utf-8
+require 'sinatra'
+require 'json'
+require 'yaml'
+require 'erb'
+require 'aws-sdk'
+require 'openssl'
+require 'net/ssh'
+require 'peach'
+
+class GaptoolServer < Sinatra::Application
+  enable :sessions
+  disable :show_exceptions
+  enable :raise_errors
+
+  before do
+    error 401 unless $redis.hget('users', env['HTTP_X_GAPTOOL_USER']) == env['HTTP_X_GAPTOOL_KEY']
+    error 401 unless env['HTTP_X_GAPTOOL_USER'] && env['HTTP_X_GAPTOOL_KEY']
+  end
+
+  helpers do
+    include Rack::Utils
+    alias_method :h, :escape_html
+  end
+end
+
+require_relative 'helpers/init'
+require_relative 'models/init'
+require_relative 'routes/init'

data/lib/helpers/gaptool-base.rb

@@ -0,0 +1,155 @@
+# encoding: utf-8
+module GaptoolBaseHelpers
+  def hash2redis( key, hash )
+    hash.keys.each do |hkey|
+      $redis.hset key, hkey, hash[hkey]
+    end
+  end
+
+  def putkey( host )
+    @key = OpenSSL::PKey::RSA.new 2048
+    @pubkey = "#{@key.ssh_type} #{[@key.to_blob].pack('m0')} GAPTOOL_GENERATED_KEY"
+    ENV['SSH_AUTH_SOCK'] = ''
+    Net::SSH.start(host, 'admin', :key_data => [$redis.hget('config', 'gaptoolkey')], :config => false, :keys_only => true, :paranoid => false) do |ssh|
+      ssh.exec! "grep -v GAPTOOL_GENERATED_KEY ~/.ssh/authorized_keys > /tmp/pubkeys"
+      ssh.exec! "cat /tmp/pubkeys > ~/.ssh/authorized_keys"
+      ssh.exec! "rm /tmp/pubkeys"
+      ssh.exec! "echo #{@pubkey} >> ~/.ssh/authorized_keys"
+    end
+    return @key.to_pem
+  end
+
+  def gt_securitygroup(role, environment, zone)
+    AWS.config(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{zone.chop}.amazonaws.com")
+    @ec2 = AWS::EC2.new
+    groupname = "#{role}-#{environment}"
+    default_list = [ 22 ]
+    @ec2.security_groups.each do |group|
+      if group.name == "#{role}-#{environment}"
+        return group.id
+      end
+    end
+    internet = ['0.0.0.0/0']
+    sg = @ec2.security_groups.create("#{role}-#{environment}")
+    sg.authorize_ingress :tcp, 22, *internet
+    return sg.id
+  end
+
+  def runservice(host, role, environment, service, keys, state)
+    ENV['SSH_AUTH_SOCK'] = ''
+    Net::SSH.start(host, 'admin', :key_data => [$redis.hget('config', 'gaptoolkey')], :config => false, :keys_only => true, :paranoid => false) do |ssh|
+      if state == 'start'
+        ssh.exec! "echo '#{keys.to_yaml}' > /tmp/apikeys-#{service}.yml"
+        ssh.exec! "sudo restart #{service} || sudo start #{service} || exit 0"
+        $redis.lpush("running", "{:hostname => '#{host}', :role => '#{role}', :environment => '#{environment}', :service => '#{service}'}")
+      elsif state == 'stop'
+        ssh.exec! "rm /tmp/apikeys-#{service}.yml"
+        ssh.exec! "sudo stop #{service} || exit 0"
+        $redis.lrem("running", -1, "{:hostname => '#{host}', :role => '#{role}', :environment => '#{environment}', service => '#{service}'}")
+      end
+    end
+  end
+
+  def balanceservices(role, environment)
+    @runable = Array.new
+    @available = Array.new
+    @totalcap = 0
+    @volume = 0
+    $redis.keys("host:#{role}:#{environment}:*").each do |host|
+      @available << {
+        :hostname => $redis.hget(host, 'hostname'),
+        :instance => $redis.hget(host, 'instance'),
+        :capacity => $redis.hget(host, 'capacity').to_i,
+      }
+      @totalcap = @totalcap + $redis.hget(host, 'capacity').to_i
+    end
+    $redis.keys("service:#{role}:#{environment}:*").each do |service|
+      unless service =~ /:count/
+        if $redis.hget(service, 'run').to_i == 1
+          @runable << {
+            :name => $redis.hget(service, 'name'),
+            :keys => eval($redis.hget(service, 'keys')),
+            :weight => $redis.hget(service, 'weight').to_i
+          }
+        end
+      end
+    end
+    @volume = 0
+    @runable.each do |service|
+      @volume += service[:weight]
+    end
+    if @totalcap < @volume
+      return { :error => true, :message => "This would overcommit, remove some resources or add nodes", :totalcap => @totalcap, :volume => @volume }
+    else
+      @runable.sort! { |x, y| x[:weight] <=> y[:weight] }
+      @available.sort! { |x, y| x[:capacity] <=> y[:capacity] }
+      @runlist = Array.new
+      @svctab = Hash.new
+      @runable.each do |event|
+        @svctab[event[:name]] = Array.new
+      end
+      @exitrunable = 0
+      while @runable != []
+        break if @exitrunable == 1
+        @available.each do |host|
+          break if @runable.last.nil?
+          @exitrunable = 1 if @svctab[@runable.last[:name]].include? host[:hostname]
+          break if @svctab[@runable.last[:name]].include? host[:hostname]
+          if host[:capacity] >= @runable.last[:weight]
+            host[:capacity] = host[:capacity] - @runable.last[:weight]
+            @svctab[@runable.last[:name]] << host[:hostname]
+            @runlist << { :host => host, :service => @runable.pop }
+          end
+        end
+      end
+      return @runlist
+    end
+  end
+
+  def servicestopall(role, environment)
+    $redis.lrange('running', 0, -1).peach do |service|
+      line = eval(service)
+      if line[:role] == role && line[:environment] == environment
+        runservice(line[:hostname], role, environment, line[:service], nil, 'stop')
+      end
+    end
+  end
+
+  def hostsgen(zone)
+    AWS.config(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{zone}.amazonaws.com")
+    @ec2 = AWS::EC2.new
+    hosts = Hash.new
+    $redis.keys("host:*").each do |host|
+      $redis.hset(host, 'hostname', @ec2.instances[$redis.hget(host, 'instance')].dns_name)
+      hosts.merge!(host.gsub(/host:/, '').gsub(/:/,'-') => Resolv.getaddress($redis.hget(host, 'hostname')))
+      if $redis.hget(host, 'alias')
+        hosts.merge!($redis.hget(host, 'alias') => Resolv.getaddress($redis.hget(host, 'hostname')))
+      end
+    end
+
+    hostsfile = "# DO NOT EDIT, GENERATED BY GAPTOOL\n127.0.0.1 localhost\n::1 localhost\n"
+    hosts.keys.each do |key|
+      hostsfile += "#{hosts[key]} #{key} # PLACED BY GT\n"
+    end
+#    $redis.keys("host:*").peach do |host|
+#      ENV['SSH_AUTH_SOCK'] = ''
+#      Net::SSH.start($redis.hget(host, 'hostname'), 'admin', :key_data => [$redis.hget('config', 'gaptoolkey')], :config => false, :keys_only => true, :paranoid => false) do |ssh|
+#        ssh.exec! "echo \"127.0.0.1 #{currenthost}\n\" \"#{hostsfile}\" > /etc/hosts.generated"
+#      end
+#    end
+    return hosts
+  end
+
+  def getservices()
+    services = Array.new
+    $redis.keys('service:*').each do |service|
+      unless service =~ /:count/
+        line = $redis.hgetall(service)
+        line['keys'] = eval(line['keys'])
+        services << line
+      end
+    end
+    return services
+  end
+
+end

data/lib/helpers/init.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+require_relative 'partials'
+GaptoolServer.helpers PartialPartials
+
+require_relative 'nicebytes'
+GaptoolServer.helpers NiceBytes
+
+require_relative 'gaptool-base'
+GaptoolServer.helpers GaptoolBaseHelpers

data/lib/helpers/nicebytes.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+module NiceBytes
+	K = 2.0**10
+	M = 2.0**20
+	G = 2.0**30
+	T = 2.0**40
+	def nice_bytes( bytes, max_digits=3 )
+		value, suffix, precision = case bytes
+			when 0...K
+				[ bytes, 'B', 0 ]
+			else
+				value, suffix = case bytes
+					when K...M then [ bytes / K, 'kiB' ]
+					when M...G then [ bytes / M, 'MiB' ]
+					when G...T then [ bytes / G, 'GiB' ]
+					else            [ bytes / T, 'TiB' ]
+				end
+				used_digits = case value
+					when   0...10   then 1
+					when  10...100  then 2
+					when 100...1000 then 3
+					else 4
+				end
+				leftover_digits = max_digits - used_digits
+				[ value, suffix, leftover_digits > 0 ? leftover_digits : 0 ]
+		end
+		"%.#{precision}f#{suffix}" % value
+	end
+	module_function :nice_bytes
+end

data/lib/helpers/partials.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+module PartialPartials
+	def spoof_request(uri,env_modifications={})
+		call(env.merge("PATH_INFO" => uri).merge(env_modifications)).last.join
+	end
+end

data/lib/models/init.rb

@@ -0,0 +1,6 @@
+# encoding: utf-8
+#require 'sequel'
+# DB = Sequel.postgres 'dbname', user:'bduser', password:'dbpass', host:'localhost'
+# DB << "SET CLIENT_ENCODING TO 'UTF8';"
+
+#require_relative 'user'

data/lib/models/user.rb

@@ -0,0 +1,4 @@
+# encoding: utf-8
+#class User < Sequel::Model
+	# ...
+#end

data/lib/public/css/common.css

@@ -0,0 +1,4 @@
+html, body { background:#eee; margin:0; padding:0; color:#000 }
+h1,h2,h3,h4,h5,h6 { font-family:sans-serif; border-bottom:1px solid #aaa; margin:2em 0 0.5em 0 }
+h1#title { background:#333; margin-top:0; color:#ccc; padding:0.1em 0.5em; font-size:105% }
+#content { margin:1em 2em }

data/lib/public/js/manifest.txt

@@ -0,0 +1 @@
+ 

data/lib/routes/init.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+require_relative 'main'

data/lib/routes/main.rb

@@ -0,0 +1,199 @@
+# encoding: utf-8
+class GaptoolServer < Sinatra::Application
+
+  get '/' do
+    "You must be lost. Read the instructions."
+  end
+
+  get '/servicebalance/:role/:environment' do
+    runlist = balanceservices(params[:role], params[:environment])
+    unless runlist.kind_of? Hash
+      servicestopall(params[:role], params[:environment])
+      runlist.peach do |event|
+        runservice(event[:host][:hostname], params[:role], params[:environment], event[:service][:name], event[:service][:keys], 'start')
+      end
+    end
+    runlist.to_json
+  end
+
+  put '/service/:role/:environment' do
+    data = JSON.parse request.body.read
+    count = $redis.incr("service:#{params[:role]}:#{params[:environment]}:#{data['name']}:count")
+    key = "service:#{params[:role]}:#{params[:environment]}:#{data['name']}:#{count}"
+    $redis.hset(key, 'name', data['name'])
+    $redis.hset(key, 'keys', data['keys'])
+    $redis.hset(key, 'weight', data['weight'])
+    $redis.hset(key, 'role', params[:role])
+    $redis.hset(key, 'environment', params[:environment])
+    $redis.hset(key, 'run', data['enabled'])
+    {
+      :role => params[:role],
+      :environment => params[:environment],
+      :service => data['name'],
+      :count => count,
+    }.to_json
+  end
+
+  delete '/service/:role/:environment/:service' do
+    if $redis.get("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:count") == '0'
+      count = 0
+    else
+      count = $redis.decr("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:count")
+      service = eval($redis.range("running", 0, -1).grep(/scoring/).last)
+      runservice(service[:hostname], params[:role], params[:environment], params[:service], 'stop')
+      $redis.del("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:#{count + 1}")
+    end
+    {
+      :role => params[:role],
+      :environment => params[:environment],
+      :service => params[:service],
+      :count => count,
+    }.to_json
+  end
+
+  get '/services' do
+    getservices().to_json
+  end
+
+  post '/regenhosts' do
+    data = JSON.parse request.body.read
+    hostsgen(data['zone'])
+    hosts.to_json
+  end
+
+  post '/init' do
+    data = JSON.parse request.body.read
+    AWS.config(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone'].chop}.amazonaws.com")
+    @ec2 = AWS::EC2.new
+    # create shared secret to reference in /register
+    @secret = (0...8).map{65.+(rand(26)).chr}.join
+    data.merge!("secret" => @secret)
+    sgid = gt_securitygroup(data['role'], data['environment'], data['zone'])
+    if data['mirror']
+      instance = @ec2.instances.create(
+        :image_id => $redis.hget("amis", data['zone'].chop),
+        :availability_zone => data['zone'],
+        :instance_type => data['itype'],
+        :key_name => "gaptool",
+        :security_group_ids => sgid,
+        :user_data => "#!/bin/bash\ncurl --silent -H 'X-GAPTOOL-USER: #{env['HTTP_X_GAPTOOL_USER']}' -H 'X-GAPTOOL-KEY: #{env['HTTP_X_GAPTOOL_KEY']}' #{$redis.hget('config', 'url')}/register -X PUT --data '#{data.to_json}' | bash",
+        :block_device_mappings => {
+          "/dev/sdf" => {
+            :volume_size => data['mirror'].to_i,
+            :delete_on_termination => false
+          },
+          "/dev/sdg" => {
+            :volume_size => data['mirror'].to_i,
+            :delete_on_termination => false
+          }
+        }
+      )
+    else
+      instance = @ec2.instances.create(
+        :image_id => $redis.hget("amis", data['zone'].chop),
+        :availability_zone => data['zone'],
+        :instance_type => data['itype'],
+        :key_name => "gaptool",
+        :security_group_ids => sgid,
+        :user_data => "#!/bin/bash\ncurl --silent -H 'X-GAPTOOL-USER: #{env['HTTP_X_GAPTOOL_USER']}' -H 'X-GAPTOOL-KEY: #{env['HTTP_X_GAPTOOL_KEY']}' #{$redis.hget('config', 'url')}/register -X PUT --data '#{data.to_json}' | bash"
+      )
+    end
+    # Add host tag
+    instance.add_tag('Name', :value => "#{data['role']}-#{data['environment']}-#{instance.id}")
+    # Create temporary redis entry for /register to pull the instance id
+    $redis.set("instance:#{data['role']}:#{data['environment']}:#{@secret}", instance.id)
+    "{\"instance\":\"#{instance.id}\"}"
+  end
+
+  post '/terminate' do
+    data = JSON.parse request.body.read
+    AWS.config(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone']}.amazonaws.com")
+    @ec2 = AWS::EC2.new
+    begin
+      @instance = @ec2.instances[data['id']]
+      res = @instance.terminate
+      res = $redis.del($redis.keys("*#{data['id']}"))
+      out = {data['id'] => {'status'=> 'terminated'}}
+    rescue
+      error 404
+    end
+    out.to_json
+  end
+
+  put '/register' do
+    data = JSON.parse request.body.read
+    AWS.config(:access_key_id => $redis.hget('config', 'aws_id'), :secret_access_key => $redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone'].chop}.amazonaws.com")
+    @ec2 = AWS::EC2.new
+    @instance = @ec2.instances[$redis.get("instance:#{data['role']}:#{data['environment']}:#{data['secret']}")]
+    hostname = @instance.dns_name
+    delete = $redis.del("instance:#{data['role']}:#{data['environment']}:#{data['secret']}")
+    @apps = Array.new
+    $redis.keys("app:*").each do |app|
+      if $redis.hget(app, 'role') == data['role']
+        @apps << app.gsub('app:', '')
+      end
+    end
+    data.merge!("capacity" => $redis.hget('capacity', data['itype']))
+    data.merge!("hostname" => hostname)
+    data.merge!("apps" => @apps.to_json)
+    data.merge!("instance" => @instance.id)
+    hash2redis("host:#{data['role']}:#{data['environment']}:#{@instance.id}", data)
+    @json = {
+      'hostname' => hostname,
+      'recipe' => 'init',
+      'number' => @instance.id,
+      'run_list' => ['recipe[init]'],
+      'role' => data['role'],
+      'environment' => data['environment'],
+      'chefrepo' => $redis.hget('config', 'chefrepo'),
+      'chefbranch' => $redis.hget('config', 'chefbranch'),
+      'identity' => $redis.hget('config','initkey'),
+      'appuser' => $redis.hget('config','appuser'),
+      'apps' => @apps
+    }.to_json
+    erb :init
+  end
+
+  get '/hosts' do
+    out = Array.new
+    $redis.keys("host:*").each do |host|
+      out << $redis.hgetall(host)
+    end
+    out.to_json
+  end
+
+  get '/apps' do
+    out = Hash.new
+    $redis.keys("app:*").each do |app|
+      out.merge!(app => $redis.hgetall(app))
+    end
+    out.to_json
+  end
+
+  get '/hosts/:role' do
+    out = Array.new
+    $redis.keys("host:#{params[:role]}:*").each do |host|
+      out << $redis.hgetall(host)
+    end
+    out.to_json
+  end
+
+  get '/hosts/:role/:environment' do
+    out = Array.new
+    $redis.keys("host:#{params[:role]}:#{params[:environment]}*").each do |host|
+      out << $redis.hgetall(host)
+    end
+    out.to_json
+  end
+
+  get '/host/:role/:environment/:instance' do
+    $redis.hgetall("host:#{params[:role]}:#{params[:environment]}:#{params[:instance]}").to_json
+  end
+
+  get '/ssh/:role/:environment/:instance' do
+    @host = $redis.hget("host:#{params[:role]}:#{params[:environment]}:#{params[:instance]}", 'hostname')
+    @key = putkey(@host)
+    {'hostname' => @host, 'key' => @key, 'pubkey' => @pubkey}.to_json
+  end
+
+end

data/lib/{gaptool-server/views → views}/hosts.erb

@@ -2,7 +2,7 @@
   <body>
     <ul>
     <% @hosts.each do |host|%>
-      <li><%=host%> <%=@redis.get host%></li>
+      <li><%=host%> <%=$redis.get host%></li>
     <% end %>
     </ul>
   </body>

data/lib/{gaptool-server/views → views}/init.erb

@@ -1,10 +1,10 @@
 apt-get install -y zsh git libssl-dev ruby1.9.1-full build-essential
 gem install --bindir /usr/local/bin --no-ri --no-rdoc chef
 cat << 'EOFKEY' > /root/.ssh/id_rsa
-<%=@redis.hget('config', 'initkey')%>
+<%=$redis.hget('config', 'initkey')%>
 EOFKEY
 chmod 600 /root/.ssh/id_rsa
 echo 'StrictHostKeyChecking no' > /root/.ssh/config
-git clone -b <%=@redis.hget('config', 'chefbranch')%> <%=@redis.hget('config', 'chefrepo')%> /root/ops
+git clone -b <%=$redis.hget('config', 'chefbranch')%> <%=$redis.hget('config', 'chefrepo')%> /root/ops
 echo '<%=@json%>' > /root/init.json
 chef-solo -c /root/ops/cookbooks/init.rb -j /root/init.json && (rm /root/.ssh/id_rsa; userdel -r ubuntu; rm -rf /root/.ssh; rm -rf /root/ops; rm -rf /root/init.json)

metadata

@@ -2,7 +2,7 @@
 name: gaptool-server
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors: 
 - Matt Bailey
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-12-17 00:00:00 Z
+date: 2013-01-05 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sinatra
@@ -176,9 +176,19 @@ files:
 - bin/gaptool-server
 - config.ru
 - gaptool-server.gemspec
-- lib/gaptool-server/app.rb
-- lib/gaptool-server/views/hosts.erb
-- lib/gaptool-server/views/init.erb
+- lib/app.rb
+- lib/helpers/gaptool-base.rb
+- lib/helpers/init.rb
+- lib/helpers/nicebytes.rb
+- lib/helpers/partials.rb
+- lib/models/init.rb
+- lib/models/user.rb
+- lib/public/css/common.css
+- lib/public/js/manifest.txt
+- lib/routes/init.rb
+- lib/routes/main.rb
+- lib/views/hosts.erb
+- lib/views/init.erb
 - setup.rb
 - test/helper.rb
 - test/test_gaptool-server.rb
@@ -195,7 +205,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: -3959515583740972889
+      hash: 1050799946438420370
       segments: 
       - 0
       version: "0"

data/lib/gaptool-server/app.rb

@@ -1,336 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'sinatra/base'
-require 'sinatra'
-require 'json'
-require 'redis'
-require 'yaml'
-require 'erb'
-require 'aws-sdk'
-require 'openssl'
-require 'net/ssh'
-require 'peach'
-
-ENV['REDIS_HOST'] = 'localhost' unless ENV['REDIS_HOST']
-ENV['REDIS_PORT'] = '6379' unless ENV['REDIS_PORT']
-ENV['REDIS_PASS'] = nil unless ENV['REDIS_PASS']
-
-class GaptoolServer < Sinatra::Base
-  # Don't generate fancy HTML for stack traces.
-  disable :show_exceptions
-  # Allow errors to get out of the app so Cucumber can display them.
-  enable :raise_errors
-
-  def hash2redis( key, hash )
-    hash.keys.each do |hkey|
-      @redis.hset key, hkey, hash[hkey]
-    end
-  end
-
-  before do
-    @redis = Redis.new(:host => ENV['REDIS_HOST'], :port => ENV['REDIS_PORT'], :password => ENV['REDIS_PASS'])
-    error 401 unless @redis.hget('users', env['HTTP_X_GAPTOOL_USER']) == env['HTTP_X_GAPTOOL_KEY']
-    error 401 unless env['HTTP_X_GAPTOOL_USER'] && env['HTTP_X_GAPTOOL_KEY']
-  end
-
-  def putkey( host )
-    @key = OpenSSL::PKey::RSA.new 2048
-    @pubkey = "#{@key.ssh_type} #{[@key.to_blob].pack('m0')} GAPTOOL_GENERATED_KEY"
-    ENV['SSH_AUTH_SOCK'] = ''
-    Net::SSH.start(host, 'admin', :key_data => [@redis.hget('config', 'gaptoolkey')], :config => false, :keys_only => true, :paranoid => false) do |ssh|
-      ssh.exec! "grep -v GAPTOOL_GENERATED_KEY ~/.ssh/authorized_keys > /tmp/pubkeys"
-      ssh.exec! "cat /tmp/pubkeys > ~/.ssh/authorized_keys"
-      ssh.exec! "rm /tmp/pubkeys"
-      ssh.exec! "echo #{@pubkey} >> ~/.ssh/authorized_keys"
-    end
-    return @key.to_pem
-  end
-
-  def gt_securitygroup(role, environment, zone)
-    AWS.config(:access_key_id => @redis.hget('config', 'aws_id'), :secret_access_key => @redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{zone.chop}.amazonaws.com")
-    @ec2 = AWS::EC2.new
-    groupname = "#{role}-#{environment}"
-    default_list = [ 22 ]
-    @ec2.security_groups.each do |group|
-      if group.name == "#{role}-#{environment}"
-        return group.id
-      end
-    end
-    internet = ['0.0.0.0/0']
-    sg = @ec2.security_groups.create("#{role}-#{environment}")
-    sg.authorize_ingress :tcp, 22, *internet
-    return sg.id
-  end
-
-  def runservice(host, service, keys)
-    ENV['SSH_AUTH_SOCK'] = ''
-    Net::SSH.start(host, 'admin', :key_data => [@redis.hget('config', 'gaptoolkey')], :config => false, :keys_only => true, :paranoid => false) do |ssh|
-      ssh.exec! "echo '#{keys.to_yaml}' > /tmp/apikeys-#{service}.yml"
-      ssh.exec! "sudo restart #{service} || sudo start #{service} || exit 0"
-    end
-  end
-
-  def balanceservices(role, environment)
-    @runable = Array.new
-    @available = Array.new
-    @totalcap = 0
-    @volume = 0
-    @redis.keys("host:#{role}:#{environment}:*").each do |host|
-      @available << {
-        :hostname => @redis.hget(host, 'hostname'),
-        :instance => @redis.hget(host, 'instance'),
-        :capacity => @redis.hget(host, 'capacity').to_i,
-      }
-      @totalcap = @totalcap + @redis.hget(host, 'capacity').to_i
-    end
-    @redis.keys("service:#{role}:#{environment}:*").each do |service|
-      unless service =~ /:count/
-        if @redis.hget(service, 'run').to_i == 1
-          @runable << {
-            :name => @redis.hget(service, 'name'),
-            :keys => eval(@redis.hget(service, 'keys')),
-            :weight => @redis.hget(service, 'weight').to_i
-          }
-        end
-      end
-    end
-    @volume = 0
-    @runable.each do |service|
-      @volume += service[:weight]
-    end
-    if @totalcap < @volume
-      return {'error' => true,"message" => "This would overcommit, remove some resources or add nodes","totalcap" => @totalcap, "volume" => @volume}.to_json
-    else
-      @runable.sort! { |x, y| x[:weight] <=> y[:weight] }
-      @available.sort! { |x, y| x[:capacity] <=> y[:capacity] }
-      @runlist = Array.new
-      @svctab = Hash.new
-      @runable.each do |event|
-        @svctab[event[:name]] = Array.new
-      end
-      @exitrunable = 0
-      while @runable != []
-        break if @exitrunable == 1
-        @available.each do |host|
-          break if @runable.last.nil?
-          @exitrunable = 1 if @svctab[@runable.last[:name]].include? host[:hostname]
-          break if @svctab[@runable.last[:name]].include? host[:hostname]
-          if host[:capacity] >= @runable.last[:weight]
-            host[:capacity] = host[:capacity] - @runable.last[:weight]
-            @svctab[@runable.last[:name]] << host[:hostname]
-            @runlist << { :host => host, :service => @runable.pop }
-          end
-        end
-      end
-      return @runlist
-    end
-  end
-
-  get '/' do
-    "You must be lost. Read the instructions."
-  end
-
-  get '/servicebalance/:role/:environment' do
-    runlist = balanceservices(params[:role], params[:environment])
-    runlist.peach do |event|
-      runservice(event[:host][:hostname], event[:service][:name], event[:service][:keys])
-    end
-    runlist.to_json
-  end
-
-  put '/service/:role/:environment' do
-    data = JSON.parse request.body.read
-    count = @redis.incr("service:#{params[:role]}:#{params[:environment]}:#{data['name']}:count")
-    key = "service:#{params[:role]}:#{params[:environment]}:#{data['name']}:#{count}"
-    @redis.hset(key, 'name', data['name'])
-    @redis.hset(key, 'keys', data['keys'])
-    @redis.hset(key, 'weight', data['weight'])
-    @redis.hset(key, 'role', params[:role])
-    @redis.hset(key, 'environment', params[:environment])
-    @redis.hset(key, 'run', data['enabled'])
-    {
-      :role => params[:role],
-      :environment => params[:environment],
-      :service => data['name'],
-      :count => count,
-    }.to_json
-  end
-
-  delete '/service/:role/:environment/:service' do
-    if @redis.get("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:count") == '0'
-      count = 0
-    else
-      count = @redis.decr("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:count")
-      @redis.del("service:#{params[:role]}:#{params[:environment]}:#{params[:service]}:#{count + 1}")
-    end
-    {
-      :role => params[:role],
-      :environment => params[:environment],
-      :service => params[:service],
-      :count => count,
-    }.to_json
-  end
-
-  def getservices()
-    services = Array.new
-    @redis.keys('service:*').each do |service|
-      unless service =~ /:count/
-        line = @redis.hgetall(service)
-        line['keys'] = eval(line['keys'])
-        services << line
-      end
-    end
-    return services
-  end
-
-  get '/services' do
-    getservices().to_json
-  end
-
-  post '/regenhosts' do
-    data = JSON.parse request.body.read
-    AWS.config(:access_key_id => @redis.hget('config', 'aws_id'), :secret_access_key => @redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone']}.amazonaws.com")
-    @ec2 = AWS::EC2.new
-    @redis.keys("host:*").each do |host|
-      out = @redis.hset(host, 'hostname', @ec2.instances[@redis.hget(host, 'instance')].dns_name)
-    end
-    "{\"regen\":\"running\"}"
-  end
-
-  post '/init' do
-    data = JSON.parse request.body.read
-    AWS.config(:access_key_id => @redis.hget('config', 'aws_id'), :secret_access_key => @redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone'].chop}.amazonaws.com")
-    @ec2 = AWS::EC2.new
-    # create shared secret to reference in /register
-    @secret = (0...8).map{65.+(rand(26)).chr}.join
-    data.merge!("secret" => @secret)
-    sgid = gt_securitygroup(data['role'], data['environment'], data['zone'])
-    if data['mirror']
-      instance = @ec2.instances.create(
-        :image_id => @redis.hget("amis", data['zone'].chop),
-        :availability_zone => data['zone'],
-        :instance_type => data['itype'],
-        :key_name => "gaptool",
-        :security_group_ids => sgid,
-        :user_data => "#!/bin/bash\ncurl --silent -H 'X-GAPTOOL-USER: #{env['HTTP_X_GAPTOOL_USER']}' -H 'X-GAPTOOL-KEY: #{env['HTTP_X_GAPTOOL_KEY']}' #{@redis.hget('config', 'url')}/register -X PUT --data '#{data.to_json}' | bash",
-        :block_device_mappings => {
-          "/dev/sdf" => {
-            :volume_size => data['mirror'].to_i,
-            :delete_on_termination => false
-          },
-          "/dev/sdg" => {
-            :volume_size => data['mirror'].to_i,
-            :delete_on_termination => false
-          }
-        }
-      )
-    else
-      instance = @ec2.instances.create(
-        :image_id => @redis.hget("amis", data['zone'].chop),
-        :availability_zone => data['zone'],
-        :instance_type => data['itype'],
-        :key_name => "gaptool",
-        :security_group_ids => sgid,
-        :user_data => "#!/bin/bash\ncurl --silent -H 'X-GAPTOOL-USER: #{env['HTTP_X_GAPTOOL_USER']}' -H 'X-GAPTOOL-KEY: #{env['HTTP_X_GAPTOOL_KEY']}' #{@redis.hget('config', 'url')}/register -X PUT --data '#{data.to_json}' | bash"
-      )
-    end
-    # Add host tag
-    instance.add_tag('Name', :value => "#{data['role']}-#{data['environment']}-#{instance.id}")
-    # Create temporary redis entry for /register to pull the instance id
-    @redis.set("instance:#{data['role']}:#{data['environment']}:#{@secret}", instance.id)
-    "{\"instance\":\"#{instance.id}\"}"
-  end
-
-  post '/terminate' do
-    data = JSON.parse request.body.read
-    AWS.config(:access_key_id => @redis.hget('config', 'aws_id'), :secret_access_key => @redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone']}.amazonaws.com")
-    @ec2 = AWS::EC2.new
-    begin
-      @instance = @ec2.instances[data['id']]
-      res = @instance.terminate
-      res = @redis.del(@redis.keys("*#{data['id']}"))
-      out = {data['id'] => {'status'=> 'terminated'}}
-    rescue
-      error 404
-    end
-    out.to_json
-  end
-
-  put '/register' do
-    data = JSON.parse request.body.read
-    AWS.config(:access_key_id => @redis.hget('config', 'aws_id'), :secret_access_key => @redis.hget('config', 'aws_secret'), :ec2_endpoint => "ec2.#{data['zone'].chop}.amazonaws.com")
-    @ec2 = AWS::EC2.new
-    @instance = @ec2.instances[@redis.get("instance:#{data['role']}:#{data['environment']}:#{data['secret']}")]
-    hostname = @instance.dns_name
-    delete = @redis.del("instance:#{data['role']}:#{data['environment']}:#{data['secret']}")
-    @apps = Array.new
-    @redis.keys("app:*").each do |app|
-      if @redis.hget(app, 'role') == data['role']
-        @apps << app.gsub('app:', '')
-      end
-    end
-    data.merge!("capacity" => @redis.hget('capacity', data['itype']))
-    data.merge!("hostname" => hostname)
-    data.merge!("apps" => @apps.to_json)
-    data.merge!("instance" => @instance.id)
-    hash2redis("host:#{data['role']}:#{data['environment']}:#{@instance.id}", data)
-    @json = {
-      'hostname' => hostname,
-      'recipe' => 'init',
-      'number' => @instance.id,
-      'run_list' => ['recipe[init]'],
-      'role' => data['role'],
-      'environment' => data['environment'],
-      'chefrepo' => @redis.hget('config', 'chefrepo'),
-      'chefbranch' => @redis.hget('config', 'chefbranch'),
-      'identity' => @redis.hget('config','initkey'),
-      'appuser' => @redis.hget('config','appuser'),
-      'apps' => @apps
-    }.to_json
-    erb :init
-  end
-
-  get '/hosts' do
-    out = Array.new
-    @redis.keys("host:*").each do |host|
-      out << @redis.hgetall(host)
-    end
-    out.to_json
-  end
-
-  get '/apps' do
-    out = Hash.new
-    @redis.keys("app:*").each do |app|
-      out.merge!(app => @redis.hgetall(app))
-    end
-    out.to_json
-  end
-
-  get '/hosts/:role' do
-    out = Array.new
-    @redis.keys("host:#{params[:role]}:*").each do |host|
-      out << @redis.hgetall(host)
-    end
-    out.to_json
-  end
-
-  get '/hosts/:role/:environment' do
-    out = Array.new
-    @redis.keys("host:#{params[:role]}:#{params[:environment]}*").each do |host|
-      out << @redis.hgetall(host)
-    end
-    out.to_json
-  end
-
-  get '/host/:role/:environment/:instance' do
-    @redis.hgetall("host:#{params[:role]}:#{params[:environment]}:#{params[:instance]}").to_json
-  end
-
-  get '/ssh/:role/:environment/:instance' do
-    @host = @redis.hget("host:#{params[:role]}:#{params[:environment]}:#{params[:instance]}", 'hostname')
-    @key = putkey(@host)
-    {'hostname' => @host, 'key' => @key, 'pubkey' => @pubkey}.to_json
-  end
-
-
-end