gv-valley 0.0.1

data/lib/gv/valley/addon.rb

@@ -0,0 +1,62 @@
+require 'gv/bedrock/service'
+require 'gv/bedrock/config'
+require 'gv/common/docker_helper'
+require 'gv/common/host_helper'
+
+module GV
+  module Valley
+  
+    ##
+    # Addon Service
+    #
+  
+    class Addon < GV::Bedrock::Service
+      
+      include GV::Common::HostHelper      
+      include GV::Common::DockerHelper      
+      
+      PORT = nil
+      
+      attr_reader :image, :params, :cmd
+      
+      def initialize
+        super
+        
+        pull_image_if_does_not_exists self.image
+        
+        @home = GV::Bedrock::Config.service.get("home")
+        @name ||= File.basename(self.image) 
+        
+      end
+      
+      def create app_name
+        @app_name = app_name
+        addon_name = "#{@name}.#{app_name}"
+        return nil if ps? addon_name
+        pipe "docker run --name #{addon_name} -d -p #{self.external_ip}::#{self.class::PORT} -e PORT=#{self.class::PORT} #{self.params} #{self.image} #{self.cmd}"
+      end
+      
+      def destroy app_name
+        @app_name = app_name
+        addon_name = "#{@name}.#{app_name}"
+        batch addon_name, "stop", true
+        batch addon_name, "rm", true        
+      end
+      
+      def info app_name
+        @app_name = app_name
+        addon_name = "#{@name}.#{app_name}"
+        info(container_id(addon_name))
+      end
+      
+      def port app_name
+        @app_name = app_name
+        addon_name = "#{@name}.#{app_name}"
+        container_port addon_name, self.external_ip, self.class::PORT
+      end
+      
+      
+    end
+    
+  end
+end

data/lib/gv/valley/app.rb

@@ -0,0 +1,108 @@
+require 'gv/bedrock/config'
+require 'gv/valley/etcd'
+require 'gv/valley/file_server'
+require 'gv/common/docker_helper'
+require 'json'
+require 'etcd'
+
+module GV
+  module Valley
+    
+    class App
+      
+      PORT = 5000
+      
+      class << self
+        
+        def etcd_clt
+          @@etcd_clt ||= begin
+            server = GV::Valley::Etcd.service  
+            ::Etcd.client(config = {
+              host: server.external_ip, 
+              port: server.port
+            })
+          end
+        end
+        
+        def all
+          etcd_clt.get("/apps").children.map{ |node| find(File.basename(node.key)) }
+        end
+        
+        def find name
+          begin
+            find! name
+          rescue ::Etcd::KeyNotFound
+            nil
+          end
+        end
+        
+        def find! name
+          value = etcd_clt.get("/apps/#{name}").value
+          new(ensure_defaults(JSON.load(value))) 
+        end        
+        
+        def create name
+          save name, {"name" => name, "config" => default_config(name), "ps" => {}, "domains" => [default_domain(name)]}
+          find name
+        end
+        
+        def save name, data
+          etcd_clt.set("/apps/#{name}", { value: JSON.dump(ensure_defaults(data)) })
+        end
+        
+        def delete name
+          etcd_clt.delete("/apps/#{name}",{recursive: true}) rescue nil
+        end
+        
+        def domain
+          GV::Bedrock::Config.service.get("domain")
+        end
+        
+        def default_config name
+          config = {
+            "SLUG_URL" => "#{GV::Valley::FileServer.service.url}/#{name}/slug.tgz",
+            "PORT" => PORT
+          }
+        end
+        
+        def default_domain name
+          "#{name}.#{self.domain}"
+        end
+        
+        def ensure_defaults data
+          data["config"] ||= {}
+          data["config"].update(default_config(data["name"]))
+          
+          data["domains"] ||= []
+          data["domains"] << default_domain(data["name"]) unless data["domains"].include? default_domain(data["name"])
+          data
+        end
+        
+      end
+      
+      def initialize attributes = {}
+        @attributes = attributes
+        @name = @attributes['name']
+        @attributes = attributes
+      end
+      
+      def save
+        self.class.save(@name,@attributes)
+      end
+      
+      def delete
+        self.class.delete @name
+      end
+      
+      def [](key)
+        @attributes[key]
+      end
+      
+      def []=(key,value)
+        @attributes[key]=value
+      end  
+      
+    end
+    
+  end
+end

data/lib/gv/valley/balancer.rb

@@ -0,0 +1,61 @@
+require 'gv/bedrock/service'
+require 'gv/valley/app'
+require 'gv/common/pipe_helper'
+
+module GV
+  module Valley
+    
+    class Balancer < GV::Bedrock::Service
+      
+      include GV::Common::PipeHelper
+      INDENT = "        "      
+      
+      ##
+      # reloads haproxy config
+      
+      def reload &block
+        
+        @block = block
+        
+        indicate "Loading Haproxy config"
+        
+        config = File.read("#{GV::Valley.root}/scripts/haproxy.cfg")
+        target_file = "/etc/haproxy/haproxy.cfg"
+        acl = ""
+        backend = ""
+        
+        App.all.each do |app|
+          
+          app["domains"].each do |domain|
+            acl << "#{INDENT}use_backend b_#{app["name"]} if { hdr(host) -i #{domain} }\n"
+          end
+          
+          backend << "backend b_#{app["name"]}\n"
+          app["ps"]["web"]["containers"].each do |container|
+            host = container['HostConfig']['PortBindings']["#{App::PORT}/tcp"].first
+            backend << "#{INDENT}server srv_#{container['ID'][0..6]} #{host['HostIp']}:#{host['HostPort']}\n"
+          end
+          
+        end
+        
+        config.gsub!(/^\#FRONT$/,acl) 
+        config.gsub!(/^\#BACK$/,backend) 
+        
+        pipe "rm #{target_file}"
+        File.open(target_file,File::RDWR|File::CREAT){|f| f.puts config }
+    
+        pipe "chmod 0770 #{target_file}"
+        pipe "chgrp haproxy #{target_file}"
+        pipe "service haproxy reload", &block
+        
+      end
+      
+      private
+        
+      
+        
+    end 
+      
+  end
+    
+end

data/lib/gv/valley/deployer.rb

@@ -0,0 +1,104 @@
+require 'yaml'
+require 'gv/bedrock/service'
+require 'gv/valley/app'
+require 'gv/valley/runner'
+require 'gv/common/pipe_helper'
+
+
+module GV
+  module Valley
+    
+    class Deployer < GV::Bedrock::Service
+      
+      include GV::Common::PipeHelper
+      
+      ##
+      # deploys app
+      
+      def deploy name, &block
+        
+        # find or create app
+        unless app = App.find(name)
+          app = App.create(name)
+        end
+        
+        # set block for output helpers
+        @block = block
+          
+        indicate "Deploying App"
+        
+        # read procfile  
+        host = GV::Valley::Runner.random_service
+        procfile = host.run(app["name"], "cat /app/Procfile")
+        procfile_types = YAML.load(procfile).keys
+        
+        stop app
+          
+        # add new Procfile process types or reset jobs array for existing types
+        procfile_types.each do |type|
+          unless app["ps"].keys.include? type
+            app["ps"][type] = {"scale" => 1, "containers" => []}
+          else
+            app["ps"][type]["containers"] = []
+          end
+        end
+        
+        app.save
+          
+        # remove the old types
+        app["ps"].keys.each do |type|
+          unless procfile_types.include? type
+            app["ps"].delete(type)
+          end
+        end
+        
+        app.save
+        
+        start app
+          
+        app.save
+          
+      end
+      
+      ##
+      # stops all running procfile processes
+      
+      def stop app, &block
+        # stop and remove all running procfile processes
+        tuple = [:ps, /#{app['name']}\./, nil, nil ]
+        while (self.class.space.read(tuple,0) rescue nil) do
+          if host = (self.class.space.take(tuple,0)[2] rescue nil)
+            host.remove app["name"]
+          end
+        end
+      end
+      
+      ##
+      # starts all procfile processes
+            
+      def start app, &block
+        # run available process types
+        app["ps"].each do |type,ps|
+          ps["scale"].times do |index|
+            host = GV::Valley::Runner.random_service                
+            app["ps"][type]["containers"] << host.start(app["name"], type, index, &block)
+          end
+        end
+      end
+      
+      private
+      
+      def indicate string
+        say %(-----> #{string}), &@block
+      end
+
+      def say string
+        pipe %(echo '\e[1G#{string}'), &@block      
+      end 
+        
+        
+    end 
+      
+  end
+    
+end

data/lib/gv/valley/etcd.rb

@@ -0,0 +1,40 @@
+require 'gv/bedrock/service'
+require 'gv/bedrock/config'
+require 'gv/common/docker_helper'
+require 'gv/common/host_helper'
+
+module GV
+  module Valley
+  
+    ##
+    # Etcd Service
+    #
+  
+    class Etcd < GV::Bedrock::Service
+      
+      include GV::Common::HostHelper      
+      include GV::Common::DockerHelper      
+      
+      PORT = 4001
+      
+      def initialize
+        super
+        
+        pull_image_if_does_not_exists "flynn/etcd"
+        
+        home = GV::Bedrock::Config.service.get("home")
+        
+        unless ps? 'etcd'
+          cleanup
+          pipe "docker run --name etcd -d -p #{self.external_ip}::#{PORT} -v #{home}/etcd:/data/db:rw flynn/etcd --name=greenvalley -data-dir=/data/db"
+        end
+      end
+      
+      def port
+        container_port 'etcd', self.external_ip, PORT
+      end
+      
+    end
+    
+  end
+end

data/lib/gv/valley/file_server.rb

@@ -0,0 +1,115 @@
+require 'gv/bedrock/service'
+require 'gv/bedrock/config'
+require 'gv/common/host_helper'
+require 'goliath/api'
+require 'goliath/runner'
+require 'uri' 
+
+module GV
+  module Valley
+    
+    class FileServer < GV::Bedrock::Service
+      
+      include GV::Common::HostHelper
+      
+      class FileSystem
+        CHUNKSIZE = 65536
+
+        def initialize(path)
+          @path  = path
+        end
+
+        def get
+          open(@path, 'rb') do |file|
+            yield file.read(CHUNKSIZE) until file.eof?
+          end
+        end
+  
+      end
+      
+      class Api < Goliath::API
+  
+        use Goliath::Rack::DefaultMimeType
+        use Goliath::Rack::Render, 'json'
+        use Goliath::Rack::Heartbeat
+
+        use Goliath::Rack::Validation::RequestMethod, %w(GET PUT DELETE)
+
+        def on_headers(env, headers)
+          env['async-headers'] = headers
+        end
+
+        def on_body(env, data)
+          (env['async-body'] ||= '') << data
+        end
+
+        def response(env)
+    
+          path = "#{ENV['GV_HOME']}/#{env['REQUEST_PATH']}"
+    
+          case env['REQUEST_METHOD']
+      
+          when 'GET'
+    
+            headers = {'X-filename' => path}
+
+            raise Goliath::Validation::NotFoundError unless File.file?(path)
+
+            operation = proc do
+              FileSystem.new(path).get { |chunk| env.chunked_stream_send(chunk) }
+            end
+
+            callback = proc do |result|
+              env.chunked_stream_close
+            end
+
+            EM.defer operation, callback
+
+            headers.merge!( 'X-Stream' => 'Goliath')
+            chunked_streaming_response(200, headers)
+      
+          when 'PUT'
+            
+            File.delete(path) rescue nil
+            result = File.open(path, File::RDWR|File::CREAT){ |f| f.puts env['async-body'] }
+            [ 200, {}, {body: "OK"} ]        
+    
+          when 'DELETE'  
+            result = File.delete(path)
+            [ 200, {}, {body: result } ]                
+          end
+        end
+
+      end 
+            
+      def url
+        "http://#{self.external_ip}:#{self.port}"
+      end
+    
+      def port
+        ENV['PORT'] ||= '9000'
+      end
+  
+      def initialize
+        super
+        ENV['GV_HOME'] ||= GV::Bedrock::Config.service.get("home")
+        runner = Goliath::Runner.new(ARGV, nil)
+        runner.api = Api.new
+        runner.app = Goliath::Rack::Builder.build(Api, runner.api)
+        runner.port = self.port
+        runner.log_file = "/var/log/gv-file_server.log"
+        runner.pid_file = "/var/run/gv-file_server.pid"        
+        runner.daemonize = true
+        runner.run
+        at_exit { 
+          pid = File.read("/var/run/gv-file_server.pid").chomp.to_i
+          Process.kill("TERM",pid)
+          File.delete("/var/run/gv-file_server.pid")          
+          File.delete("/var/log/gv-file_server.log")
+          File.delete("/var/log/gv-file_server.log_stdout.log")          
+        }
+      end
+      
+    end
+  end
+end

data/lib/gv/valley/runner.rb

@@ -0,0 +1,138 @@
+require 'gv/bedrock/service'
+require 'gv/common/host_helper'
+require 'gv/common/docker_helper'
+require 'gv/valley/app'
+require 'yaml'
+
+module GV
+  module Valley
+    
+    class Runner < GV::Bedrock::Service
+      
+      include GV::Common::HostHelper
+      include GV::Common::DockerHelper
+      
+      def initialize
+        super
+        pull_image_if_does_not_exists "flynn/slugrunner"
+        register_all
+      end
+      
+      
+      ##
+      # runs one-off job
+      
+      def run name, cmd, &block
+        app = App.find!(name)  
+                      
+        cleanup
+ 
+        params =  %(--rm -a stdout -a stderr ) #note that space at the end!
+        params << %(-i -a stdin) if cmd =~ Sticks::Pipe::INTERACTIVE_COMMANDS
+        cmd    =  %(docker run --name=#{app[:name]}.run.#{genuuid()} #{params} #{getenv(app)} flynn/slugrunner #{cmd})
+
+        debug "Runner#run name:#{name}, cmd:#{cmd}"     
+    
+        result = pipe cmd, &block
+        
+        sleep 2
+        
+        cleanup
+        result
+      end 
+      
+      
+      ##
+      # force removes all matching jobs
+      
+      def remove name, type = nil, &block
+        debug "Runner#remove name:#{name}, type:#{type}"     
+        
+        batch "#{name}.#{type}", "kill", true
+        batch "#{name}.#{type}", "rm", true   
+        unregister_all(/#{name}\.#{type}\./)
+      end 
+      
+      
+      ##
+      # starts process
+      
+      def start name, type, index, &block
+        raise "AppNotFound" unless app = App.find(name) 
+        
+        info "Starting #{name}.#{type}.#{index}"
+        
+        cleanup
+    
+        params = %(-d -p #{self.external_ip}::#{App::PORT})
+        cmd = %(docker run --name=#{name}.#{type}.#{index} #{params} #{getenv(app)} flynn/slugrunner start #{type})
+        
+        debug "Runner#start name:#{name}, type:#{type} index:#{index}"     
+                
+        container_id = pipe(cmd)
+        
+        if container_id =~ /^[a-zA-Z0-9]{64}$/
+          
+          register "#{name}.#{type}.#{index}"
+          info(container_id)
+          
+        elsif container_id.include? "Error: Conflict"
+          error "Container name conflict: #{name}.#{type}.#{index}, docker service reloading..." 
+    
+          restart_docker!
+
+          remove name, type
+          start name, type, index, &block
+        else
+          raise container_id
+        end
+        
+      end 
+      
+      
+      ## 
+      # retrives logs
+      
+      def logs name, follow = false, &block
+        debug "Runner#logs name:#{name}, follow:#{follow}"     
+        
+        pipe "docker logs #{follow ? "-f" : nil} #{container_id(name)}", &block
+      end
+      
+      
+      private
+      
+      def getenv app
+        app['config'].map{ |k,v| "-e #{k}=#{v}" }.join(" ")
+      end
+      
+      def genuuid
+        rand(2**64).to_s(36) # from heroku/slugcompiler
+      end 
+
+      def register_all
+        pipe("docker ps | grep runner/init").chomp.split("\n").each{|l| 
+          cols = l.split(/\s{3,}/)
+          name = cols.last
+          unregister_all(/#{name}/)
+          register name
+        }
+      end
+      
+      def unregister_all name
+        unregister name while (self.class.space.read([:ps, name, nil, self.external_ip],0) rescue nil)
+      end
+      
+      def register name
+        self.class.space.write [:ps, name, self, self.external_ip ]
+      end
+      
+      def unregister name
+        self.class.space.take([:ps, name, nil, self.external_ip],0) rescue nil
+      end
+       
+      
+    end
+    
+  end
+end

data/lib/gv/valley/version.rb

@@ -0,0 +1,5 @@
+module GV
+  module Valley
+    VERSION = "0.0.1"
+  end
+end

data/lib/gv/valley.rb

@@ -0,0 +1,10 @@
+require "gv/common/logging"
+require "gv/valley/version"
+
+module GV
+  module Valley
+    def self.root
+      @@root ||= File.expand_path("../../../",__FILE__)
+    end
+  end
+end

data/scripts/haproxy.cfg

@@ -0,0 +1,47 @@
+global
+        log /dev/log        local0
+        log /dev/log        local1 notice
+        chroot /var/lib/haproxy
+        user haproxy
+        group haproxy
+        daemon
+
+defaults
+        log global
+        mode http
+        
+        contimeout 5000
+        clitimeout 50000
+        srvtimeout 50000
+        
+        option httplog
+        option dontlognull
+        option redispatch
+        option abortonclose
+        option httpclose
+        option forwardfor
+                
+        stats enable
+        stats auth admin:password
+        stats uri /monitor
+        stats refresh 5s
+        retries 5
+
+        balance roundrobin
+        
+        errorfile 400 /etc/haproxy/errors/400.http
+        errorfile 403 /etc/haproxy/errors/403.http
+        errorfile 408 /etc/haproxy/errors/408.http
+        errorfile 500 /etc/haproxy/errors/500.http
+        errorfile 502 /etc/haproxy/errors/502.http
+        errorfile 503 /etc/haproxy/errors/503.http
+        errorfile 504 /etc/haproxy/errors/504.http        
+
+
+frontend http
+        bind *:80
+        monitor-uri /haproxy
+
+#FRONT
+
+#BACK