judo 0.3.6 → 0.3.7

data/Rakefile

@@ -8,7 +8,7 @@ Jeweler::Tasks.new do |s|
 	s.email = "orion@heroku.com"
 	s.homepage = "http://github.com/orionz/judo"
 	s.rubyforge_project = "judo"
-	s.files = FileList["[A-Z]*", "{bin,lib,spec}/**/*"]
+	s.files = FileList["[A-Z]*", "{bin,default,lib,spec}/**/*"]
 	s.executables = %w(judo)
 	s.add_dependency "aws",  [">= 2.3.8"]
 	s.add_dependency "json"

data/TODO

@@ -1,6 +1,5 @@
 ### NEEDED for new gem launch
 
-### [ ] ubuntu 10.04
 ### [ ] snapshot percent complete in do_snapshots()
 ### [ ] judo swap (x) (y) -> swaps elastic IP's and name
 ### [ ] snapshots/init/virgin

data/VERSION

@@ -1 +1 @@
-0.3.6
+0.3.7

data/bin/judo

@@ -10,9 +10,7 @@ defaults = Judo::Base.default_options(Dir.pwd)
 
 options = {}
 
-action = ARGV.shift
-
-ARGV.unshift "-h" unless action
+ARGV.unshift "-h" if ARGV.size == 0
 
 optparse = OptionParser.new do|opts|
   opts.banner = <<banner
@@ -70,14 +68,16 @@ banner
   opts.on( '-r', '--repo DIR', 'Specify the location of the repo dir' ) do |dir|
     options[:repo] = dir
   end
-  opts.on( '-D', '--data DATA', 'Specify an instance specific string to be used during boot on create or launch' ) do |data|
-    options[:data] = data
-  end
   opts.on( '-t', '--type TYPE', 'Specify an instance type different from the config on server start' ) do |type|
     options[:instance_type] = type
   end
-  opts.on( '-B', '--boot BOOT', 'Instance specific string (like --data) but for this boot only' ) do |data|
-    options[:boot] = data
+  opts.on( '-D', '--data KEY=VALUE', 'Specify metadata to attach to a server at creation time' ) do |data|
+    key,value = data.split("=")
+    (options[:metadata] ||= {})[key] = value
+  end
+  opts.on( '-B', '--boot KEY=VALUE', 'Specify special boot parameters' ) do |data|
+    key,value = data.split("=")
+    (options[:boot] ||= {})[key] = value
   end
   opts.on( '-i', '--ip IP', 'Specify an exsiting elastic_ip address on server creation' ) do |ip|
     options[:elastic_ip] = ip
@@ -104,9 +104,6 @@ banner
   opts.on( '-g', '--group GROUP', 'Specify the default group of the repo dir' ) do |group|
     options[:group] = group
   end
-  opts.on( '-n', '--note NOTE', 'Add a note to a server as you create or launch it') do |note|
-    options[:note] = note
-  end
   opts.on( '-v', '--version VERSION', 'Update the servers config version on create/start/launch' ) do |version|
     options[:version] = version
   end
@@ -118,6 +115,7 @@ end
 
 optparse.parse!
 
+action = ARGV.shift
 judo = Judo::Base.new(defaults.merge(options))
 judo.check_version if action != "setup"
 
@@ -141,7 +139,7 @@ begin
     when "snapshots" then do_snapshots(judo, ARGV)
     when "config" then
       puts "Judo DB Version: #{judo.db_version}"
-      puts "Judo Keypair:    #{judo.key_name}"
+      puts "Judo Key Pair:   #{judo.key_name}"
       puts "Judo Bucket:     #{judo.bucket_name}"
     when "rename" then
       raise JudoError, "usage: judo rename SERVER SERVER" unless ARGV.size == 2
@@ -174,7 +172,7 @@ begin
       snapshot.animate(new_server).start(options)
     when "destroy" then
       raise JudoError, "You must specify what servers to destroy" if ARGV.empty?
-      find_either(judo, ARGV) do |i|
+      find_servers(judo, ARGV) do |i|
         i.destroy
       end
     else

data/default/config.json

@@ -0,0 +1,8 @@
+{
+  "ami32":"ami-2d4aa444", // public ubuntu 10.04 ami - 32 bit
+  "ami64":"ami-fd4aa494", // public ubuntu 10.04 ami - 64 bit
+  "user":"ubuntu",        // this is for ssh acccess - defaults to ubuntu not root
+  "security_group":"judo",
+  "my_data": "world world",
+  "availability_zone":"us-east-1d"
+}

data/default/setup.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+

data/default/userdata.erb

@@ -0,0 +1,37 @@
+#!/bin/bash
+<% 
+
+## the variables available for use in the erb are as follows
+
+##   id - this is random string that uniqily identifies the server for its lifetime
+##   name - the name of the server
+##   group_name - the name of group the server is in
+##   domain - this is the JUDO_DOMAIN currently running
+##   secret - this is a 128bit random string assigned to an instance at creation time - great for passwords
+##   metadata - a hash of metadata associated with the server with --data KEY=VALUE
+##   boot - a hash of boot options passed to the server with --boot KEY=VALUE
+##   first_boot? - is set to true or false depending on if this is the instance's first boot
+##   url - the URL to the group verson file in the s3 bucket
+
+%>
+
+export JUDO_ID='<%= id %>'
+export JUDO_NAME='<%= name %>'
+export JUDO_DOMAIN='<%= domain %>'
+export JUDO_SECRET='<%= secret %>'
+export JUDO_FIRST_BOOT='<%= first_boot? %>'
+
+export DEBIAN_FRONTEND="noninteractive"
+export DEBIAN_PRIORITY="critical"
+apt-get update
+apt-get install -y ruby rubygems ruby-dev irb libopenssl-ruby libreadline-ruby
+
+cd /tmp/ ; curl --silent '<%= url %>' | tar xvz ; cd <%= group_name %>
+
+gem install kuzushi --no-rdoc --no-ri --version
+echo 'export PATH=`ruby -r rubygems -e "puts Gem.bindir"`:$PATH' >> /etc/profile ; . /etc/profile
+export LOG=/var/log/kuzushi.log
+
+kuzushi-setup >> $LOG  ## this will properly mount / configure and formal all ebs drives
+bash setup.sh >> $LOG
+

data/lib/judo.rb

@@ -1,4 +1,5 @@
 require 'rubygems'
+require 'erb'
 require 'active_support'
 require 'right_aws'
 require 'socket'

data/lib/judo/base.rb

@@ -1,27 +1,3 @@
-
-=begin
-module Aws
-  class Ec2
-    API_VERSION       = "2009-11-30"  ## this didnt work
-    def start_instances(instance_id)
-      link = generate_request("StartInstances",  { 'InstanceId' => instance_id } )
-      request_info(link, QEc2TerminateInstancesParser.new(:logger => @logger))
-    rescue Exception
-      on_exception
-    end
-
-    def stop_instances(instance_id)
-      link = generate_request("StopInstances",  { 'InstanceId' => instance_id } )
-      puts link.inspect
-      result = request_info(link, QEc2TerminateInstancesParser.new(:logger => @logger))
-      puts result.inspect
-    rescue Exception
-      on_exception
-    end
-  end
-end
-=end
-
 module Judo
   class Base
     attr_accessor :judo_dir, :repo, :group, :domain
@@ -162,9 +138,17 @@ module Judo
       rand(2**32).to_s(36)
     end
 
+    def default_group_dir
+        File.expand_path(File.dirname(__FILE__) + "/../../default")
+    end
+
+    def default_file(name)
+      "#{default_group_dir}/#{name}"
+    end
+
     def mk_server_name(group)
         index = servers.map { |s| (s.name =~ /^#{s.group.name}.(\d*)$/); $1.to_i }.sort.last.to_i + 1
-        "#{group.name}.#{index}"
+        "#{group}.#{index}"
     end
 
     def create_server(name, group, options)
@@ -452,9 +436,8 @@ module Judo
     def default_config
         <<DEFAULT
 {
-  "instance_type":"m1.small",
-  "ami32":"ami-bb709dd2", // public ubuntu 9.10 ami - 32 bit
-  "ami64":"ami-55739e3c", // public ubuntu 9.10 ami - 64 bit
+  "ami32":"ami-2d4aa444", // public ubuntu 10.04 ami - 32 bit
+  "ami64":"ami-fd4aa494", // public ubuntu 10.04 ami - 64 bit
   "user":"ubuntu",
   "security_group":"judo",
   "availability_zone":"us-east-1d"

data/lib/judo/commandline_helpers.rb

@@ -46,19 +46,6 @@ module JudoCommandLineHelpers
     servers.flatten.each { |s| judo_yield(s, blk) if blk }
   end
 
-  def find_either(judo, args, &blk)
-    results = []
-    args.each do |arg|
-      server,group = split(arg)
-      if server != ""
-        results << judo.servers.select { |s| s.name == server }
-      else
-        results << judo.groups.select { |g| g.name == group }
-      end
-    end
-    results.flatten.each { |i| judo_yield(i, blk) if blk }
-  end
-
   def find_servers(judo, args, use_default = true, &blk)
     servers = judo.servers if args.empty?
     servers ||= args.map { |a| find_server(judo, a, use_default) }.flatten
@@ -161,6 +148,9 @@ module JudoCommandLineHelpers
         printf "    %-24s: %s\n",k, i[k]
       end
     end
+    sub_info("METADATA", server.metadata.keys) do |key|
+      printf("    %-24s: %s\n", key, server.metadata[key] )
+    end
     sub_info("VOLUMES", server.ec2_volumes) do |v|
       printf "    %-13s %-10s %-10s %4d  %-10s %-8s\n",
       v[:aws_id],

data/lib/judo/group.rb

@@ -8,12 +8,12 @@ module Judo
       @version = version
     end
 
-    def config
-      @config ||= load_config
+    def userdata(version)
+      (@userdata ||= {})[version] ||= load_userdata(version)
     end
 
-    def build_config
-      @build_config ||= all_configs.reverse.inject({}) { |sum,conf| sum.merge(conf) }
+    def config(version)
+      (@config ||= {})[version] ||= load_config(version)
     end
 
     def server_ids
@@ -24,8 +24,14 @@ module Judo
       @base.servers.select { |s| server_ids.include?(s.name) }
     end
 
-    def load_config
-        JSON.load @base.s3_get(version_config_file)
+    def load_userdata(version)
+        @base.s3_get(version_userdata_file(version))
+      rescue Aws::AwsError
+       raise JudoError, "No userdata stored: try 'judo commit #{to_s}'"
+    end
+
+    def load_config(version)
+        JSON.load @base.s3_get(version_config_file(version))
       rescue Aws::AwsError
        raise JudoError, "No config stored: try 'judo commit #{to_s}'"
     end
@@ -35,41 +41,34 @@ module Judo
     end
 
     def compile
-      tmpdir = "/tmp/kuzushi/#{name}"
       @base.task("Compiling #{self} version #{version + 1}") do
         @version = @version + 1
-        FileUtils.rm_rf(tmpdir)
-        FileUtils.mkdir_p(tmpdir)
-        new_config = build_config
-        Dir.chdir(tmpdir) do |d|
-          attachments(new_config).each do |to,from|
-            FileUtils.mkdir_p(File.dirname(to))
-            if from =~ /^http:\/\//
-              # puts "curl '#{from}'"
-              system "curl '#{from}' > #{to}"
-              # puts "#{to} is #{File.stat(to).size} bytes"
-            else
-              FileUtils.cp(from,to)
-            end
-          end
-          File.open("config.json", "w") { |f| f.write(new_config.to_json) }
-          Dir.chdir("..") do
+        raise JudoError, "can not find group folder #{dir}" unless File.exists?(dir)
+        conf = JSON.parse(read_file('config.json'))
+        raise JudoError, "config option 'import' no longer supported" if conf["import"]
+        Dir.chdir(@base.repo) do |d|
             puts ""
             system "tar czvf #{tar_file} #{name}"
             puts "Uploading config to s3..."
-            @base.s3_put(version_config_file, new_config.to_json)
+            @base.s3_put(version_config_file(@version), conf.to_json)
+            puts "Uploading userdata.erb to s3..."
+            @base.s3_put(version_userdata_file(@version), read_file('userdata.erb'))
             puts "Uploading tar file to s3..."
             @base.s3_put(tar_file, File.new(tar_file).read(File.stat(tar_file).size))
-          end
+            File.delete(tar_file)
         end
         set_version
       end
     end
 
-    def version_config_file
+    def version_config_file(version)
       "#{name}.#{version}.json"
     end
 
+    def version_userdata_file(version)
+      "#{name}.#{version}.erb"
+    end
+
     def tar_file
       "#{name}.#{version}.tar.gz"
     end
@@ -78,66 +77,6 @@ module Judo
       @url = @base.s3_url(tar_file)
     end
 
-    def cp_file
-      FileUtil.mkdir_p(tmpdir)
-    end
-
-    def extract_file(type, name, files)
-      path = "#{dir}/#{type}s/#{name}"
-      found = Dir[path]
-      if not found.empty?
-        found.each { |f| files["#{type}s/#{File.basename(f)}"] = f }
-      elsif parent
-        parent.extract_file(type, name, files)
-      else
-        raise "Cannot find file #{name} of type #{type}"
-      end
-    end
-
-    def extract(config, files)
-      config.each do |key,value|
-        [value].flatten.each do |v|   ### cover "packages" : ["a","b"], "packages" : "a", "packages":[{ "file" : "foo.pkg"}]
-          if v.is_a? Hash
-            extract(v, files)
-          else
-            case key
-            when *[ "init", "before", "after" ]
-              extract_file(:script, v, files) unless v =~ /^#!/
-            when "package"
-              files["packages/#{v}_i386.deb"]  = "#{config["source"]}#{v}_i386.deb"
-              files["packages/#{v}_amd64.deb"] = "#{config["source"]}#{v}_amd64.deb"
-            when "local_packages"
-              extract_file(:package, "#{v}_i386.deb", files)
-              extract_file(:package, "#{v}_amd64.deb", files)
-            when "git"
-              ## do nothing - is a remote
-            when "template"
-              extract_file(:template, v, files)
-            when "source"
-              extract_file(:file, v, files) unless config["template"] or config["package"]
-            when "file"
-              ## this is getting messy
-              extract_file(:file, File.basename(v), files) unless config["template"] or config["source"] or config["git"]
-            end
-          end
-        end
-      end
-      files
-    end
-
-    def attachments(c = config)
-      extract(c, {})
-    end
-
-    def raw_config
-      @raw_config ||= read_config
-    end
-
-    def parent
-      @parent ||= @base.groups.detect { |p| p.name == raw_config["import"] }
-      (@parent ||= Judo::Group.new(@base, raw_config["import"], 0 )) if raw_config["import"]  ## make a mock incase the parent was never committed
-      @parent
-    end
 
     def destroy
       servers.each { |s| s.destroy }
@@ -148,22 +87,20 @@ module Judo
       end
     end
 
-    def all_configs
-      parent ? parent.all_configs.clone.unshift(raw_config) : [ raw_config ]
-    end
-
     def dir
       "#{@base.repo}/#{name}/"
     end
 
-    def config_file
-      "#{dir}/config.json"
+    def default_userdata_file
+        File.expand_path(File.dirname(__FILE__) + "/../../default/userdata.erb")
     end
 
-    def read_config
-        JSON.parse(File.read(config_file))
+    def read_file(name)
+        File.read("#{dir}/#{name}")
       rescue Errno::ENOENT
-        raise JudoError, "No config file #{config_file}"
+        default = @base.default_file(name)
+        puts "File #{name} not found: using #{default} instead"
+        File.read default
     end
 
     def delete_server(server)

data/lib/judo/server.rb

@@ -1,11 +1,12 @@
 module Judo
   class Server
-    attr_accessor :id,:group_name
+    attr_accessor :id
 
     def initialize(base, id, group, version = nil)
       @base = base
       @id = id
-      @group_name = group
+      @group = group
+      @judo_domain = base.domain
     end
 
     def create(name, options)
@@ -14,8 +15,7 @@ module Judo
       options[:virgin] = true if options[:virgin].nil?
 
       snapshots  = options[:snapshots]
-      note       = options[:note]        ## a user defined note field
-      data       = options[:data]        ## instance specific data passed in JUDO_DATA
+      metadata   = options[:metadata]
       ip         = options[:elastic_ip]  ## if the ip was allocated beforehand
       virgin     = options[:virgin]      ## should the server init?
       clone      = options[:clone]       ## if it was cloned from a snapshot
@@ -27,11 +27,11 @@ module Judo
       raise JudoError, "there is already a server with id #{id}" if @base.servers.detect { |s| s.id == id and s != self}
 
       task("Creating server #{name}") do
-        update "name" => name,         "group" => group_name,
-               "note" => note,         "virgin" => virgin,
+        update("name" => name,         "group" => group_name,
+               "virgin" => virgin,     "elastic_ip" => ip,
                "secret" => new_secret, "version" => version,
-               "data" => data,         "elastic_ip" => ip,
-               "clone" => clone,       "created_at" => Time.now.to_i
+               "clone" => clone,       "created_at" => Time.now.to_i)
+        set_metadata(metadata) if metadata
         @base.sdb.put_attributes(@base.base_domain, "groups", group_name => id)
       end
 
@@ -41,8 +41,37 @@ module Judo
       self
     end
 
+    def set_metadata(new_metadata)
+      puts new_metadata.inspect
+      clean_metadata = new_metadata.inject({}) { |buf,(k,v)| 
+        puts [ buf, k , v ].inspect
+        buf[k.to_s] = v.to_s; buf }
+      puts clean_metadata.inspect
+      keys_to_remove = clean_metadata.keys - clean_metadata.keys
+      @metadata = clean_metadata
+      update encode_metadata(clean_metadata) unless clean_metadata.empty?
+      keys_to_remove.each { |key| remove key }
+    end
+
+    def encode_metadata(data)
+      data.inject({}) { |buf,(key,value)| buf["metadata_#{key.to_s}"] = [value].to_json; buf }
+    end
+
+    def metadata
+      @metadata ||= get_metadata
+    end
+
+    def get_metadata
+      state.inject({}) do |buf,(key,value)|
+        if key =~ /^metadata_(.*)/
+          buf[$1] = JSON.load(value.first)
+        end
+        buf
+      end
+    end
+
     def group
-      @group ||= @base.groups.detect { |g| g.name == group_name }
+      @group_obj ||= @base.groups.detect { |g| g.name == @group }
     end
 
     def fetch_state
@@ -53,10 +82,6 @@ module Judo
       @base.servers_state[id] ||= fetch_state
     end
 
-    def get(key)
-      state[key] && [state[key]].flatten.first
-    end
-
     def created_at
       Time.at(get("created_at").to_i)
     end
@@ -125,6 +150,10 @@ module Judo
       !!clone
     end
 
+    def first_boot?
+      virgin?
+    end
+
     def virgin?
       get("virgin").to_s == "true"  ## I'm going to set it to true and it will come back from the db as "true" -> could be "false" or false or nil also
     end
@@ -141,6 +170,12 @@ module Judo
       (state["volumes"] || []).inject({}) { |out, kv| k, v = kv.split(':'); out[k] = v; out }
     end
 
+######## begin simple DB access  #######
+
+    def get(key)
+      state[key] && [state[key]].flatten.first
+    end
+
     def update(attrs)
       @base.sdb.put_attributes(@base.server_domain, id, attrs, :replace)
       state.merge! attrs
@@ -170,11 +205,11 @@ module Judo
 
     def instance_type
      ## need instance_size to be backward compatible - needed for configs made before v 0.3.0
-     get("instance_type") || config["instance_type"] || config["instance_size"]
+     get("instance_type") || config["instance_type"] || config["instance_size"] || "m1.small"
     end
 
     def config
-      group.config
+      group.config(version)
     end
 
     def to_s
@@ -295,6 +330,7 @@ module Judo
     end
 
     def start(options = {})
+      @boot= options[:boot]
       new_version = options[:version]
       set_instance_type(options[:instance_type]) if options[:instance_type]
 
@@ -302,7 +338,7 @@ module Judo
       invalid "No config has been commited yet, type 'judo commit'" unless group.version > 0
 
       task("Updating server version")      { update_version(options[:version]) } if options[:version]
-      task("Starting server #{name}")      { launch_ec2(options[:boot]) }
+      task("Starting server #{name}")      { launch_ec2 }
       task("Wait for server")              { wait_for_running } if elastic_ip or has_volumes?
       task("Attaching ip")                 { attach_ip } if elastic_ip
       task("Attaching volumes")            { attach_volumes } if has_volumes?
@@ -344,11 +380,11 @@ module Judo
       update "stopped_at" => Time.now.to_i
     end
 
-    def launch_ec2(boot = nil)
+    def launch_ec2
 #      validate
 
       ## EC2 launch_instances
-      ud = user_data(boot)
+      ud = user_data
       debug(ud)
       result = @base.ec2.launch_instances(ami,
         :instance_type => instance_type,
@@ -361,7 +397,9 @@ module Judo
 
     def debug(str)
       return unless ENV['JUDO_DEBUG'] == "1"
-      puts "<JUDO_DEBUG>#{str}</JUDO_DEBUG>"
+      puts "<DEBUG>"
+      puts str
+      puts "</DEBUG>"
     end
 
     def security_groups
@@ -516,29 +554,18 @@ module Judo
       @base.servers_state.delete(id)
     end
 
-    def user_data(judo_boot = nil)
-      <<USER_DATA
-#!/bin/sh
-
-export DEBIAN_FRONTEND="noninteractive"
-export DEBIAN_PRIORITY="critical"
-export JUDO_ID='#{id}'
-export JUDO_NAME='#{name}'
-export JUDO_DOMAIN='#{@base.domain}'
-export JUDO_BOOT='#{judo_boot}'
-export JUDO_DATA='#{data}'
-export SECRET='#{secret}'
-apt-get update
-apt-get install ruby rubygems ruby-dev irb libopenssl-ruby libreadline-ruby -y
-gem install kuzushi --no-rdoc --no-ri
-GEM_BIN=`ruby -r rubygems -e "puts Gem.bindir"`
-echo "$GEM_BIN/kuzushi #{virgin? && "init" || "start"} '#{url}'" > /var/log/kuzushi.log
-$GEM_BIN/kuzushi #{virgin? && "init" || "start"} '#{url}' >> /var/log/kuzushi.log 2>&1
-USER_DATA
+    def user_data
+      erb = group.userdata(version)
+      debug erb
+      ERB.new(erb, 0, '<>').result(binding)
     end
 
     def url
-      @url ||= group.s3_url
+      group.s3_url
+    end
+
+    def domain
+      @base.domain
     end
 
     def validate
@@ -596,12 +623,20 @@ USER_DATA
     end
 
     def <=>(s)
-      [group.name, name] <=> [s.group.name, s.name]
+      [group.to_s, name.to_s] <=> [s.group.to_s, s.name.to_s]
     end
 
     def new_secret
       rand(2 ** 128).to_s(36)
     end
 
+    def group_name
+      @group
+    end
+
+    def boot
+      @boot || {}
+    end
+
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 3
-  - 6
-  version: 0.3.6
+  - 7
+  version: 0.3.7
 platform: ruby
 authors: 
 - Orion Henry
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-05-24 00:00:00 -04:00
+date: 2010-05-31 00:00:00 -04:00
 default_executable: judo
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -58,6 +58,9 @@ files:
 - TODO
 - VERSION
 - bin/judo
+- default/config.json
+- default/setup.sh
+- default/userdata.erb
 - lib/judo.rb
 - lib/judo/base.rb
 - lib/judo/commandline_helpers.rb