ironfan 3.1.5 → 3.1.6

data/lib/ironfan/discovery.rb

@@ -170,6 +170,19 @@ module Ironfan
     @fog_keypairs = {}.tap{|hsh| Ironfan.fog_connection.key_pairs.each{|kp| hsh[kp.name] = kp } }
   end
 
+  def self.dry_run?
+    Ironfan.chef_config[:dry_run]
+  end
+
+  def self.placement_groups
+    return @placement_groups if @placement_groups
+    Chef::Log.debug("Using fog to catalog all placement_groups")
+    resp = self.fog_connection.describe_placement_groups unless dry_run?
+    return {} unless resp.respond_to?(:body) && resp.body.present?
+    arr = resp.body['placementGroupSet']
+    @placement_groups = arr.inject({}){|acc, pg| acc[pg['groupName']] = pg ; acc }
+  end
+
   def safely *args, &block
     Ironfan.safely(*args, &block)
   end

data/lib/ironfan/fog_layer.rb

@@ -24,9 +24,11 @@ module Ironfan
         if client_key.body then cloud.user_data.merge({ :client_key     => client_key.body })
         else                    cloud.user_data.merge({ :validation_key => cloud.validation_key }) ; end
       #
-      {
+      description = {
         :image_id             => cloud.image_id,
         :flavor_id            => cloud.flavor,
+        :vpc_id               => cloud.vpc,
+        :subnet_id            => cloud.subnet,
         :groups               => cloud.security_groups.keys,
         :key_name             => cloud.keypair.to_s,
         # Fog does not actually create tags when it creates a server.
@@ -38,15 +40,20 @@ module Ironfan
         :block_device_mapping => block_device_mapping,
         :availability_zone    => self.default_availability_zone,
         :monitoring           => cloud.monitoring,
-        # :disable_api_termination => cloud.permanent,
-        # :instance_initiated_shutdown_behavior => instance_initiated_shutdown_behavior,
+        # permanence is applied during sync
       }
+      if needs_placement_group?
+        ui.warn "1.3.1 and earlier versions of Fog don't correctly support placement groups, so your nodes will land willy-nilly. We're working on a fix"
+        description[:placement] = { 'groupName' => cloud.placement_group.to_s }
+      end
+      description
     end
 
     #
     # Takes key-value pairs and idempotently sets those tags on the cloud machine
     #
     def fog_create_tags(fog_obj, desc, tags)
+      tags['Name'] ||= tags['name'] if tags.has_key?('name')
       tags_to_create = tags.reject{|key, val| fog_obj.tags[key] == val.to_s }
       return if tags_to_create.empty?
       step("  tagging #{desc} with #{tags_to_create.inspect}", :green)
@@ -107,6 +114,23 @@ module Ironfan
       end
     end
 
+    def ensure_placement_group
+      return unless needs_placement_group?
+      pg_name = cloud.placement_group.to_s
+      desc = "placement group #{pg_name} for #{self.fullname} (vs #{Ironfan.placement_groups.inspect}"
+      return if Ironfan.placement_groups.include?(pg_name)
+      safely do
+        step("  creating #{desc}", :blue)
+        unless_dry_run{ Ironfan.fog_connection.create_placement_group(pg_name, 'cluster') }
+        Ironfan.placement_groups[pg_name] = { 'groupName' => pg_name, 'strategy' => 'cluster' }
+      end
+      pg_name
+    end
+
+    def needs_placement_group?
+      cloud.flavor_info[:placement_groupable]
+    end
+
     def associate_public_ip
       address = self.cloud.public_ip
       return unless self.in_cloud? && address
@@ -130,6 +154,21 @@ module Ironfan
       end
     end
 
+    def set_instance_attributes
+      return unless self.in_cloud? && (not self.cloud.permanent.nil?)
+      desc = "termination flag #{permanent?} for #{self.fullname}"
+      # the EC2 API does not surface disable_api_termination as a value, so we
+      # have to set it every time.
+      safely do
+        step("  setting #{desc}", :blue)
+        unless_dry_run do
+          Ironfan.fog_connection.modify_instance_attribute(self.fog_server.id, {
+              'DisableApiTermination.Value' => permanent?, })
+        end
+        true
+      end
+    end
+
   end
 
   class ServerSlice

data/lib/ironfan/security_group.rb

@@ -28,7 +28,10 @@ module Ironfan
       def self.get_all
         groups_list = Ironfan.fog_connection.security_groups.all
         @@all = groups_list.inject(Mash.new) do |hsh, fog_group|
-          hsh[fog_group.name] = fog_group ; hsh
+          # AWS security_groups are strangely case sensitive, allowing upper-case but colliding regardless
+          #  of the case. This forces all names to lowercase, and matches against that below.
+          #  See https://github.com/infochimps-labs/ironfan/pull/86 for more details.
+          hsh[fog_group.name.downcase] = fog_group ; hsh
         end
       end
 
@@ -37,6 +40,7 @@ module Ironfan
       end
 
       def self.get_or_create(group_name, description)
+        group_name = group_name.to_s.downcase
         # FIXME: the '|| Ironfan.fog' part is probably unnecessary
         fog_group = all[group_name] || Ironfan.fog_connection.security_groups.get(group_name)
         unless fog_group
@@ -89,7 +93,7 @@ module Ironfan
           step("authorizing access from all machines in #{other_name} to #{name}", :blue)
           self.class.get_or_create(other_name, "Authorized to access #{name}")
           begin  fog_group.authorize_group_and_owner(other_name, authed_owner)
-          rescue StandardError => e ; ui.warn e ; end
+          rescue StandardError => err ; handle_security_group_error(err) ; end
         end
         @group_authorized_by.uniq.each do |other_name|
           authed_owner = self.owner_id
@@ -97,13 +101,21 @@ module Ironfan
           next if group_permission_already_set?(other_group, self.name, authed_owner)
           step("authorizing access to all machines in #{other_name} from #{name}", :blue)
           begin  other_group.authorize_group_and_owner(self.name, authed_owner)
-          rescue StandardError => e ; ui.warn e ; end
+          rescue StandardError => err ; handle_security_group_error(err) ; end
         end
         @range_authorizations.uniq.each do |range, cidr_ip, ip_protocol|
           next if range_permission_already_set?(fog_group, range, cidr_ip, ip_protocol)
           step("opening #{ip_protocol} ports #{range} to #{cidr_ip}", :blue)
           begin  fog_group.authorize_port_range(range, { :cidr_ip => cidr_ip, :ip_protocol => ip_protocol })
-          rescue StandardError => e ; ui.warn e ; end
+          rescue StandardError => err ; handle_security_group_error(err) ; end
+        end
+      end
+
+      def handle_security_group_error(err)
+        if (/has already been authorized/ =~ err.to_s)
+          Chef::Log.debug err
+        else
+          ui.warn(err)
         end
       end
 

data/lib/ironfan/server.rb

@@ -84,7 +84,12 @@ module Ironfan
       in_chef?
     end
 
+    def permanent?
+      !! self.cloud.permanent
+    end
+
     def killable?
+      return false if permanent?
       in_chef? || created?
     end
 
@@ -101,6 +106,10 @@ module Ironfan
       @tags[key]
     end
 
+    def public_hostname
+      give_me_a_hostname_from_one_of_these_seven_ways_you_assholes
+    end
+
     def chef_server_url()        Chef::Config.chef_server_url        ; end
     def validation_client_name() Chef::Config.validation_client_name ; end
     def validation_key()         Chef::Config.validation_key         ; end
@@ -235,6 +244,8 @@ module Ironfan
       attach_volumes
       create_tags
       associate_public_ip
+      ensure_placement_group
+      set_instance_attributes
     end
 
     def sync_to_chef
@@ -280,5 +291,23 @@ module Ironfan
       chef_node.save
     end
 
+  protected
+
+    def give_me_a_hostname_from_one_of_these_seven_ways_you_assholes
+      # note: there are not actually seven ways. That is the least absurd part of this situation.
+      case
+      when cloud.public_ip
+        cloud.public_ip
+      when fog_server && fog_server.respond_to?(:public_ip_address) && fog_server.public_ip_address.present?
+        fog_server.public_ip_address
+      when fog_server && fog_server.respond_to?(:ipaddress)
+        fog_server.ipaddress
+      when fog_server && fog_server.respond_to?(:dns_name)
+        fog_server.dns_name
+      else
+        nil
+      end
+    end
+
   end
 end

data/lib/ironfan.rb

@@ -29,7 +29,7 @@ module Ironfan
 
   # path to search for cluster definition files
   def self.cluster_path
-    return Chef::Config[:cluster_path] if Chef::Config[:cluster_path]
+    return Array(Chef::Config[:cluster_path]) if Chef::Config[:cluster_path]
     raise "Holy smokes, you have no cookbook_path or cluster_path set up. Follow chef's directions for creating a knife.rb." if Chef::Config[:cookbook_path].blank?
     cl_path = Chef::Config[:cookbook_path].map{|dir| File.expand_path('../clusters', dir) }.uniq
     ui.warn "No cluster path set. Taking a wild guess that #{cl_path.inspect} is \nreasonable based on your cookbook_path -- but please set cluster_path in your knife.rb"

data/notes/INSTALL-cloud_setup.md

@@ -4,20 +4,20 @@
   - copy the knife/example-credentials directory
   - best to not live on github: use a private server and run
    
-    ``` 
-    repo=ORGANIZATION-credentials ; repodir=/gitrepos/$repo.git ; mkdir -p $repodir ; ( GIT_DIR=$repodir git init --shared=group --bare  && cd $repodir && git --bare update-server-info && chmod a+x hooks/post-update ) 
-    ```
+        ``` 
+        repo=ORGANIZATION-credentials ; repodir=/gitrepos/$repo.git ; mkdir -p $repodir ; ( GIT_DIR=$repodir git init --shared=group --bare  && cd $repodir && git --bare update-server-info && chmod a+x hooks/post-update ) 
+        ```
     
   - git submodule it into knife as `knife/yourorg-credentials`
   - or, if somebody has added it,
   
-    ```
-    git pull
-    git submodule update --init
-    find . -iname '*.pem' -exec chmod og-rw {} \;
-    cp knife/${OLD_CHEF_ORGANIZATION}-credentials/knife-user-${CHEF_USER}.rb knife/${CHEF_ORGANIZATION}-credentials
-    cp knife/${OLD_CHEF_ORGANIZATION}-credentials/${CHEF_USER}.pem knife/${CHEF_ORGANIZATION}-credentials/
-    ```
+        ```
+        git pull
+        git submodule update --init
+        find . -iname '*.pem' -exec chmod og-rw {} \;
+        cp knife/${OLD_CHEF_ORGANIZATION}-credentials/knife-user-${CHEF_USER}.rb knife/${CHEF_ORGANIZATION}-credentials
+        cp knife/${OLD_CHEF_ORGANIZATION}-credentials/${CHEF_USER}.pem knife/${CHEF_ORGANIZATION}-credentials/
+        ```
     
 * create AWS account
   - [sign up for AWS + credit card + password]
@@ -30,14 +30,17 @@
 ## Populate Chef Server
 
 * create `prod` and `dev` environments by using 
+
   ```
   knife environment create dev
   knife environment create prod
+  knife environment create stag
+  knife environment from file environments/stag.json  
   knife environment from file environments/dev.json
   knife environment from file environments/prod.json
   ```
 
-  ```ruby
+  ```
   knife cookbook upload --all
   rake roles 
   # if you have data bags, do that too

data/notes/INSTALL.md

@@ -15,6 +15,10 @@ In all of the below,
 
 _Before you begin, you may wish to fork homebase repo, as you'll be making changes to personalize it for your platform that you may want to share with teammates. If you do so, replace all references to infochimps-labs/ironfan-homebase with your fork's path._
 
+1. Install system prerequisites (libXML and libXSLT). The following works under Debian/Ubuntu:
+
+        sudo apt-get install libxml2-dev libxslt1-dev
+
 1. Install the Ironfan gem (you may need to use `sudo`):
 
         gem install ironfan
@@ -23,6 +27,7 @@ _Before you begin, you may wish to fork homebase repo, as you'll be making chang
 
         git clone https://github.com/infochimps-labs/ironfan-homebase homebase
         cd homebase
+        bundle install
         git submodule update --init
         git submodule foreach git checkout master
 
@@ -126,4 +131,4 @@ The README file in each of the subdirectories for more information about what go
 
         knife cluster launch el_ridiculoso-gordo --bootstrap
 
-For more information about configuring Knife, see the [Knife documentation](http://wiki.opscode.com/display/chef/knife).
+For more information about configuring Knife, see the [Knife documentation](http://wiki.opscode.com/display/chef/knife).

data/notes/core_concepts.md

@@ -22,7 +22,7 @@
 
 * [Systems *Bind* to provisioned resources](#binding)
 
-* [Binding declarations enable *Reasource Sharing*](#resource-sharing)
+* [Binding declarations enable *Resource Sharing*](#resource-sharing)
 
 <a name="overview"></a>
 ### Overview

data/notes/ec2-pricing_and_capacity.md

@@ -1,29 +1,35 @@
 ## Compute Costs
 
-    code    	    $/mo	$/day	$/hr	CPU/$	Mem/$	 mem	cpu	cores	cpcore	storage	 bits	IO             	type      	name
-    t1.micro    	  14	 0.48	$0.02	10.00	33.50	 0.67	 0.2	1	 0.2  	   0	   64	Low          	Micro    	Micro
-    m1.small    	  61	 2.04	$0.085	11.76	20.00	 1.7	 1  	1	 1   	 160	   32	Moderate    	Standard	Small
-    c1.medium   	 123	 4.08	$0.17	29.41	10.00	 1.7	 5  	2	 2.5	 350	   32	Moderate    	High-CPU	Medium
-    m1.large    	 246	 8.16	$0.34	11.76	22.06	 7.5	 4  	2	 2  	 850	   64	High         	Standard	Large
-    m2.xlarge   	 363	12.00	$0.50	13.00	35.40	17.7	 6.5	2	 3.25	 420	   64	Moderate    	High-Memory	Extra Large
-    c1.xlarge   	 493	16.32	$0.68	29.41	10.29	 7  	20  	8	 2.5	1690	   64	High         	High-CPU	Extra Large
-    m1.xlarge   	 493	16.32	$0.68	11.76	22.06	15  	 8  	4	 2  	1690	   64	High        	Standard	Extra Large
-    m2.2xlarge  	 726	24.00	$1.00	13.00	34.20	34.2	13  	4	 3.25	 850	   64	High        	High-Memory	Double Extra Large
-    m2.4xlarge  	1452	48.00	$2.00	13.00	34.20	68.4	26  	8	 3.25	1690	   64	High         	High-Memory	Quadruple Extra Large
-    cc1.4xlarge 	1161	38.40	$1.60	20.94	14.38	23  	33.5	2	16.75	1690	   64	Very High 10GB	Compute  	Quadruple Extra Large
-    cg1.4xlarge 	1524	50.40	$2.10	15.95	10.48	22  	33.5	2	16.75	1690	   64	Very High 10GB	Cluster GPU	Quadruple Extra Large
+
+    code         	$/mo	 $/day	 $/hr	CPU/$	Mem/$	  mem 	  cpu 	cores	cpcore	storage	bits	IO      	type       	name
+    t1.micro       	  15	  0.48	  .02	   13	   13	  0.61	  0.25	 0.25	  1   	      0	   32	Low     	Micro      	Micro
+    m1.small       	  58	  1.92	  .08	   13	   21	  1.7 	  1   	 1   	  1   	    160	   32	Moderate	Standard   	Small
+    m1.medium      	 116	  3.84	  .165	   13	   13	  3.75	  2   	 2   	  1   	    410	   32	Moderate	Standard   	Medium
+    c1.medium      	 120	  3.96	  .17	   30	   10	  1.7 	  5   	 2   	  2.5 	    350	   32	Moderate	High-CPU   	Medium
+    m1.large       	 232	  7.68	  .32	   13	   23	  7.5 	  4   	 2   	  2   	    850	   64	High    	Standard   	Large
+    m2.xlarge      	 327	 10.80	  .45	   14	   38	 17.1 	  6.5 	 2   	  3.25	    420	   64	Moderate	High-Memory	Extra Large
+    m1.xlarge      	 465	 15.36	  .64	   13	   23	 15   	  8   	 4   	  2   	   1690	   64	High    	Standard   	Extra Large
+    c1.xlarge      	 479	 15.84	  .66	   30	   11	  7   	 20   	 8   	  2.5 	   1690	   64	High    	High-CPU   	Extra Large
+    m2.2xlarge     	 653	 21.60	  .90	   14	   38	 34.2 	 13   	 4   	  3.25	    850	   64	High    	High-Memory	Double Extra Large
+    cc1.4xlarge    	 944	 31.20	 1.30	   26	   18	 23   	 33.5 	 2   	 16.75	   1690	   64	10GB    	Compute    	Quadruple Extra Large
+    m2.4xlarge     	1307	 43.20	 1.80	   14	   38	 68.4 	 26   	 8   	  3.25	   1690	   64	High    	High-Memory	Quadruple Extra Large
+    cg1.4xlarge    	1525	 50.40	 2.10	   16	   10	 22   	 33.5 	 2   	 16.75	   1690	   64	10GB    	Cluster GPU	Quadruple Extra Large
+    cc2.8xlarge    	1742	 57.60	 2.40	   37	   25	 60.5 	 88   	 2   	 44   	   3370	   64	10GB    	Compute    	Eight Extra Large
+
+    dummy header ln	  15	  0.48	 0.02	12345	12345	  0.61	  0.25	 0.25	  1.00	6712345	32123	Low     	Micro      	Micro                
 
 
 ## Storage Costs
 
-                            $/GB.hr  	$/GB.mo		$/GB.mo  	$/Mio	
+                            $/GB..mo		$/GB.mo  	$/Mio	
     EBS Volume     			$0.10				
-    EBS Snapshot S3			$0.14				
-    EBS I/O       			$0.10 			
+    EBS I/O       			            	         	$0.10
+    EBS Snapshot S3			$0.083				
 
-    S3 1st tb            	$0.14  /gb/month
-    S3 next 49tb         	$0.125 /gb/month
-    S3 next 450tb        	$0.110 /gb/month
+                         	Std $/GB.mo		Red.Red. $/GB.mo
+    S3 1st tb            	$0.125     	 	$0.093
+    S3 next 49tb         	$0.110     	 	$0.083 
+    S3 next 450tb        	$0.095  		$0.073 
 
 ### Storing 1TB data
 

data/notes/version-3_2.md

@@ -0,0 +1,273 @@
+
+# v3.2.0 (future): Revamped undercarriage, spec coverage, standalone usage
+
+This is a Snow Leopard-style version change. No new features to speak of, but a much more solid and predictable foundation. 
+
+* **significantly cleaner DSL mixin**: uses the new, awesome `Gorillib::Builder`, giving it a much cleaner handling of fields and collections
+
+* **attributes are late-resolved**: in previous versions, the way you 'resolved' a server was to collapse the entire attribute set of cluster/facet/server hard onto the server model, a consistent source of bugs. Resolution is now done with the `Gorillib::Record::Overlay` mechanism, which means that you can set an attribute on the cluster and read it from the facet; change it later an all lower layers see the update.
+
+* **standalone usable**: can use ironfan-knife as a standalone library.
+
+# v3.3.x (future): Coherent universe of Servers, Components, Aspects
+
+* **spec coverage**: 
+
+* **coherent data model**: 
+
+    ComputeLayer   -- common attributes of Provider, Cluster, Facet, Server
+      - overlay_stack of Cloud attributes
+
+    Universe        -- across organizations
+    Organization    -- one or many providers
+    Provider        -- 
+    - has_many  :clusters
+    Cluster         --
+    - has_many  :providers
+    - overlays  :main_provider
+    Facet           --
+    - has_one  :cluster
+    - overlays :cluster
+    Server         
+    - has_one  :facet
+    - overlays :cluster
+    - has_one chef_node
+    - has_one machine
+    
+    
+    System            Role          Cookbook
+    Component                       Cookbook+Recipes
+    
+     
+
+* **improved discovery**: 
+
+* **config isolation**: 
+
+
+### Nitpicks
+
+
+* make bootstrap_distro and image_name follow from os_version
+
+* minidash just publishes announcements
+* silverware is always included; it subsumes volumes 
+
+* if you add a `data_dir_for    :hadoop` to 
+
+* volumes should name their `mount_point` after themselves by default
+
+### Components
+
+* components replace roles (they are auto-generated by the component, and tie strictly to it)
+* 
+
+### Clusters 
+
+If clusters are more repeatable they won't be so bothersomely multi-provider:
+    
+    Ironfan.cluster :gibbon do 
+      cloud(:ec2) do
+        backing         'ebs' 
+        permanent       false
+      end
+      stack             :systemwide
+      stack             :devstack
+      stack             :monitoring
+      stack             :log_handling
+      
+      component         :hadoop_devstack
+      component         :hadoop_dedicated
+      
+      discovers         :zookeeper, :realm => :zk
+      discovers         :hbase,     :realm => :hbase
+
+      facet :master do
+        component		:hadoop_namenode
+        component		:hadoop_secondarynn
+        component		:hadoop_jobtracker
+      end
+      facet :worker do
+        component		:hadoop_datanode
+        component		:hadoop_tasktracker
+      end
+      
+      volume :hadoop_data do
+        data_dir_for    :hadoop_datanode, :hadoop_namenode, :hadoop_secondarynn
+        device          '/dev/sdj1'
+        size            100
+        keep            true
+      end
+    end
+
+
+Here are ideas about how to get there
+
+    # silverware is always included; it subsumes volumes 
+
+    organization :infochimps do
+      cloud(:ec2) do
+        availability_zones  ['us-east-1d']
+        backing             :ebs
+        image_name          'ironfan-natty'
+        bootstrap_distro    'ironfan-natty'
+        chef_client_script  'client.rb'
+        permanent           true
+      end
+      
+      volume(:default) do
+        keep                true
+        snapshot_name       :blank_xfs
+        resizable           true
+        create_at_launch    true
+      end
+      
+      stack :systemwide do
+        system(:chef_client) do
+          run_state         :on_restart
+        end
+        component		    :set_hostname
+        component		    :minidash
+        component           :org_base
+        component           :org_users
+        component           :org_final
+      end
+      
+      stack :devstack do
+        component		    :ssh
+        component		    :nfs_client
+        component		    :package_set
+      end
+
+      stack :monitoring do
+        component		:zabbix_agent
+      end
+      
+      stack :log_handling do
+        component		:log_handling
+      end
+    end
+    
+    stack :hadoop do
+    end
+    
+    stack :hadoop_devstack do
+      component         :pig
+      component         :jruby
+      component         :rstats
+    end
+    
+    stack :hadoop_dedicated do
+      component         :tuning
+    end
+
+    system :hadoop do
+      stack :hadoop_devstack
+      stack :zookeeper_client
+      stack :hbase_client
+    end   
+        
+    Ironfan.cluster :gibbon do 
+      cloud(:ec2) do
+        backing             'ebs' 
+        permanent           false
+      end
+      
+      system :systemwide do
+        exclude_stack   :monitoring
+      end
+      
+      # how are its components configured? distributed among machines?
+      system :hadoop do 
+            
+        # all servers will
+        # * have the `hadoop` role
+        # * have run_state => false for components with a daemon aspect by default
+        
+        facet :master do
+          # component :hadoop_namenode means
+          # * this facet has the `hadoop_namenode` role
+          # * it has the component's security_groups
+          # * it sets node[:hadoop][:namenode][:run_state] = true
+          # * it will mount the volumes that adhere to this component
+          component :hadoop_namenode
+        end
+        
+        # something gains eg zookeeper client if it discovers a zookeeper in another realm
+        # zookeeper must explicitly admit it discovers zookeeper, but can do that in the component
+        
+        # what volumes should it use on those machines?
+        # create the volumes, pair it to components
+        # if a component is on a server, it adds its volumes. 
+        # you can also add them explicitly.
+        
+        # volume tags are applied automagically from their adherance to components
+
+        volume :hadoop_data do                            # will be assigned to servers with components it lists
+          data_dir_for    :hadoop_datanode, :hadoop_namenode, :hadoop_secondarynn
+        end
+
+### Providers
+
+I want to be able to:
+
+* on a compute layer, modify its behavior depending on provider:
+  - example:
+  
+      facet(:bob) do 
+        cloud do
+          security_group  :bob
+          authorize       :from => :bobs_friends, :to => :bob
+        end
+        cloud(:ec2,       :flavor => 'm1.small')
+        cloud(:rackspace, :flavor => '2GB')
+        cloud(:vagrant,   :ram_mb =>  256 )
+      end
+
+  - Any world that understands security groups will endeavor to make a `bob` security group, and authorize the `bobs_friends` group to use it.
+  - On EC2 and rackspace, the `flavor` attribute is set explicitly
+  - On vagrant (which got no `flavor`), we instead specify how much ram to supply
+  - On any other provider the flavor and machine ram will follow defaults.
+
+* see all machines and clusters within an organization
+
+
+### Organizations
+
+* see the entire universe; this might get hairy, but not ridiculous
+  - each org describes its providers; only those are used
+  - you don't have to do much to add a provider, just say `provider(:ec2)`
+  - you can configure the provider like this:
+  
+      organization(:infochimps_test, :doc => 'Infochimps test cloud') do
+        provider(:vagrant)
+        provider(:ec2) do
+          access_key         '...' 
+          secret_access_key  '...'
+        end
+        provider(:hp_cloud) do
+          access_key         '...'
+          secret_access_key  '...'
+        end
+      end
+      
+      organization(:demo, :doc => 'Live client demo cloud') do
+        provider(:vagrant)
+        provider(:ec2)       do  #... end
+        provider(:hp_cloud)  do  #... end
+        provider(:rackspace) do  #... end
+      end
+
+  - clusters can be declared directly or imported from other organizations:
+  
+      organization :infochimps_test do
+        # developers' sandboxes
+        cluster  :dev_sandboxes
+        # all the example clusters, for development
+        organization(:examples).clusters.each do |cl|
+          add_cluster cl
+        end
+      end  
+
+  - if just starting, should see clusters; 
+    - per-org cluster dirs