agile_solo 0.1.0

data/spec/chef_repo/cookbooks/metachef/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--drb

data/spec/chef_repo/cookbooks/metachef/.watchr

@@ -0,0 +1,20 @@
+# -*- ruby -*-
+
+def run_spec(file)
+  file = File.expand_path(file, File.dirname(__FILE__))
+  unless File.exist?(file)
+    Watchr.debug "#{file} does not exist"
+    return
+  end
+
+  Watchr.debug "Running #{file}"
+  system       "rspec #{file}"
+end
+
+watch("spec/.*_spec\.rb") do |match|
+  run_spec(match[0])
+end
+
+watch("libraries/(.*)\.rb") do |match|
+  run_spec(%{spec/#{match[1]}_spec.rb})
+end

data/spec/chef_repo/cookbooks/metachef/Gemfile

@@ -0,0 +1,15 @@
+source "http://rubygems.org"
+
+gem 'chef',          "~> 0.10.4"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem 'bundler',      "~> 1"
+  gem 'yard',         "~> 0.6.7"
+  gem 'jeweler',      "~> 1.6.4"
+  gem 'rspec',        "~> 2.7.0"
+  gem 'watchr',       "~> 0.7"
+  # gem 'ruby-fsevent', "~> 0.2"
+  # gem 'rev'
+end

data/spec/chef_repo/cookbooks/metachef/README.md

@@ -0,0 +1,97 @@
+# metachef chef cookbook
+
+Cluster orchestration -- coordinates discovery, integration and decoupling of cookbooks
+
+## Overview
+
+Cookbooks repeatably express these and other aspects:
+* "I launch these daemons: ..."
+* "I haz a bukkit, itz naem '/var/log/lol'"
+* "I have a dashboard at 'http://....:...'"
+* ... and much more.
+
+Wouldn't it be nice if announcing a log directory caused...
+  - my log rotation system to start rotating my logs?
+  - a 'disk free space' gauge to be added to the monitoring dashboard for that service?
+  - flume (or whatever) began picking up my logs and archiving them to a predictable location?
+  - in the case of standard apache logs, a listener to start counting the rate of requests, 200s, 404s and so forth?
+Similarly, announcing ports should mean
+  - the firewall and security groups configure themselves correspondingly
+  - the monitor system starts regularly pinging the port for uptime and latency 
+  - and pings the interfaces that it should *not* appear on to ensure the firewall is in place?
+
+Cluster chef make those aspects standardized and predictable, and provides integration and discovery hooks. The key is to make integration *inevitable*: No more forgetting to rotate or monitor a service, or having a config change over here screw up a dependent system over there.
+
+__________________________________________________________________________
+
+(*below is a planning document and may not perfectly reflect reality*)
+
+FIXME: **update for version_3 release**
+
+Attributes are scoped by *cookbook* and then by *component*.
+* If I declare `i_haz_a_service_itz('redis)`, it will look in `node[:redis]`.
+* If I declare `i_haz_a_service_itz('hadoop-namenode')`, it will look in `node[:hadoop]` for cookbook-wide concerns and `node[:hadoop][:namenode]` for component-specific concerns.
+
+* The cookbook scope is always named for its cookbook. Its attributes live in`node[:cookbook_name]`.
+  - if everything in the cookbook shares a concern, it sits at cookbook level. So the hadoop log directory (shared by all its components) is at `(scratch_root)/hadoop/log`.
+* If there is only one component, it can be implicitly named for its cookbook. In this case, it is omitted: the component attributes live in `node[:cookbook_name]` (which is the same as the component name).
+* If there are multiple components, they will live in `node[:cookbook_name][:component_name]` (eg `[:hadoop][:namenode]` or `[:flume][:master]`. In file names, these become `(whatever)/cookbook_name/component_name/(whatever)`; in other cases they are joined as `cookbook_name-component_name`.
+
+Allow nodes to discover the location for a given service at runtime, adapting when new services register.
+
+### Discovery
+
+Allow nodes to discover the location for a given service at runtime, adapting
+when new services register.
+
+#### Operations:
+
+* register for a service. A timestamp records the last registry.
+* discover all chef nodes that have registered for the given service.
+* discover the most recent chef node for that service.
+* get the 'public_ip' for a service -- the address that nodes in the larger
+  world should use
+* get the 'public_ip' for a service -- the address that nodes on the local
+  subnet / private cloud should use
+
+#### Implementation
+
+Nodes register a service by calling `announce`, which sets a hash containing
+'timestamp' (the time of registry) and other metadata passed in.
+
+## Recipes 
+
+* `default`                  - Base configuration for metachef
+
+## Integration
+
+Supports platforms: debian and ubuntu
+
+
+
+## Attributes
+
+* `[:metachef][:conf_dir]`            -  (default: "/etc/metachef")
+* `[:metachef][:log_dir]`             -  (default: "/var/log/metachef")
+* `[:metachef][:home_dir]`            -  (default: "/etc/metachef")
+* `[:metachef][:user]`                -  (default: "root")
+* `[:users][:root][:primary_group]`   -  (default: "root")
+
+## License and Author
+
+Author::                Philip (flip) Kromer - Infochimps, Inc (<coders@infochimps.com>)
+Copyright::             2011, Philip (flip) Kromer - Infochimps, Inc
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+> readme generated by [cluster_chef](http://github.com/infochimps/cluster_chef)'s cookbook_munger

data/spec/chef_repo/cookbooks/metachef/attributes/default.rb

@@ -0,0 +1,18 @@
+
+default[:metachef][:conf_dir] = '/etc/metachef'
+default[:metachef][:log_dir]  = '/var/log/metachef'
+default[:metachef][:home_dir] = '/etc/metachef'
+
+default[:metachef][:user]     = 'root'
+
+# Request user account properties here.
+default[:users]['root'][:primary_group] = value_for_platform(
+  "openbsd"   => { "default" => "wheel" },
+  "freebsd"   => { "default" => "wheel" },
+  "mac_os_x"  => { "default" => "wheel" },
+  "default"   => "root"
+)
+
+default[:announces] ||= Mash.new
+
+default[:discovers] ||= Mash.new

data/spec/chef_repo/cookbooks/metachef/definitions/daemon_user.rb

@@ -0,0 +1,68 @@
+
+#
+# If present, we will use node[(name)][(component)] *and then* node[(name)] to
+# look up scoped default values.
+#
+# So, daemon_user('ntp') looks for a username in node[:ntp][:user], while
+# daemon_user('ganglia.server') looks first in node[:ganglia][:server][:user]
+# and then in node[:ganglia][:user].
+#
+define(:daemon_user,
+  :action       => [:create, :manage],  # action. You typically want [:create, :manage] or [:create]
+  :component    => nil,                 # if present, will use node[(name)][(component)] *and then* node[(name)] to look up values.
+  :user         => nil,                 # username to create.      default: `scoped_hash[:user]`
+  :home         => nil,                 # home directory for daemon. default: `scoped_hash[:pid_dir]`
+  :shell        => '/bin/false',        # shell to set. default: `/bin/false`
+  :group        => nil,                 # group for daemon.          default: `scoped_hash[:group]`
+  :comment      => nil,                 # comment for user info
+  :create_group => true                 # Action to take on the group: `true` means `[:create]`, false-y means do nothing, or you can supply explicit actions (eg `[:create, :manage]`). default: true
+  ) do
+
+  sys, subsys = params[:name].to_s.split(".", 2).map(&:to_sym)
+  component = ClusterChef::Component.new(node, sys, subsys)
+
+  params[:user]         ||= component.node_attr(:user, :required)
+  params[:group]        ||= component.node_attr(:group) || params[:user]
+  params[:home]         ||= component.node_attr(:pid_dir, :required)
+  params[:comment]      ||= "#{component.name} daemon"
+  #
+  user_val                = params[:user].to_s
+  group_val               = params[:group].to_s
+  begin
+    uid_val                 = node[:users ][user_val ] && node[:users ][user_val ][:uid]
+  rescue
+    uid_val = nil
+  end
+  begin
+    gid_val                 = node[:groups][group_val] && node[:groups][group_val][:gid]
+  rescue
+    gid_val = nil
+  end
+
+  params[:create_group]   = [:create] if (params[:create_group] == true)
+  params[:create_group]   = false     if (group_val == 'nogroup')
+
+  #
+  # Make the group
+  #
+  if params[:create_group] && (group_val != 'nogroup')
+    group group_val do
+      gid       gid_val
+      action    params[:create_group]
+    end
+  end
+
+  #
+  # Make the user
+  #
+  user user_val do
+    uid           uid_val
+    gid           group_val
+    password      nil
+    shell         params[:shell]
+    home          params[:home]
+    supports      :manage_home => false # you must create standard dirs yourself
+    action        params[:action]
+  end
+
+end

data/spec/chef_repo/cookbooks/metachef/definitions/kill_old_service.rb

@@ -0,0 +1,35 @@
+
+define(:kill_old_service,
+  :script       => nil,
+  :hard         => false,
+  ) do
+  params[:script] ||= "/etc/init.d/#{params[:name]}"
+
+  # cheating: don't bother if the script isn't there
+  if (File.exists?(params[:script]))
+
+    service params[:name] do
+      action      [:stop, :disable]
+      pattern     params[:pattern] if params[:pattern]
+      only_if{ File.exists?(params[:script]) }
+    end
+
+    ruby_block("stop #{params[:name]}") do
+      block{   }
+      action      :create
+      notifies    :stop, "service[#{params[:name]}]", :immediately
+      only_if{ File.exists?(params[:script]) }
+    end
+
+    if params[:hard] # sometimes these shitty little things gotta be told thrice
+      bash "stop again" do
+        code "service #{params[:name]} stop ; sleep 1 ; service #{params[:name]} stop ; true"
+        only_if{ File.exists?(params[:script]) }
+      end
+    end
+
+    file(params[:script]) do
+      action :delete
+    end
+  end
+end

data/spec/chef_repo/cookbooks/metachef/definitions/standard_dirs.rb

@@ -0,0 +1,54 @@
+STANDARD_DIRS = Mash.new({
+  :home_dir     => { :uid => 'root', :gid => 'root', },
+  :conf_dir     => { :uid => 'root', :gid => 'root', },
+  :lib_dir      => { :uid => 'root', :gid => 'root', },
+  :log_dir      => { :uid => :user,  :gid => :group, :mode => "0775", },
+  :pid_dir      => { :uid => :user,  :gid => :group, },
+  :tmp_dir      => { :uid => :user,  :gid => :group, },
+  :data_dir     => { :uid => :user,  :gid => :group, },
+  :data_dirs    => { :uid => :user,  :gid => :group, },
+  :journal_dir  => { :uid => :user,  :gid => :group, },
+  :journal_dirs => { :uid => :user,  :gid => :group, },
+  :cache_dir    => { :uid => :user,  :gid => :group, },
+}) unless defined?(STANDARD_DIRS)
+
+#
+# If present, we will use node[(name)][(subsys)] *and then* node[(name)] to
+# look up scoped default values.
+#
+# So, daemon_user('ntp') looks for its :log_dir in node[:ntp][:log_dir], while
+# daemon_user('ganglia.server') looks first in node[:ganglia][:server][:log_dir]
+# and then in node[:ganglia][:log_dir].
+#
+define(:standard_dirs,
+  :subsys      => nil, # if present, will use node[(name)][(subsys)] *and then* node[(name)] to look up values.
+  :directories => [],
+  :log_dir     => nil,
+  :home_dir    => nil,
+  :user        => nil, # username to create.      default: `scoped_hash[:user]`
+  :group       => nil  # group for user.          default: `scoped_hash[:group]`
+  ) do
+
+  sys, subsys = params[:name].to_s.split(".", 2).map(&:to_sym)
+  component = ClusterChef::Component.new(node, sys, subsys)
+
+  params[:user]       ||= component.node_attr(:user, :required)
+  params[:group]      ||= component.node_attr(:group) || params[:user]
+
+  [params[:directories]].flatten.each do |dir_type|
+    dir_paths = component.node_attr(dir_type, :required) or next
+    hsh = (STANDARD_DIRS.include?(dir_type) ? STANDARD_DIRS[dir_type].dup : Mash.new)
+    hsh[:uid] = params[:user]  if (hsh[:uid] == :user )
+    hsh[:gid] = params[:group] if (hsh[:gid] == :group)
+    [dir_paths].flatten.each do |dir_path|
+      directory dir_path do
+        owner       hsh[:uid]
+        group       hsh[:gid]
+        mode        hsh[:mode] || '0755'
+        action      :create
+        recursive   true
+      end
+    end
+  end
+
+end

data/spec/chef_repo/cookbooks/metachef/libraries/aspect.rb

@@ -0,0 +1,125 @@
+require File.expand_path('metachef.rb', File.dirname(__FILE__))
+
+module ClusterChef
+
+  # An *aspect* is an external property, commonly encountered across multiple
+  # systems, that decoupled agents may wish to act on.
+  #
+  # For example, many systems have a Dashboard aspect -- phpMySQL, the hadoop
+  # jobtracker web console, a one-pager generated by the dashpot recipe, or a
+  # purpose-built backend for your website. The following independent concerns
+  # can act on such dashboard aspects: * a dashboard dashboard creates a page
+  # linking to all of them * your firewall grants access from internal machines
+  # and denies access on public interfaces * the monitoring system checks that
+  # the port is open and listening
+  #
+  # Aspects are able to do the following:
+  #
+  # * Convert to and from a plain hash,
+  #
+  # * ...and thusly to and from plain node metadata attributes
+  #
+  # * discover its manifestations across all systems (on all or some
+  #   machines): for example, all dashboards, or all open ports.
+  #
+  # * identify instances from a system's by-convention metadata. For
+  #   example, given a chef server system at 10.29.63.45 with attributes
+  #     `:chef_server => { :server_port => 4000, :dash_port => 4040 }`
+  #   the PortAspect class would produce instances for 4000 and 4040, since by
+  #   convention an attribute ending in `_port` means "I have a port aspect`;
+  #   the DashboardAspect would recognize the `dash_port` attribute and
+  #   produce an instance for `http://10.29.63.45:4040`.
+  #
+  # Note:
+  #
+  # * separate *identifiable conventions* from *concrete representation* of
+  #   aspects. A system announces that it has a log aspect, and by convention
+  #   declares a `:log_dir` attribute. At that point it is regularized into a
+  #   LogAspect instance and stored in the `node[:aspects]` tree. External
+  #   concerns should only inspect these concrete Aspects, and never go
+  #   hunting for thins with a `:log_dir` attribute.
+  #
+  # * conventions can be messy, but aspects are perfectly uniform
+  #
+  class Aspect
+    include AttrStruct
+    extend  ClusterChef::NodeUtils
+
+    dsl_attr(:component, :kind_of => ClusterChef::Component)
+    dsl_attr(:name,      :kind_of => [String, Symbol])
+
+    # checks that the aspect is well-formed. returns non-empty array if there is lint.
+    #
+    # @abstract
+    #   override to provide guidance, filling an array with warning strings. Include
+    #       errors + super
+    #   as the last line.
+    #
+    def lint
+      []
+    end
+
+    def lint!
+      lint.flatten.compact.each{|l| Chef::Log.warn(l) }
+    end
+
+    def lint_flavor
+      self.class.allowed_flavors.include?(self.flavor) ? [] : ["Unexpected #{self.class.handle} flavor #{flavor.inspect}"]
+    end
+
+      # include AttrStruct::ClassMethods
+      # include ClusterChef::NodeUtils
+
+      def self.register!
+        ClusterChef::Component.has_aspect(self)
+      end
+
+      #
+      # Extract attributes matching the given pattern.
+      #
+      # @param [Hash]   info   -- hash of key-val pairs
+      # @param [Regexp] regex  -- filter for keys matching this pattern
+      #
+      # @yield on each match
+      # @yieldparam [String, Symbol] key   -- the matching key
+      # @yieldparam [Object]         val   -- its value in the info hash
+      # @yieldparam [MatchData]      match -- result of the regexp match
+      # @yieldreturn [Aspect]        block should return an aspect
+      #
+      # @return [Array<Aspect>] collection of the block's results
+      def self.attr_matches(component, regexp)
+        results = Mash.new
+        component.node_info.each do |key, val|
+          next unless (match = regexp.match(key.to_s))
+          result = yield(key, val, match) or next
+          result.lint!
+          results[result.name] ||= result
+        end
+        results
+      end
+
+      def self.rsrc_matches(rsrc_clxn, resource_name, cookbook_name)
+        results = Mash.new
+        rsrc_clxn.each do |rsrc|
+          next unless rsrc.resource_name.to_s == resource_name.to_s
+          next unless rsrc.cookbook_name.to_s =~ /#{cookbook_name}/
+          result = yield(rsrc) or next
+          results[result.name] ||= result
+        end
+        results
+      end
+
+      # strip off module part and '...Aspect' from class name
+      # @example ClusterChef::FooAspect.handle # :foo
+      def self.handle
+        @handle ||= self.name.to_s.gsub(/.*::(\w+)Aspect\z/,'\1').gsub(/([a-z\d])([A-Z])/,'\1_\2').downcase.to_sym
+      end
+
+      def self.plural_handle
+        "#{handle}s".to_sym
+      end
+
+    # end
+    # def self.included(base) ; base.extend(ClassMethods) ; end
+  end
+end

data/spec/chef_repo/cookbooks/metachef/libraries/aspects.rb

@@ -0,0 +1,170 @@
+require File.expand_path('metachef.rb', File.dirname(__FILE__))
+module ClusterChef
+
+  #
+  # * scope[:run_state]
+  #
+  # from the eponymous service resource,
+  # * service.path
+  # * service.pattern
+  # * service.user
+  # * service.group
+  #
+  class DaemonAspect < Aspect
+    register!
+    dsl_attr(:service_name, :kind_of => String)
+    dsl_attr(:pattern,      :kind_of => String)
+    dsl_attr(:run_state,    :kind_of => [String, Symbol])
+    dsl_attr(:service_name, :kind_of => String)
+
+    def self.harvest(run_context, component)
+      rsrc_matches(run_context.resource_collection, :service, component.sys) do |rsrc|
+        next unless rsrc.name =~ /#{component.name}/
+        svc = self.new(component, rsrc.name, rsrc.service_name, rsrc.pattern)
+        svc.run_state(component.node_info[:run_state])
+        svc
+      end
+    end
+
+    def lint
+      errs = super
+      if  not %w[stop start nothing].include?(run_state.to_s)
+        badness = run_state ? "Odd run_state #{run_state}" : "No run_state"
+        err = "#{badness} for daemon #{name}: set node[:#{component.sys}][:#{component.subsys}] to :stop, :start or :nothing"
+        Chef::Log.warn(err)
+        errs << err
+      end
+      errs
+    end
+  end
+
+  class PortAspect < Aspect
+    register!
+    dsl_attr(:flavor,    :kind_of => Symbol, :coerce => :to_sym)
+    dsl_attr(:port_num,  :kind_of => String)
+    dsl_attr(:addrs,     :kind_of => Array)
+
+    ALLOWED_FLAVORS = [ :ssh, :ntp, :ldap, :smtp, :ftp, :http, :pop, :nntp, :imap, :tcp, :https, :telnet ]
+    def self.allowed_flavors() ALLOWED_FLAVORS ; end
+
+    def self.harvest(run_context, component)
+      attr_aspects = attr_matches(component, /^((.+_)?port)$/) do |key, val, match|
+        name   = match[1]
+        flavor = match[2].to_s.empty? ? :port : match[2].gsub(/_$/, '').to_sym
+        # p [match.captures, name, flavor].flatten
+        self.new(component, name, flavor, val.to_s)
+      end
+    end
+  end
+
+  class DashboardAspect < Aspect
+    register!
+    dsl_attr(:flavor,    :kind_of => Symbol, :coerce => :to_sym)
+    dsl_attr(:url,       :kind_of => String)
+
+    ALLOWED_FLAVORS = [ :http, :jmx ]
+    def self.allowed_flavors() ALLOWED_FLAVORS ; end
+
+    def self.harvest(run_context, component)
+      attr_aspects = attr_matches(component, /^(.*dash)_port(s)?$/) do |key, val, match|
+        name   = match[1]
+        flavor = (name == 'dash') ? :http_dash : name.to_sym
+        url    = "http://#{private_ip_of(run_context.node)}:#{val}/"
+        self.new(component, name, flavor, url)
+      end
+    end
+  end
+
+  #
+  # * scope[:log_dirs]
+  # * scope[:log_dir]
+  # * flavor: http, etc
+  #
+  class LogAspect < Aspect
+    register!
+    dsl_attr(:flavor,    :kind_of => Symbol, :coerce => :to_sym)
+    dsl_attr(:dirs,      :kind_of => Array)
+
+    ALLOWED_FLAVORS = [ :http, :log4j, :rails ]
+
+    def self.harvest(run_context, component)
+      attr_matches(component, /^log_dir(s?)$/) do |key, val, match|
+        name = 'log'
+        dirs = Array(val)
+        self.new(component, name, name.to_sym, dirs)
+      end
+    end
+  end
+
+  #
+  # * attributes with a _dir or _dirs suffix
+  #
+  class DirectoryAspect < Aspect
+    def self.plural_handle() :directories ; end
+    register!
+    dsl_attr(:flavor,    :kind_of => Symbol, :coerce => :to_sym)
+    dsl_attr(:dirs,      :kind_of => Array)
+
+    ALLOWED_FLAVORS = [ :home, :conf, :log, :tmp, :pid, :data, :lib, :journal, ]
+    def self.allowed_flavors() ALLOWED_FLAVORS ; end
+
+    def self.harvest(run_context, component)
+      attr_aspects = attr_matches(component, /(.*)_dir(s?)$/) do |key, val, match|
+        name = match[1]
+        val  = Array(val)
+        self.new(component, name, name.to_sym, val)
+      end
+      rsrc_aspects = rsrc_matches(run_context.resource_collection, :directory, component.sys) do |rsrc|
+        rsrc
+      end
+      # [attr_aspects, rsrc_aspects].flatten.each{|x| p x }
+      attr_aspects
+    end
+  end
+
+  #
+  # Code assets (jars, compiled libs, etc) that another system may wish to
+  # incorporate
+  #
+  class ExportedAspect < Aspect
+    register!
+    dsl_attr(:flavor,    :kind_of => Symbol, :coerce => :to_sym)
+    dsl_attr(:files,     :kind_of => Array)
+
+    ALLOWED_FLAVORS = [:jars, :confs, :libs]
+    def self.allowed_flavors() ALLOWED_FLAVORS ; end
+
+    def lint
+      super() + [lint_flavor]
+    end
+
+    def self.harvest(run_context, component)
+      attr_matches(component, /^exported_(.*)$/) do |key, val, match|
+        name = match[1]
+        self.new(component, name, name.to_sym, val)
+      end
+    end
+  end
+
+  #
+  # manana
+  #
+
+  # # usage constraints -- ulimits, java heap size, thread count, etc
+  # class UsageLimitAspect
+  # end
+  # # deploy
+  # # package
+  # # account (user / group)
+  # class CookbookAspect < Struct.new(:component, :name,
+  #     :deploys, :packages, :users, :groups, :depends, :recommends, :supports,
+  #     :attributes, :recipes, :resources, :authors, :license, :version )
+  # end
+  #
+  # class CronAspect
+  # end
+  #
+  # class AuthkeyAspect
+  # end
+
+end