zonify 0.4.0.5

data/README

@@ -0,0 +1,165 @@
+SYNOPSIS
+       zonify ... (-h|-[?]|--help) ...
+       zonify ec2 <rewrite rules>* > zone.ec2.yaml
+       zonify ec2/r53 <domain> <rewrite rules>* > changes.yaml
+       zonify r53 <domain> > zone.r53.yaml
+       zonify diff zone.r53.yaml zone.ec2.yaml > changes.yaml
+       zonify rewrite <rewrite rules>* < zone.ec2.yaml
+       zonify summarize < changes.yaml
+       zonify apply < changes.yaml
+       zonify sync <domain> <rewrite rules>*
+       zonify normalize <domain>
+       zonify eips
+
+DESCRIPTION
+       The  zonify  tool  allows  one to create DNS entries for all instances,
+       tags and load balancers in EC2 and synchronize a  Route  53  zone  with
+       these entries.
+
+       The  zonify tool and libraries intelligently insert a final and initial
+       '.' as needed to conform to DNS conventions. One may enter the  domains
+       at the command line as example.com or example.com.; it will work either
+       way.
+
+       For access to AWS APIs, zonify uses the  the  conventional  environment
+       variables to select regions and specify credentials:
+
+       AWS_ACCESS_KEY AWS_ACCESS_KEY_ID
+       AWS_SECRET_KEY AWS_SECRET_ACCESS_KEY
+       EC2_URL
+
+       These  variables  are used by many AWS libraries and tools. As a conve-
+       nience, the environment variable AWS_REGION may  be  used  with  region
+       nicknames:
+
+       AWS_REGION=eu-west-1
+
+       The  Zonify  subcommands  allow  staged  generation, transformation and
+       auditing of entries as well as straightforward,  one-step  synchroniza-
+       tion.
+
+          ec2 (--srv-singleton|--no-srv-singleton)?
+                 Organizes  instances,  load  balancers,  security  groups and
+                 instance metadata into DNS entries, with the  generic  suffix
+                 '.' (intended to be transformed by later commands).
+
+          ec2/r53 (--types CNAME,SRV)? (--srv-singleton|--no-srv-singleton)?
+                 Creates  a  changes  file,  describing  how records under the
+                 given suffix would be created and deleted to bring it  in  to
+                 sync with EC2. By default, only records of type CNAME and SRV
+                 are examined and changed.
+
+          r53    Capture all Route 53 records under the given suffix.
+
+          diff (--types CNAME,SRV,A,MX,...)?
+                 Describe changes (which can be fed to the  apply  subcommand)
+                 needed to bring a Route 53 domain in the first file into sync
+                 with domain described in the second file. The suffix is taken
+                 from  the  first  file.  The  default with diff (unlike other
+                 zonify subcommands) is to examine all record types.
+
+          rewrite (--srv-singleton|--no-srv-singleton)?
+                 Apply rewrite rules to the domain file.
+
+          summarize
+                 Summarize changes in a changes file, writing to STDOUT.
+
+          apply  Apply a changes file.
+
+          sync (--types CNAME,SRV)? (--srv-singleton|--no-srv-singleton)?
+                 Sync the given domain with EC2. By default, only  records  of
+                 type CNAME and SRV are examined and changed.
+
+          normalize
+                 Create  CNAMEs  for  SRV records that have only one server in
+                 them and rebase records on to the given domain.
+
+          eips   List all Elastic IPs and DNS entries that map to them.
+
+       The --[no-]srv-singleton options control creation of CNAMEs for single-
+       ton  SRV  records. They are enabled by default; but it can be useful to
+       disable them for pre-processing the YAML and then adding them with nor-
+       malize. For example:
+
+       The  --[no-]srv-singleton  options  also  control  creation of weighted
+       round-robin CNAMEs, an infelicity in nomenclature.
+
+       zonify r53 amz.example.com > r53.yaml
+       zonify ec2 --no-srv-singleton > ec2.yaml
+       my-yaml-rewriter < ec2.yaml > adjusted.yaml
+       zonify normalize amz.example.com < adjusted.yaml > normed.yaml
+       zonify diff r53.yaml normed.yaml | zonify apply
+
+SYNC POLICY
+       Zonify assumes the domain given on the command line is  entirely  under
+       the  control of Zonify; records not reflecting the present state of EC2
+       are scheduled for deletion in the generated  changesets.  This  can  be
+       controlled to some degree with the --types option.
+
+       The sync scopes over the domain and not necessarily the entire Route 53
+       zone.  Say, for example, one has example.com in a Route 53 zone and one
+       plans to use amz.example.com for Amazon instance records.  In this sce-
+       nario, Zonify will only specify changes that delete or  create  records
+       under amz.example.com; www.example.com, s0.mobile.example.com and simi-
+       lar records will not be affected.
+
+YAML OUTPUT
+       All records and change sets are sorted by name on output. The data com-
+       ponents of records are also sorted. This ensures consistent output from
+       run to run; and allows the diff tool to return meaningful results  when
+       outputs are compared.
+
+       One  exception  to this rule is the r53 subcommand, which preserves the
+       order of data as it was found in Route 53.
+
+REWRITE RULES
+       Rewrite rules take the  form  <domain>(:<domain>)+.  To  shorten  names
+       under the apache security group to web.amz.example.com, use:
+
+       apache.sg:web
+
+       To keep both forms, use the rule:
+
+       apache.sg:apache.sg:web
+
+GENERATED RECORDS AND QUERYING
+       For  records  where  there  are  potentially  many  servers -- security
+       groups, tags, load balancers -- Zonify creates SRV records. When a  SRV
+       record  has  only one entry under it, a simple CNAME is created. When a
+       SRV record contains multiple  records,  multiple  weighted  round-robin
+       CNAMEs are created, one for each server in the SRV record.
+
+       Records created include:
+
+          i-ABCD1234.inst.
+                 Individual instances.
+
+          _*._*.<value>.<key>.tag.
+                 SRV records for tags.
+
+          _*._*.<name>.sg.
+                 SRV records for security groups.
+
+          _*._*.<name>.elb
+                 SRV records for instances behind Elastic Load Balancers.
+
+          domU-*.priv., ip-*.priv
+                 Records  pointing  to  the default hostname, derived from the
+                 private DNS entry, set by many AMIs.
+
+       A list of all instances is placed under inst  --  continuing  with  our
+       example above, this would be the SRV record _*._*.inst.amz.example.com.
+       To obtain the list of all instances with dig:
+
+       dig @8.8.8.8 +tcp +short _*._*.inst.amz.example.com SRV | cut -d' ' -f4
+
+       The cut call is necessary to remove some  values,  always  nonces  with
+       Zonify, that are part of standard format SRV records.
+
+EXAMPLES
+       # Create records under amz.example.com, with instance names appearing
+       # directly under .amz.example.com.
+       zone sync amz.example.com name.tag:.
+       # Similar to above but stores changes to disk for later application.
+       zone ec2/r53 amz.example.com name.tag:. > changes.yaml
+

data/bin/zonify

@@ -0,0 +1,254 @@
+#!/usr/bin/env ruby
+root = File.expand_path("#{File.dirname(__FILE__)}/..")
+$LOAD_PATH.unshift("#{root}/lib") if File.directory?("#{root}/lib")
+USAGE = File.read("#{root}/README")
+
+require 'readline'
+require 'yaml'
+
+require 'rubygems'
+require 'fog'
+
+require 'zonify'
+
+
+class CLIAWS
+  attr_reader :options
+  def initialize(access=nil, secret=nil)
+    @options = {}
+    @options.merge!(options) if options
+    env_adapter
+    { :region                => ENV['AWS_REGION'],
+      :aws_access_key_id     => (access or ENV['AWS_ACCESS_KEY']),
+      :aws_secret_access_key => (secret or ENV['AWS_SECRET_KEY'])
+    }.each{|k,v| @options.merge!(k=>v) if v }
+  end
+  def respond_to?(sym)
+    aws.respond_to?(sym)
+  end
+  def env_adapter
+    [ %w| AWS_ACCESS_KEY_ID     AWS_ACCESS_KEY |,
+      %w| AWS_SECRET_ACCESS_KEY AWS_SECRET_KEY | ].each do |a, b|
+      ENV[a] = ENV[b] if ENV[b]     # Ensure new variable takes precedence.
+      ENV[b] = ENV[a] unless ENV[b]
+    end
+    # Sometimes libraries can not not handle being passed a region if the
+    # EC2_URL is set.
+    ENV['EC2_URL'] = nil if ENV['AWS_REGION']
+  end
+  def method_missing(sym, *args, &block)
+    begin
+      aws.send(sym, *args, &block)
+    rescue Fog::Errors::Error => e
+      abort "AWS error: #{e}"
+    end
+  end
+  def aws
+    @aws ||= Zonify::AWS.create(@options)
+  end
+end
+
+def rejected(rejected_changes)
+  unless rejected_changes.empty?
+    STDERR.puts 'Rejected some changes, because they were too large.'
+    rejected_changes.each do |change|
+      STDOUT.puts <<YAML
+- :name: #{change[:name]}
+  :type: #{change[:type]}
+YAML
+    end
+  end
+end
+
+def check_name(suffix)
+  abort 'No domain given.' unless suffix
+  unless suffix.split('.').all?{|s| s.empty? or Zonify::LDH_RE.match(s) }
+    abort "Not a conventional, LDH domain name: #{suffix}"
+  end
+end
+
+def display(changes)
+  if changes.empty?
+    'No changes; nothing to do.'
+  else
+    summary = changes.inject({}) do |acc, change|
+      name, type, set = [change[:name], change[:type], change[:set_identifier]]
+      key = "#{name} #{type}" + ( set ? " (#{set})" : "" )
+      acc[key] ||= []
+      acc[key]  << change[:action]
+      acc
+    end.map do |k, v|
+      case v
+      when ['DELETE', 'CREATE'] then [k, 'replace']
+      when ['DELETE']          then [k, 'delete']
+      when ['CREATE']          then [k, 'create']
+      end
+    end.sort
+    len = summary.map{|name, op| [name.length, 64].min }.max
+    formatted = summary.map{|name, op| name.ljust(len) + "  --->  " + op }
+    formatted.unshift("There are #{summary.length} changes.").join("\n")
+  end
+end
+
+def yaml_with_default(source, default)
+  s = source.read.rstrip
+  unless s.empty?
+    yaml = YAML.load(s)
+    abort 'Bad YAML parse.' unless yaml
+    yaml
+  else
+    default
+  end
+end
+
+def normed_with_mappings(mapping_args, suffix, records, norm=true)
+  mappings = mapping_args.map{|s| Zonify::Mappings.parse(s) }
+  abort 'Bad mapping parse.' if mappings.any?{|r| r.nil? }
+  mappings.each{|k, v| ([k] + v).each{|s| check_name(s) } }
+  clipped  = Zonify::Mappings.rewrite(records, [[suffix,['.']]])
+  mapped   = Zonify::Mappings.rewrite(clipped, mappings)
+  restored = Zonify::Mappings.rewrite(mapped, [['.',[suffix]]])
+  norm ? Zonify.normalize(restored) : restored
+end
+
+def confirm_dialogue
+  # Restore terminal state on SIGINT. Needed due to use of Readline.
+  stty_save = `stty -g 2>/dev/null`.chomp
+  trap('INT') do
+    system('stty', stty_save)
+    exit
+  end
+  perform = false
+  msg = 'Perform changes?'
+  while line = Readline.readline("#{msg} [Y/n]  ")
+    case line
+    when /^(n|no)$/i
+      perform = false
+      STDERR.puts 'Abandoning changes...'
+      break
+    when /^(y|yes|)$/i
+      perform = true
+      STDERR.puts 'Initiating changes...'
+      break
+    when nil
+      perform = false
+      break
+    else
+      msg = "Perform changes? Please answer with 'y' or 'n'."
+    end
+  end
+  perform
+end
+
+def main
+  if ARGV.any?{|arg| %w| help -h --help -? |.member?(arg) }
+    puts USAGE
+    exit
+  end
+  #confirm = [ARGV.delete("-c"), ARGV.delete("--confirm")].any?
+  confirm = false
+  quiet      = (ARGV.delete("-q") or ARGV.delete("--quiet"))
+  netlogging = (ARGV.delete("-n") or ARGV.delete("--net"))
+  norm       = (ARGV.delete("--srv-singleton") or not
+                ARGV.delete("--no-srv-singleton"))
+  aws = CLIAWS.new
+  types = nil
+  while i = ARGV.index('--types')
+    ARGV.delete_at(i)
+    types = ARGV[i].split(/ +| *, */)
+    ARGV.delete_at(i)
+    bad = types.map{|s| s if not Zonify::RRTYPE_RE.match(s) }.compact
+    abort "Bad RR types: #{bad.join(' ')}" unless bad.empty?
+  end
+  case ARGV[0]
+  when 'ec2'
+    data = aws.ec2_zone
+    mappings = (ARGV[1..-1] or [])
+    mapped = normed_with_mappings(mappings, '.', data, norm)
+    STDOUT.write Zonify::YAML.format(mapped, '.')
+  when 'r53'
+    suffix = ARGV[1]
+    check_name(suffix)
+    zone, data = aws.route53_zone(suffix)
+    if zone and data
+      STDOUT.write Zonify::YAML.format(data, suffix)
+    else
+      STDERR.puts 'No zone found; outputting nonce zone.'
+      STDOUT.write Zonify::YAML.format({}, '.')
+    end
+  when 'eips'
+    result = aws.eip_scan
+    entries = result.keys.sort.map do |k|
+      dumped = ::YAML.dump(k=>result[k])
+      Zonify::YAML.trim_lines(dumped).join.sub(/ *\[\]$/,'')
+    end.join
+    STDOUT.write entries
+  when 'diff'
+    suffix, old_records = Zonify::YAML.read(File.read(ARGV[1]))
+    new_suffix, new_records = Zonify::YAML.read(File.read(ARGV[2]))
+    qualified = Zonify::Mappings.rewrite(new_records, [[new_suffix,[suffix]]])
+    changes = Zonify.diff(qualified, old_records, (types or %w| * |))
+    STDERR.puts(display(changes)) unless quiet
+    STDOUT.write(Zonify::YAML.trim_lines(YAML.dump(changes)))
+  when 'sync'
+    suffix = ARGV[1]
+    check_name(suffix)
+    new_records = aws.ec2_zone
+    mappings = (ARGV[2..-1] or [])
+    mapped = normed_with_mappings(mappings, suffix, new_records)
+    _, old_records = aws.route53_zone(suffix)
+    changes = Zonify.diff(mapped, old_records, (types or %w| CNAME SRV |))
+    STDERR.puts(display(changes)) if confirm or not quiet
+    perform = if confirm and not changes.empty?
+                confirm_dialogue
+              else
+                true
+              end
+    rejected(aws.apply(changes)) if perform
+  when 'ec2/r53'
+    suffix = ARGV[1]
+    check_name(suffix)
+    new_records = aws.ec2_zone
+    mappings = (ARGV[2..-1] or [])
+    mapped = normed_with_mappings(mappings, suffix, new_records)
+    _, old_records = aws.route53_zone(suffix)
+    changes = Zonify.diff(mapped, old_records, (types or %w| CNAME SRV |))
+    STDERR.puts(display(changes)) unless quiet
+    STDOUT.write(Zonify::YAML.trim_lines(YAML.dump(changes)))
+  when 'summarize'
+    handle = ARGV[1] ? File.open(ARGV[1]) : STDIN
+    changes = yaml_with_default(handle, [])
+    STDOUT.puts(display(changes))
+  when 'apply'
+    handle = ARGV[1] ? File.open(ARGV[1]) : STDIN
+    changes = yaml_with_default(handle, [])
+    STDERR.puts "" unless changes
+    STDERR.puts(display(changes)) if confirm or not quiet
+    perform = if confirm and not changes.empty?
+                confirm_dialogue
+              else
+                true
+              end
+    rejected(aws.apply(changes)) if perform
+  when 'rewrite'
+    suffix, records = Zonify::YAML.read(STDIN)
+    mappings = (ARGV[1..-1] or [])
+    abort 'No mappings given.' if mappings.empty?
+    mapped = normed_with_mappings(mappings, suffix, records, norm)
+    STDOUT.write(Zonify::YAML.format(mapped, suffix))
+  when 'normalize'
+    suffix, records = Zonify::YAML.read(STDIN)
+    desired_suffix = ARGV[1]
+    normed = if desired_suffix
+               normed_with_mappings([], desired_suffix, records)
+             else
+               Zonify.normalize(records)
+             end
+    STDOUT.write(Zonify::YAML.format(normed, (desired_suffix or suffix)))
+  else
+    abort 'Argument error.'
+  end
+end
+
+main
+

data/lib/zonify.rb

@@ -0,0 +1,582 @@
+require 'yaml'
+
+require 'fog'
+
+
+module Zonify
+
+# Set up for AWS interfaces and access to EC2 instance metadata.
+class AWS
+  class << self
+    # Retrieve the EC2 instance ID of this instance.
+    def local_instance_id
+      s = `curl -s http://169.254.169.254/latest/meta-data/instance-id`
+      s.strip if $?.success?
+    end
+    # Initialize all AWS interfaces with the same access keys and logger
+    # (probably what you want to do). These are set up lazily; unused
+    # interfaces will not be initialized.
+    def create(options)
+      options_ec2 = options.merge( :provider=>'AWS',
+                                   :connection_options=>{:nonblock=>false} )
+      ec2 = Proc.new{|| Fog::Compute.new(options_ec2) }
+      options_elb = options_ec2.dup.delete_if{|k, _| k == :provider }
+      elb = Proc.new{|| Fog::AWS::ELB.new(options_elb) }
+      options_r53 = options_ec2.dup.delete_if{|k, _| k == :region }
+      r53 = Proc.new{|| Fog::DNS.new(options_r53) }
+      Zonify::AWS.new(:ec2_proc=>ec2, :elb_proc=>elb, :r53_proc=>r53)
+    end
+  end
+  attr_reader :ec2_proc, :elb_proc, :r53_proc
+  def initialize(opts={})
+    @ec2      = opts[:ec2]
+    @elb      = opts[:elb]
+    @r53      = opts[:r53]
+    @ec2_proc = opts[:ec2_proc]
+    @elb_proc = opts[:elb_proc]
+    @r53_proc = opts[:r53_proc]
+  end
+  def ec2
+    @ec2 ||= @ec2_proc.call
+  end
+  def elb
+    @elb ||= @elb_proc.call
+  end
+  def r53
+    @r53 ||= @r53_proc.call
+  end
+  # Generate DNS entries based on EC2 instances, security groups and ELB load
+  # balancers under the user's AWS account.
+  def ec2_zone
+    Zonify.tree(Zonify.zone(instances, load_balancers))
+  end
+  # Retrieve Route53 zone data -- the zone ID as well as resource records --
+  # relevant to the given suffix. When there is any ambiguity, the zone with
+  # the longest name is chosen.
+  def route53_zone(suffix)
+    suffix_ = Zonify.dot_(suffix)
+    relevant_zone = r53.zones.select do |zone|
+      suffix_.end_with?(zone.domain)
+    end.sort_by{|zone| zone.domain.length }.last
+    if relevant_zone
+      relevant_records = relevant_zone.records.all!.map do |rr|
+        if rr.name.end_with?(suffix_)
+          rr.attributes.merge(:name=>Zonify.read_octal(rr.name))
+        end
+      end.compact
+      [relevant_zone, Zonify.tree_from_right_aws(relevant_records)]
+    end
+  end
+  # Apply a changeset to the records in Route53. The records must all be under
+  # the same zone and suffix.
+  def apply(changes, comment='Synced with Zonify tool.')
+    # Dumb way to do this because I can not figure out #reject!
+    keep = changes.select{|c| c[:value].length <= 100 }
+    filtered = changes.select{|c| c[:value].length > 100 }
+    unless keep.empty?
+      suffix  = keep.first[:name] # Works because of longest submatch rule.
+      zone, _ = route53_zone(suffix)
+      Zonify.chunk_changesets(keep).each do |changeset|
+        rekeyed = changeset.map do |record|
+          record.inject({}) do |acc, pair|
+            k, v = pair
+            k_ = k == :value ? :resource_records : k
+            acc[k_] = v
+            acc
+          end
+        end
+        r53.change_resource_record_sets(zone.id, rekeyed, :comment=>comment)
+      end
+    end
+    filtered
+  end
+  def instances
+    ec2.servers.inject({}) do |acc, i|
+      dns = i.dns_name
+      # The default hostname for EC2 instances is derived from their internal
+      # DNS entry.
+      unless dns.nil? or dns.empty?
+        groups = (i.groups or [])
+        attrs = { :sg => groups,
+                  :tags => (i.tags or []),
+                  :dns => Zonify.dot_(dns).downcase }
+        if i.private_dns_name
+          attrs[:priv] = i.private_dns_name.split('.').first.downcase
+        end
+        acc[i.id] = attrs
+      end
+      acc
+    end
+  end
+  def load_balancers
+    elb.load_balancers.map do |elb|
+      { :instances => elb.instances,
+        :prefix    => Zonify.cut_down_elb_name(elb.dns_name) }
+    end
+  end
+  def eips
+    ec2.addresses
+  end
+  def eip_scan
+    addresses = eips.map{|eip| eip.public_ip }
+    result = {}
+    addresses.each{|a| result[a] = [] }
+    r53.zones.sort_by{|zone| zone.domain.reverse }.each do |zone|
+      zone.records.all!.each do |rr|
+        check = case rr.type
+                when 'CNAME'
+                  rr.value.map do |s|
+                    Zonify.ec2_dns_to_ip(s)
+                  end.compact
+                when 'A','AAAA'
+                  rr.value
+                end
+        check ||= []
+        found = addresses.select{|a| check.member? a }.sort
+        unless found.empty?
+          name = Zonify.read_octal(rr.name)
+          found.each{|a| result[a] << name }
+        end
+      end
+    end
+    result
+  end
+end
+
+extend self
+
+
+module Resolve
+SRV_PREFIX = '_*._*'
+end
+
+# Records are all created with functions in this module, which ensures the
+# necessary SRV prefixes, uniform TTLs and final dots in names.
+module RR
+extend self
+  def srv(service, name)
+    { :type=>'SRV', :value=>"0 0 0 #{Zonify.dot_(name)}",
+      :ttl=>'100',  :name=>"#{Zonify::Resolve::SRV_PREFIX}.#{service}" }
+  end
+  def cname(name, dns, ttl='100')
+    { :type=>'CNAME', :value=>Zonify.dot_(dns),
+      :ttl=>ttl,      :name=>Zonify.dot_(name) }
+  end
+end
+
+# Given EC2 host and ELB data, construct unqualified DNS entries to make a
+# zone, of sorts.
+def zone(hosts, elbs)
+  host_records = hosts.map do |id,info|
+    name = "#{id}.inst."
+    priv = "#{info[:priv]}.priv."
+    [ Zonify::RR.cname(name, info[:dns], '86400'),
+      Zonify::RR.cname(priv, info[:dns], '86400'),
+      Zonify::RR.srv('inst.', name) ] +
+    info[:tags].map do |tag|
+      k, v = tag
+      next if k.nil? or v.nil? or k.empty? or v.empty?
+      tag_dn = "#{Zonify.string_to_ldh(v)}.#{Zonify.string_to_ldh(k)}.tag."
+      Zonify::RR.srv(tag_dn, name)
+    end.compact
+  end.flatten
+  elb_records = elbs.map do |elb|
+    running = elb[:instances].select{|i| hosts[i] }
+    name = "#{elb[:prefix]}.elb."
+    running.map{|host| Zonify::RR.srv(name, "#{host}.inst.") }
+  end.flatten
+  sg_records = hosts.inject({}) do |acc, kv|
+    id, info = kv
+    info[:sg].each do |sg|
+      acc[sg] ||= []
+      acc[sg]  << id
+    end
+    acc
+  end.map do |sg, ids|
+    sg_ldh = Zonify.string_to_ldh(sg)
+    name = "#{sg_ldh}.sg."
+    ids.map{|id| Zonify::RR.srv(name, "#{id}.inst.") }
+  end.flatten
+  [host_records, elb_records, sg_records].flatten
+end
+
+# Group DNS entries into a tree, with name at the top level, type at the
+# next level and then resource records and TTL at the leaves. If the records
+# are part of a weighted record set, then the record data is pushed down one
+# more level, with the "set identifier" in between the type and data.
+def tree(records)
+  records.inject({}) do |acc, record|
+    name, type, ttl, value,
+    weight, set           = [ record[:name],   record[:type],
+                              record[:ttl],    record[:value],
+                              record[:weight], record[:set_identifier] ]
+    reference = acc[name]       ||= {}
+    reference = reference[type] ||= {}
+    reference = reference[set]  ||= {} if set
+    appended                      = (reference[:value] or []) << value
+    reference[:ttl]               = ttl
+    reference[:value]             = appended.sort.uniq
+    reference[:weight]            = weight if weight
+    acc
+  end
+end
+
+# In the fully normalized tree of records, each multi-element SRV is
+# associated with a set of equally weighted CNAMEs, one for each record.
+# Singleton SRVs are associated with a single CNAME. All resource record lists
+# are sorted and deduplicated.
+def normalize(tree)
+  singles = Zonify.cname_singletons(tree)
+  merged = Zonify.merge(tree, singles)
+  remove, srvs = Zonify.srv_from_cnames(merged)
+  cleared = merged.inject({}) do |acc, pair|
+    name, info = pair
+    info.each do |type, data|
+      unless 'CNAME' == type and remove.member?(name)
+        acc[name] ||= {}
+        acc[name][type] = data
+      end
+    end
+    acc
+  end
+  stage2 = Zonify.merge(cleared, srvs)
+  multis = Zonify.cname_multitudinous(stage2)
+  stage3 = Zonify.merge(stage2, multis)
+end
+
+# For SRV records with a single entry, create a singleton CNAME as a
+# convenience.
+def cname_singletons(tree)
+  tree.inject({}) do |acc, pair|
+    name, info = pair
+    name_clipped = name.sub("#{Zonify::Resolve::SRV_PREFIX}.", '')
+    info.each do |type, data|
+      if 'SRV' == type and 1 == data[:value].length
+        rr_clipped = data[:value].map do |rr|
+          Zonify.dot_(rr.sub(/^([^ ]+ +){3}/, '').strip)
+        end
+        new_data = data.merge(:value=>rr_clipped)
+        acc[name_clipped] = { 'CNAME' => new_data }
+      end
+    end
+    acc
+  end
+end
+
+# Find CNAMEs with multiple records and create SRV records to replace them,
+# as well as returning the list of CNAMEs to replace.
+def srv_from_cnames(tree)
+  remove = []
+  srvs = tree.inject({}) do |acc, pair|
+    name, info = pair
+    name_srv = "#{Zonify::Resolve::SRV_PREFIX}.#{name}"
+    info.each do |type, data|
+      if 'CNAME' == type and 1 < data[:value].length
+        remove.push(name)
+        rr_srv = data[:value].map{|s| '0 0 0 ' + s }
+        acc[name_srv]      ||= { }
+        acc[name_srv]['SRV'] = { :ttl=>100, :value=>rr_srv }
+      end
+    end
+    acc
+  end
+  [remove, srvs]
+end
+
+# For every SRV record that is not a singleton and that does not shadow an
+# existing CNAME, we create WRRs for item in the SRV record.
+def cname_multitudinous(tree)
+  tree.inject({}) do |acc, pair|
+    name, info = pair
+    name_clipped = name.sub("#{Zonify::Resolve::SRV_PREFIX}.", '')
+    info.each do |type, data|
+      if 'SRV' == type and 1 < data[:value].length
+        wrrs = data[:value].inject({}) do |accumulator, rr|
+          server = Zonify.dot_(rr.sub(/^([^ ]+ +){3}/, '').strip)
+          id = server.split('.').first # Always the isntance ID.
+          accumulator[id] = data.merge(:value=>[server], :weight=>"16")
+          accumulator
+        end
+        acc[name_clipped] = { 'CNAME' => wrrs }
+      end
+    end
+    acc
+  end
+end
+
+# Collate RightAWS style records in to the tree format used by the tree method.
+def tree_from_right_aws(records)
+  records.inject({}) do |acc, record|
+    name, type, ttl, value,
+    weight, set           = [ record[:name],   record[:type],
+                              record[:ttl],    record[:value],
+                              record[:weight], record[:set_identifier] ]
+    reference = acc[name]       ||= {}
+    reference = reference[type] ||= {}
+    reference = reference[set]  ||= {} if set
+    reference[:ttl]               = ttl
+    reference[:value]             = (value or [])
+    reference[:weight]            = weight if weight
+    acc
+  end
+end
+
+# Merge all records from the trees, taking TTLs from the leftmost tree and
+# sorting and deduplicating resource records. (When called on a single tree,
+# this function serves to sort and deduplicate resource records.)
+def merge(*trees)
+  acc = {}
+  trees.each do |tree|
+    tree.inject(acc) do |acc, pair|
+      name, info     = pair
+      acc[name]    ||= {}
+      info.inject(acc[name]) do |acc_, pair_|
+        type, data = pair_
+        case
+        when (not acc_[type])
+          acc_[type] = data.dup
+        when (not acc_[type][:value] and not data[:value]) # WRR records.
+          d = data.merge(acc_[type])
+          acc_[type] = d
+        else # Not WRR records.
+          acc_[type][:value] = (data[:value] + acc_[type][:value]).uniq.sort
+        end
+        acc_
+      end
+      acc
+    end
+  end
+  acc
+end
+
+# Old records that have the same elements as new records should be left as is.
+# If they differ in any way, they should be marked for deletion and the new
+# record marked for creation. Old records not in the new records should also
+# be marked for deletion.
+def diff(new_records, old_records, types=['CNAME','SRV'])
+  create_set = new_records.map do |name, v|
+    old = old_records[name]
+    v.map do |type, data|
+      if types.member? '*' or types.member? type
+        old_data = ((old and old[type]) or {})
+        unless Zonify.compare_records(old_data, data)
+          Zonify.hoist(data, name, type, 'CREATE')
+        end
+      end
+    end.compact
+  end
+  delete_set = old_records.map do |name, v|
+    new = new_records[name]
+    v.map do |type, data|
+      if types.member? '*' or types.member? type
+        new_data = ((new and new[type]) or {})
+        unless Zonify.compare_records(data, new_data)
+          Zonify.hoist(data, name, type, 'DELETE')
+        end
+      end
+    end.compact
+  end
+  (delete_set.flatten + create_set.flatten).sort_by do |record|
+    # Sort actions so that creation of a record comes immediately after a
+    # deletion.
+    delete_first = record[:action] == 'DELETE' ? 0 : 1
+    [record[:name], record[:type], delete_first]
+  end
+end
+
+def hoist(data, name, type, action)
+  meta = {:name=>name, :type=>type, :action=>action}
+  if data[:value] # Not a WRR.
+    [data.merge(meta)]
+  else # Is a WRR.
+    data.map{|k,v| v.merge(meta.merge(:set_identifier=>k)) }
+  end
+end
+
+# Determine whether two resource record sets are the same in all respects
+# (keys missing in one should be missing in the other).
+def compare_records(a, b)
+  keys = ((a.keys | b.keys) - [:value]).sort_by{|s| s.to_s }
+  as, bs = [a, b].map do |record|
+    keys.map{|k| record[k] } << Zonify.normRRs(record[:value])
+  end
+  as == bs
+end
+
+# Sometimes, resource_records are a single string; sometimes, an array. The
+# array should be sorted for comparison's sake. Strings should be put in an
+# array.
+def normRRs(val)
+  case val
+  when Array then val.sort
+  else           [val]
+  end
+end
+
+def read_octal(s)
+  after = s
+  acc = ''
+  loop do
+    before, match, after = after.partition(/\\([0-9][0-9][0-9])/)
+    acc += before
+    break if match.empty?
+    acc << $1.oct
+  end
+  acc
+end
+
+ELB_DNS_RE = /^([a-z0-9-]+)-[^-.]+[.].+$/
+def cut_down_elb_name(s)
+  $1 if ELB_DNS_RE.match(s)
+end
+
+LDH_RE = /^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])$/
+def string_to_ldh_component(s)
+  LDH_RE.match(s) ? s.downcase : s.downcase.gsub(/[^a-z0-9-]/, '-').
+                                            sub(/(^[-]+|[-]+$)/, '')[0,63]
+end
+
+def string_to_ldh(s)
+  s.split('.').map{|s| string_to_ldh_component(s) }.join('.')
+end
+
+def _dot(s)
+  /^[.]/.match(s) ? s : ".#{s}"
+end
+
+def dot_(s)
+  /[.]$/.match(s) ? s : "#{s}."
+end
+
+EC2_DNS_RE = /^ec2-([0-9]+)-([0-9]+)-([0-9]+)-([0-9]+)
+               [.]compute-[0-9]+[.]amazonaws[.]com[.]?$/x
+def ec2_dns_to_ip(dns)
+  "#{$1}.#{$2}.#{$3}.#{$4}" if EC2_DNS_RE.match(dns)
+end
+
+module YAML
+extend self
+  def format(records, suffix='')
+   _suffix_ = Zonify._dot(Zonify.dot_(suffix))
+   entries = records.keys.sort.map do |k|
+     dumped = ::YAML.dump(k=>records[k])
+     Zonify::YAML.trim_lines(dumped).map{|ln| '  ' + ln }.join
+   end.join
+   "suffix: #{_suffix_}\nrecords:\n" + entries
+  end
+  def read(text)
+    yaml = ::YAML.load(text)
+    if yaml['suffix'] and yaml['records']
+      [yaml['suffix'], yaml['records']]
+    end
+  end
+  def trim_lines(yaml)
+   lines = yaml.lines.to_a
+   lines.shift if /^---/.match(lines[0])
+   lines.pop if /^$/.match(lines[-1])
+   lines
+  end
+end
+
+# The Route 53 API has limitations on query size:
+#
+#  - A request cannot contain more than 100 Change elements.
+#
+#  - A request cannot contain more than 1000 ResourceRecord elements.
+#
+#  - The sum of the number of characters (including spaces) in all Value
+#    elements in a request cannot exceed 32,000 characters.
+#
+def chunk_changesets(changes)
+  chunks = [[]]
+  changes.each do |change|
+    if fits(change, chunks.last)
+      chunks.last.push(change)
+    else
+      chunks.push([change])
+    end
+  end
+  chunks
+end
+
+def measureRRs(change)
+  [ change[:value].length,
+    change[:value].inject(0){|sum, s| s.length + sum } ]
+end
+
+# Determine whether we can add this record to the existing records, subject to
+# Amazon size constraints.
+def fits(change, changes)
+  new = changes + [change]
+  measured = new.map{|change| measureRRs(change) }
+  len, chars = measured.inject([0, 0]) do |acc, pair|
+    [ acc[0] + pair[0], acc[1] + pair[1] ]
+  end
+  new.length <= 100 and len <= 1000 and chars <= 30000 # margin of safety
+end
+
+module Mappings
+extend self
+  def parse(s)
+    k, *v = s.split(':')
+    [k, v] if k and v and not v.empty?
+  end
+  # Apply mappings to the name in order. (A hash can be used for mappings but
+  # then one will not be able to predict the order.) If no mappings apply, the
+  # empty list is returned.
+  def apply(name, mappings)
+    name_ = Zonify.dot_(name)
+    mappings.map do |k, v|
+      _k_ = Zonify.dot_(Zonify._dot(k))
+      before = Zonify::Mappings.unsuffix(name_, _k_)
+      v.map{|s| Zonify.dot_(before + Zonify._dot(s)) } if before
+    end.compact.flatten
+  end
+  # Get the names that result from the mappings, or the original name if none
+  # apply. The first name in the list is taken to be the canonical name, the
+  # one used for groups of servers in SRV records.
+  def names(name, mappings)
+    mapped = Zonify::Mappings.apply(name, mappings)
+    mapped.empty? ? [name] : mapped
+  end
+  def unsuffix(s, suffix)
+    before, _, after = s.rpartition(suffix)
+    before if after.empty?
+  end
+  def rewrite(tree, mappings)
+    tree.inject({}) do |acc, pair|
+      name, info = pair
+      names = Zonify::Mappings.names(name, mappings)
+      names.each do |name|
+        acc[name] ||= {}
+        info.inject(acc[name]) do |acc_, pair_|
+          type, data = pair_
+          acc_[type] ||= {}
+          prefix_ = Zonify.dot_(Zonify::Resolve::SRV_PREFIX)
+          rrs = if type == 'SRV' and name.start_with? prefix_ and data[:value]
+                  data[:value].map do |rr|
+                    if /^(.+) ([^ ]+)$/.match(rr)
+                      "#{$1} #{Zonify::Mappings.names($2, mappings).first}"
+                    else
+                      rr
+                    end
+                  end
+                end
+          addenda = rrs ? { :value => rrs + (acc_[type][:value] or []) } : {}
+          acc_[type] = data.merge(addenda)
+          acc_
+        end
+      end
+      acc
+    end
+  end
+end
+
+# Based on reading the Wikipedia page:
+#   http://en.wikipedia.org/wiki/List_of_DNS_record_types
+# and the IANA registry:
+#   http://www.iana.org/assignments/dns-parameters
+RRTYPE_RE = /^([*]|[A-Z0-9-]+)$/
+
+end
+

metadata

@@ -0,0 +1,85 @@
+--- !ruby/object:Gem::Specification 
+name: zonify
+version: !ruby/object:Gem::Version 
+  hash: 101
+  prerelease: 
+  segments: 
+  - 0
+  - 4
+  - 0
+  - 5
+  version: 0.4.0.5
+platform: ruby
+authors: 
+- AirBNB
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-09-27 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fog
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: |
+  Zonify provides a command line tool for generating DNS records from EC2
+  instances, instance tags, load balancers and security groups. A mechanism for
+  syncing these records with a zone stored in Route 53 is also provided.
+
+email: contact@airbnb.com
+executables: 
+- zonify
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/zonify.rb
+- README
+- bin/zonify
+homepage: https://github.com/airbnb/zonify
+licenses: 
+- BSD
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Generate DNS information from EC2 metadata.
+test_files: []
+