piculet 0.0.1

data/README.md

@@ -0,0 +1,124 @@
+# Piculet
+
+Piculet is a tool to manage Security Group.
+
+It defines the state of Security Group using DSL, and updates Security Group according to DSL.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'piculet'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install piculet
+
+## Usage
+
+```
+shell> export AWS_ACCESS_KEY_ID='...'
+shell> export AWS_SECRET_ACCESS_KEY='...'
+shell> export AWS_REGION='ap-northeast-1'
+shell> piculet -e -o Groupfile
+shell> vi Groupfile
+shell> piculet -a --dry-run
+shell> piculet -a
+```
+
+## Routefile example
+
+```ruby
+require 'other/groupfile'
+
+ec2 do
+  security_group "default" do
+    description "default group"
+
+    ingress do
+      permission :tcp, 0..65535 do
+        groups(
+          "default"
+        )
+      end
+      permission :udp, 0..65535 do
+        groups(
+          "default"
+        )
+      end
+      permission :icmp, -1..-1 do
+        groups(
+          "default"
+        )
+      end
+      permission :tcp, 22..22 do
+        ip_ranges(
+          "0.0.0.0/0"
+        )
+      end
+      permission :udp, 60000..61000 do
+        ip_ranges(
+          "0.0.0.0/0",
+        )
+      end
+    end
+  end
+end
+
+ec2 "vpc-XXXXXXXX" do
+  security_group "default" do
+    description "default VPC security group"
+
+    ingress do
+      permission :tcp, 22..22 do
+        ip_ranges(
+          "0.0.0.0/0",
+        )
+      end
+      permission :tcp, 80..80 do
+        ip_ranges(
+          "0.0.0.0/0"
+        )
+      end
+      permission :udp, 60000..61000 do
+        ip_ranges(
+          "0.0.0.0/0"
+        )
+      end
+      permission :any do
+        groups(
+          "any_other_group",
+          "default"
+        )
+      end
+    end
+
+    egress do
+      permission :any do
+        ip_ranges(
+          "0.0.0.0/0"
+        )
+      end
+    end
+  end
+
+  security_group "any_other_group" do
+    description "any_other_group"
+
+    egress do
+      permission :any do
+        ip_ranges(
+          "0.0.0.0/0"
+        )
+      end
+    end
+  end
+end
+```
+
+## Link
+* [RubyGems.org site](http://rubygems.org/gems/piculet)

data/bin/piculet

@@ -0,0 +1,124 @@
+#!/usr/bin/env ruby
+$: << File.expand_path("#{File.dirname __FILE__}/../lib")
+require 'rubygems'
+require 'piculet'
+require 'optparse'
+
+mode = nil
+file = 'Groupfile'
+output_file = '-'
+split = false
+
+options = {
+  :dry_run     => false,
+  :color       => true,
+  :debug       => false,
+}
+
+ARGV.options do |opt|
+  begin
+    access_key = nil
+    secret_key = nil
+
+    opt.on('-k', '--access-key ACCESS_KEY') {|v| access_key                   = v       }
+    opt.on('-s', '--secret-key SECRET_KEY') {|v| secret_key                   = v       }
+    opt.on('-a', '--apply')                 {|v| mode                         = :apply  }
+    opt.on('-f', '--file FILE')             {|v| file                         = v       }
+    opt.on('',   '--dry-run')               {|v| options[:dry_run]            = true    }
+    opt.on('-e', '--export')                {|v| mode                         = :export }
+    opt.on('-o', '--output FILE')           {|v| output_file                  = v       }
+    opt.on('',   '--split')                 {|v| split                        = true    }
+    opt.on('-t', '--test')                  {|v| mode                         = :test   }
+    opt.on(''  , '--no-color')              {    options[:color]              = false   }
+    opt.on(''  , '--debug')                 {    options[:debug]              = true    }
+    opt.parse!
+
+    if access_key and secret_key
+      AWS.config({
+        :access_key_id     => access_key,
+        :secret_access_key => secret_key,
+      })
+    elsif (access_key and !secret_key) or (!access_key and secret_key) or mode.nil?
+      puts opt.help
+      exit 1
+    end
+  rescue => e
+    $stderr.puts e
+    exit 1
+  end
+end
+
+String.colorize = options[:color]
+
+if options[:debug]
+  AWS.config({
+    :http_wire_trace => true,
+    :logger => Piculet::Logger.instance,
+  })
+end
+
+begin
+  logger = Piculet::Logger.instance
+  logger.set_debug(options[:debug])
+   client = Piculet::Client.new(options)
+
+  case mode
+  when :export
+    if split
+      logger.info('Export SecurityGroup')
+
+      output_file = 'Groupfile' if output_file == '-'
+      requires = []
+
+      client.export do |exported, converter|
+        exported.each do |vpc, security_groups|
+          group_file = File.join(File.dirname(output_file), "#{vpc || :classic}.group")
+          requires << group_file
+
+          logger.info("  write `#{group_file}`")
+
+          open(group_file, 'wb') do |f|
+            f.puts converter.call(vpc => security_groups)
+          end
+        end
+      end
+
+      logger.info("  write `#{output_file}`")
+
+      open(output_file, 'wb') do |f|
+        requires.each do |group_file|
+          f.puts "require '#{File.basename group_file}'"
+        end
+      end
+    else
+      if output_file == '-'
+        logger.info('# Export SecurityGroup')
+        puts client.export
+      else
+        logger.info("Export SecurityGroup to `#{output_file}`")
+        open(output_file, 'wb') {|f| f.puts client.export }
+      end
+    end
+  when :apply
+    unless File.exist?(file)
+      raise "No Groupfile found (looking for: #{file})"
+    end
+
+    msg = "Apply `#{file}` to SecurityGroup"
+    msg << ' (dry-run)' if options[:dry_run]
+    logger.info(msg)
+
+    updated = client.apply(file)
+
+    logger.info('No change'.intense_blue) unless updated
+  else
+    raise 'must not happen'
+  end
+rescue => e
+  if options[:debug]
+    raise e
+  else
+    $stderr.puts e
+    exit 1
+  end
+end

data/lib/piculet.rb

@@ -0,0 +1,3 @@
+require 'piculet/version'
+require 'piculet/client'
+require 'piculet/logger'

data/lib/piculet/client.rb

@@ -0,0 +1,148 @@
+require 'aws-sdk'
+require 'piculet/dsl'
+require 'piculet/exporter'
+require 'piculet/ext/ec2-owner-id-ext'
+require 'piculet/wrapper/ec2-wrapper'
+require 'piculet/logger'
+
+module Piculet
+  class Client
+    include Logger::ClientHelper
+
+    def initialize(options = {})
+      @options = OpenStruct.new(options)
+      @options.ec2 = AWS::EC2.new
+    end
+
+    def apply(file)
+      @options.ec2.owner_id
+      AWS.memoize { walk(file) }
+    end
+
+    def export
+      exported = AWS.memoize { Exporter.export(@options.ec2) }
+
+      if block_given?
+        converter = proc do |src|
+          DSL.convert(src, @options.ec2.owner_id)
+        end
+
+        yield(exported, converter)
+      else
+        DSL.convert(exported, @options.ec2.owner_id)
+      end
+    end
+
+    private
+    def load_file(file)
+      if file.kind_of?(String)
+        open(file) do |f|
+          DSL.define(f.read, file).result
+        end
+      elsif file.respond_to?(:read)
+        DSL.define(file.read, file.path).result
+      else
+        raise TypeError, "can't convert #{file} into File"
+      end
+    end
+
+    def walk(file)
+      dsl = load_file(file)
+
+      dsl_ec2s = dsl.ec2s
+      ec2 = EC2Wrapper.new(@options.ec2, @options)
+
+      aws_ec2s = collect_to_hash(ec2.security_groups, :has_many => true) do |item|
+        item.vpc? ? item.vpc_id : nil
+      end
+
+      dsl_ec2s.each do |vpc, ec2_dsl|
+        ec2_aws = aws_ec2s[vpc]
+
+        if ec2_aws
+          walk_ec2(vpc, ec2_dsl, ec2_aws, ec2.security_groups)
+        else
+          log(:warn, "EC2 `#{vpc || :classic}` is not found", :yellow)
+        end
+      end
+
+      ec2.updated?
+    end
+
+    def walk_ec2(vpc, ec2_dsl, ec2_aws, collection_api)
+      sg_list_dsl = collect_to_hash(ec2_dsl.security_groups, :name)
+      sg_list_aws = collect_to_hash(ec2_aws, :name)
+
+      sg_list_dsl.each do |key, sg_dsl|
+        name = key[0]
+        sg_aws = sg_list_aws.delete(key)
+
+        unless sg_aws
+          sg_aws = collection_api.create(name, :vpc => vpc, :description => sg_dsl.description)
+        end
+
+        walk_security_group(sg_dsl, sg_aws)
+      end
+
+      sg_list_aws.each do |key, sg_aws|
+        sg_aws.delete
+      end
+    end
+
+    def walk_security_group(security_group_dsl, security_group_aws)
+      unless security_group_aws.eql?(security_group_dsl)
+        security_group_aws.update(security_group_dsl)
+      end
+
+      walk_permissions(
+        security_group_dsl.ingress,
+        security_group_aws.ingress_ip_permissions)
+
+      if security_group_aws.vpc?
+        walk_permissions(
+          security_group_dsl.egress,
+          security_group_aws.egress_ip_permissions)
+      end
+    end
+
+    def walk_permissions(permissions_dsl, permissions_aws)
+      perm_list_dsl = collect_to_hash(permissions_dsl, :protocol, :port_range)
+      perm_list_aws = collect_to_hash(permissions_aws, :protocol, :port_range)
+
+      perm_list_dsl.each do |key, perm_dsl|
+        protocol, port_range = key
+        perm_aws = perm_list_aws.delete(key)
+
+        if perm_aws
+          unless perm_aws.eql?(perm_dsl)
+            perm_aws.update(perm_dsl)
+          end
+        else
+          permissions_aws.create(protocol, port_range, perm_dsl)
+        end
+      end
+
+      perm_list_aws.each do |key, perm_aws|
+        perm_aws.delete
+      end
+    end
+
+    def collect_to_hash(collection, *key_attrs)
+      options = key_attrs.last.kind_of?(Hash) ? key_attrs.pop : {}
+      hash = {}
+
+      collection.each do |item|
+        key = block_given? ? yield(item) : key_attrs.map {|k| item.send(k) }
+
+        if options[:has_many]
+          hash[key] ||= []
+          hash[key] << item
+        else
+          hash[key] = item
+        end
+      end
+
+      return hash
+    end
+  end # Client
+end # Piculet

data/lib/piculet/dsl.rb

@@ -0,0 +1,44 @@
+require 'ostruct'
+require 'piculet/dsl/ec2'
+require 'piculet/dsl/converter'
+
+module Piculet
+  class DSL
+    class << self
+      def define(source, path)
+        self.new(path) do
+          eval(source, binding)
+        end
+      end
+
+      def convert(exported, owner_id)
+        Converter.convert(exported, owner_id)
+      end
+    end # of class methods
+
+    attr_reader :result
+
+    def initialize(path, &block)
+      @path = path
+      @result = OpenStruct.new(:ec2s => {})
+      instance_eval(&block)
+    end
+
+    private
+    def require(file)
+      groupfile = File.expand_path(File.join(File.dirname(@path), file))
+
+      if File.exist?(groupfile)
+        instance_eval(File.read(groupfile))
+      elsif File.exist?(groupfile + '.rb')
+        instance_eval(File.read(groupfile + '.rb'))
+      else
+        Kernel.require(file)
+      end
+    end
+
+    def ec2(vpc = nil, &block)
+      @result.ec2s[vpc] = EC2.new(vpc, &block).result
+    end
+  end # DSL
+end # Piculet

data/lib/piculet/dsl/converter.rb

@@ -0,0 +1,120 @@
+module Piculet
+  class DSL
+    class Converter
+      class << self
+        def convert(exported, owner_id)
+          self.new(exported, owner_id).convert
+        end
+      end # of class methods
+
+      def initialize(exported, owner_id)
+        @exported = exported
+        @owner_id = owner_id
+      end
+
+      def convert
+        @exported.each.map {|vpc, security_groups|
+          output_ec2(vpc, security_groups)
+        }.join("\n")
+      end
+
+      private
+      def output_ec2(vpc, security_groups)
+        vpc = vpc ? vpc.inspect + ' ' : ''
+        security_groups = security_groups.map {|sg_id, sg|
+          output_security_group(sg_id, sg)
+        }.join("\n").strip
+
+        <<-EOS
+ec2 #{vpc}do
+  #{security_groups}
+end
+        EOS
+      end
+
+      def output_security_group(security_group_id, security_group)
+        name = security_group[:name].inspect
+        description = security_group[:description].inspect
+
+        ingress = security_group.fetch(:ingress, [])
+        egress = security_group.fetch(:egress, [])
+
+        ingress_egress = [
+          output_permissions(:ingress, ingress),
+          output_permissions(:egress, egress),
+        ].select {|i| i }
+
+        ingress_egress = ingress_egress.empty? ? '' : "\n\n    " + ingress_egress.join("\n").strip
+
+        <<-EOS
+  security_group #{name} do
+    description #{description}#{
+    ingress_egress}
+  end
+        EOS
+      end
+
+      def output_permissions(direction, permissions)
+        return nil if permissions.empty?
+        permissions = permissions.map {|i| output_perm(i) }.join.strip
+
+        <<-EOS
+    #{direction} do
+      #{permissions}
+    end
+        EOS
+      end
+
+      def output_perm(permission)
+        protocol = permission[:protocol]
+        port_range = permission[:port_range]
+        args = [protocol, port_range].select {|i| i }.map {|i| i.inspect }.join(', ') + ' '
+
+        ip_ranges = permission.fetch(:ip_ranges, [])
+        groups = permission.fetch(:groups, [])
+
+        ip_ranges_groups = [
+          output_ip_ranges(ip_ranges),
+          output_groups(groups),
+        ].select {|i| i }.join.strip
+
+        ip_ranges_groups.insert(0, "\n        ") unless ip_ranges_groups.empty?
+        
+        <<-EOS
+      permission #{args}do#{
+        ip_ranges_groups}
+      end
+        EOS
+      end
+
+      def output_ip_ranges(ip_ranges)
+        return nil if ip_ranges.empty?
+        ip_ranges = ip_ranges.map {|i| i.inspect }.join(",\n          ")
+
+        <<-EOS
+        ip_ranges(
+          #{ip_ranges}
+        )
+        EOS
+      end
+
+      def output_groups(groups)
+        return nil if groups.empty?
+
+        groups = groups.map {|i|
+          name_or_id = i[:name] || i[:id]
+          owner_id = i[:owner_id]
+
+          arg = @owner_id == owner_id ? name_or_id : [owner_id, i[:id]]
+          arg.inspect
+        }.join(",\n          ")
+
+        <<-EOS
+        groups(
+          #{groups}
+        )
+        EOS
+      end
+    end # Converter
+  end # DSL
+end # Piculet