reyes 0.0.4 → 0.0.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTNjMTNjOGU3ZmM0YTBhMWNhNjE1YTYxNjY5ODczYmJiNDRlZTc5YQ==
+    ZmI5ZGU3MDc0MDc1NWQ0ODUzYjNmZjc3MWI2Y2ZlZjdmZjE4YjY5Ng==
   data.tar.gz: !binary |-
-    MjdlNzBkMDBlY2I1MTkyZTJiNGY0ODBkOTI0MzVmZDNlNDkwM2I2Ng==
+    NzUxNTdlN2MwNTlmYWE4MGI5M2I1YjcyMzQxOWY1ZjhjYjY2ZTNiMg==
 SHA512:
   metadata.gz: !binary |-
-    MzI2MjVhMDFiMjBmZDZjYzFkMTVkMjMzZmZhMTU3NjE0NWMxNDU3OTY5N2U1
-    Y2E0YmFhY2E1ZDRlZmMzNTM3Yjk3NGIwNDFkZTk0MGZhZWYwNDgzOTlmZjc2
-    MWNmMjc2ZjBjNjJhNDA3YWQwYjdkYjM3NTBjNDhmNzNmZDM3N2E=
+    N2ViZDI2MjA0YjhjNWNiOWFkNGQzNmQwYzE2ZDQxYWUxNzI4NTI5NTUyNTQ2
+    MTQxYjBkZTk5NWJiOTQ2MjZmZGEzODViODk0N2UwZGY2YmZmZmE1NzdhMGVm
+    Njg0MTg4MmE2MjBhMWJhZTcwYTcwNzhiYjNlNTQ1ZGM0MTQwM2Y=
   data.tar.gz: !binary |-
-    ZTU5MTM0ODAwNDM5ZjkzNThhOWU1YzMxNTg0NDNmYzc5MzkzMjdmM2Y5YzFi
-    ZjJlZjFkYTMwNzFiMWJmZDhiYmU2ODAwMzRmMGJlMTE3Y2UxY2ExNmUyYWQ3
-    ZmNlZTUyZDVlMTM2ZTM0YzNiMDJkMDc2ODBmZTNkYmVkMTc4NzU=
+    YmJjYzVkNDEwOGJiYThhNTg3N2VhMmUzNTdlZDBjY2U1ZjRiZjNhMTA2ZWVi
+    YzkyY2Y1ZjgyYzM4NDYyOGYwZjUwMjAwOWEwNzIxYWJmNTg3NDI0ZWUxMzli
+    ZjRmODI5NGYwZjcxMWE5NjgzM2ZjYmQ0N2Y0NjRmNGE3MzYyZTU=

data/bin/reyes

@@ -2,16 +2,6 @@
 require 'optparse'
 require_relative '../lib/reyes'
 
-def command_dump(options)
-  instance_id = options.fetch(:instance_id)
-  region = options.fetch(:region)
-  g = Reyes::GroupManager.new(region, instance_id, options[:config])
-
-  AWS.memoize do
-    puts g.do_stuff.to_yaml
-  end
-end
-
 def command_install(options)
   instance_id = options.fetch(:instance_id)
   region = options.fetch(:region)
@@ -21,15 +11,17 @@ def command_install(options)
   end
 
   AWS.memoize do
-    g = Reyes::GroupManager.new(region, instance_id, options[:config])
+    aws = Reyes::AwsManager.new(options[:config])
+    g = Reyes::GroupManager.new(aws, region, instance_id, options[:config])
+    r = Reyes::RunManager.new(g)
 
     options[:run_options][:log_accept] = true # TODO
 
-    g.run!(options.fetch(:run_options))
+    r.run!(options.fetch(:run_options))
 
     if options[:prune]
-      g.prune_ipsets
-      # g.prune_iptables_rules
+      r.prune_ipsets
+      # Pruning IPTables rules doesn't make any sense, they are atomically replaced
     end
   end
 
@@ -93,8 +85,6 @@ Options:
   optparse.parse!
 
   case options[:command]
-  when :dump
-    command_dump(options)
   when :install
     command_install(options)
   else

data/lib/reyes/aws_manager.rb

@@ -77,6 +77,23 @@ module Reyes
       }.flatten
     end
 
+    # List instances in a given VPC security group. Use this method to get
+    # better cache behavior for repeated calls to list security group members
+    # in a VPC.
+    #
+    # @param sg [AWS::EC2::SecurityGroup]
+    #
+    # @return [Array<AWS::EC2::Instance>]
+    #
+    def instances_in_security_group(sg)
+      unless sg.vpc
+        raise ArgumentError.new("SG #{sg.security_group_id} is not in VPC")
+      end
+
+      sg.vpc.instances.find_all {|i| i.security_groups.include?(sg)}
+    end
+
+    # TODO: remove (probably not needed)
     def warm_sg_cache
       connections.fetch(:ec2).each_pair do |region, ec2|
         log.debug("Warming security group cache for #{region}")

data/lib/reyes/group_manager.rb

@@ -23,10 +23,11 @@ module Reyes
 
     attr_reader :aws
 
-    def initialize(region, instance_id, config_file=nil)
+    def initialize(aws, region, instance_id, config_file=nil, generation=nil)
       log.info("Initializing #{self.class.name} for #{region} #{instance_id}")
 
-      @aws = Reyes::AwsManager.new(config_file)
+      @aws = aws
+      @generation = generation || RunGeneration.new
       @instance_id = instance_id
       @instance = @aws.ec2(region).instances[instance_id]
     end
@@ -37,71 +38,6 @@ module Reyes
       }
     end
 
-    # @param [Hash] options
-    #
-    # @option options :empty [Boolean] (false) Generate an empty (default DROP)
-    #   rule sets without actually looking up security groups
-    # @option options :interactive [Boolean] (false) Whether to prompt for
-    #   confirmation before applying rules
-    # @option options :log_accept [Boolean] (false) Whether to log packets on
-    #   ACCEPT
-    #
-    def run!(options={})
-      options = {
-        empty: false,
-        interactive: false,
-        log_accept: false,
-      }.merge(options)
-
-      log_accept = options.fetch(:log_accept)
-
-      log.info("Starting iptables rule generation run!")
-
-      if options.fetch(:empty)
-        log.warn("Generating empty (default DROP) rule set")
-        data = generate_rules_empty
-      else
-        data = generate_rules
-      end
-
-      new_rules = generate_iptables_script_file(data, log_accept: log_accept)
-      new_ipsets = generate_ipsets(data)
-
-      show_iptables_diff(new_rules)
-      show_ipsets_diff(new_ipsets)
-
-      if options.fetch(:interactive)
-        puts 'Press enter to continue...'
-        STDIN.gets
-      end
-
-      materialize_ipsets(new_ipsets)
-      iptables_restore(new_rules)
-
-      # XXX(richo) Should we be pruning inside run! ?
-      log.info('Finished firewall configuration run')
-    end
-
-    def show_iptables_diff(new_rules)
-      diff = Diff.new
-      diff.old.puts(Subprocess.check_output(%w{iptables-save}))
-      diff.new.puts(new_rules)
-
-      log.info "Proposed IPTables diff:"
-      puts diff.diff
-    end
-
-    def iptables_restore(new_rules)
-      log.info("Restoring #{new_rules.count("\n")} lines of iptables rules")
-
-      log.info('+ iptables-restore')
-      Subprocess.check_call(['iptables-restore'],
-                            stdin: Subprocess::PIPE) do |p|
-        p.communicate(new_rules)
-      end
-      log.info('restored')
-    end
-
     def generate_rules_empty
       {:groups => {}, :ipsets => {}}
     end
@@ -119,7 +55,7 @@ module Reyes
     # any changes to the system beyond incrementing the run generation (used
     # for ipset garbage collection).
     #
-    def generate_rules
+    def generate_rules!
       run_generation_increment!
       log.info("Generating rules for generation #{run_generation}")
 
@@ -188,31 +124,6 @@ module Reyes
       end
     end
 
-    def show_ipsets_diff(new_ipsets)
-      diff = Diff.new
-
-      dump = lambda do |f, ipset|
-        ipset.sort_by(&:name).each do |ip|
-          f.puts(ip.name)
-          ip.members.sort.each do |m|
-            f.puts("\t#{m}")
-          end
-        end
-      end
-
-      dump.call(diff.old, Reyes::IPSet.load_all)
-      dump.call(diff.new, new_ipsets)
-
-      log.info "Proposed IPSets diff:"
-      puts diff.diff
-    end
-
-    def materialize_ipsets(new_ipsets)
-      new_ipsets.each do |ipset|
-        ipset.build
-      end
-    end
-
     # TODO: delurk
     def create_iptables_rules(data)
       data.fetch(:groups).each do |cluster, items|
@@ -227,53 +138,13 @@ module Reyes
       end
     end
 
-    def prune_ipsets
-      log.info('Pruning old IPSets')
-      old_ipsets = []
-      current_ipsets = []
-
-      current_gen = run_generation
-      Reyes::IPSet.load_all.each do |set|
-        s = Reyes::GroupManager.parse_ipset_name(set.name)
-        unless s
-          log.warn("Skipping unparseable ipset name #{set.name.inspect}")
-          next
-        end
-
-        if s[:generation] < current_gen
-          old_ipsets << set
-        elsif s[:generation] == current_gen
-          current_ipsets << set
-        else
-          log.error("IPSet from a future generation detected: #{set.inspect}")
-          log.error("Cowardly refusing to proceed")
-          raise Reyes::Error.new("IPSet from future generation detected")
-        end
-      end
-
-      if current_ipsets.empty?
-        log.error("No IPSets from current generation.")
-        log.error("Cowardly refusing to proceed")
-        raise Reyes::Error.new("Pruning would remove all IPSets")
-      end
-
-      old_ipsets.each do |set|
-        log.info("Pruning IPSet: #{set.name}")
-        set.drop!
-      end
-
-      log.info('Done pruning old IPSets')
-    end
-
     # @return [Integer]
     def run_generation
-      @generation ||= Reyes::RunGeneration.new
       @generation.value
     end
 
     # Increment the run generation and persist it to disk
     def run_generation_increment!
-      run_generation
       @generation.increment!
     end
 
@@ -309,7 +180,9 @@ module Reyes
     #
     def addresses_for_group(group)
       groups = foreign_groups_by_name(group.name)
-      groups.map { |g| g.instances.map(&:private_ip_address) }.flatten
+      groups.map { |g|
+        @aws.instances_in_security_group(g).map(&:private_ip_address)
+      }.flatten
     end
 
     # Look up remote VPC security groups by name.
@@ -329,7 +202,9 @@ module Reyes
     # @option options [Boolean] log_drop (true)
     # @option options [Boolean] log_accept (false)
     #
-    def generate_iptables_script_file(data, options={})
+    # @return [String]
+    #
+    def generate_iptables_script(data, options={})
       log.info("Generating script for iptables-restore")
 
       options = {

data/lib/reyes/run_manager.rb

@@ -0,0 +1,138 @@
+module Reyes
+  class RunManager
+    include Chalk::Log
+
+    def initialize(group_manager)
+      @group_manager = group_manager
+    end
+
+    # @param [Hash] options
+    #
+    # @option options :empty [Boolean] (false) Generate an empty (default DROP)
+    #   rule sets without actually looking up security groups
+    # @option options :interactive [Boolean] (false) Whether to prompt for
+    #   confirmation before applying rules
+    # @option options :log_accept [Boolean] (false) Whether to log packets on
+    #   ACCEPT
+    #
+    def run!(options={})
+      options = {
+        empty: false,
+        interactive: false,
+        log_accept: false,
+      }.merge(options)
+
+      log_accept = options.fetch(:log_accept)
+
+      log.info("Starting iptables rule generation run!")
+
+      if options.fetch(:empty)
+        log.warn("Generating empty (default DROP) rule set")
+        data = @group_manager.generate_rules_empty
+      else
+        data = @group_manager.generate_rules!
+      end
+
+      new_rules = @group_manager.generate_iptables_script(data, log_accept: log_accept)
+      new_ipsets = @group_manager.generate_ipsets(data)
+
+      show_iptables_diff(new_rules)
+      show_ipsets_diff(new_ipsets)
+
+      if options.fetch(:interactive)
+        puts 'Press enter to continue...'
+        STDIN.gets
+      end
+
+      materialize_ipsets(new_ipsets)
+      iptables_restore(new_rules)
+
+      # XXX(richo) Should we be pruning inside run! ?
+      log.info('Finished firewall configuration run')
+    end
+
+    def materialize_ipsets(new_ipsets)
+      new_ipsets.each do |ipset|
+        ipset.build
+      end
+    end
+
+    def iptables_restore(new_rules)
+      log.info("Restoring #{new_rules.count("\n")} lines of iptables rules")
+
+      log.info('+ iptables-restore')
+      Subprocess.check_call(['iptables-restore'],
+                            stdin: Subprocess::PIPE) do |p|
+        p.communicate(new_rules)
+      end
+      log.info('restored')
+    end
+
+
+    def prune_ipsets
+      log.info('Pruning old IPSets')
+      old_ipsets = []
+      current_ipsets = []
+
+      current_gen = @group_manager.run_generation
+      Reyes::IPSet.load_all.each do |set|
+        s = Reyes::GroupManager.parse_ipset_name(set.name)
+        unless s
+          log.warn("Skipping unparseable ipset name #{set.name.inspect}")
+          next
+        end
+
+        if s[:generation] < current_gen
+          old_ipsets << set
+        elsif s[:generation] == current_gen
+          current_ipsets << set
+        else
+          log.error("IPSet from a future generation detected: #{set.inspect}")
+          log.error("Cowardly refusing to proceed")
+          raise Reyes::Error.new("IPSet from future generation detected")
+        end
+      end
+
+      if current_ipsets.empty?
+        log.error("No IPSets from current generation.")
+        log.error("Cowardly refusing to proceed")
+        raise Reyes::Error.new("Pruning would remove all IPSets")
+      end
+
+      old_ipsets.each do |set|
+        log.info("Pruning IPSet: #{set.name}")
+        set.drop!
+      end
+
+      log.info('Done pruning old IPSets')
+    end
+
+    def show_iptables_diff(new_rules)
+      diff = Diff.new
+      diff.old.puts(Subprocess.check_output(%w{iptables-save}))
+      diff.new.puts(new_rules)
+
+      log.info "Proposed IPTables diff:"
+      puts diff.diff
+    end
+
+    def show_ipsets_diff(new_ipsets)
+      diff = Diff.new
+
+      dump = lambda do |f, ipset|
+        ipset.sort_by(&:name).each do |ip|
+          f.puts(ip.name)
+          ip.members.sort.each do |m|
+            f.puts("\t#{m}")
+          end
+        end
+      end
+
+      dump.call(diff.old, Reyes::IPSet.load_all)
+      dump.call(diff.new, new_ipsets)
+
+      log.info "Proposed IPSets diff:"
+      puts diff.diff
+    end
+  end
+end

data/lib/reyes/set_manager.rb

@@ -0,0 +1,15 @@
+module Reyes
+  class SetPopulator
+    include Chalk::Log
+
+    attr_reader :aws
+
+    def initialize(region, instance_id)
+      @aws = Reyes::AwsManager.new
+      @instance_id = instance_id
+      @instance = @aws.ec2(region).instances[instance_id]
+      @our_groups = @instance.security_groups
+    end
+
+  end
+end

data/lib/reyes/version.rb

@@ -1,3 +1,3 @@
 module Reyes
-  VERSION = '0.0.4' unless defined?(self::VERSION)
+  VERSION = '0.0.5' unless defined?(self::VERSION)
 end

data/lib/reyes.rb

@@ -20,3 +20,4 @@ require_relative './reyes/ipset'
 require_relative './reyes/iptables'
 require_relative './reyes/run_generation'
 require_relative './reyes/utils'
+require_relative './reyes/run_manager'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reyes
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Andy Brody
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-17 00:00:00.000000000 Z
+date: 2015-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -122,6 +122,8 @@ files:
 - lib/reyes/ipset.rb
 - lib/reyes/iptables.rb
 - lib/reyes/run_generation.rb
+- lib/reyes/run_manager.rb
+- lib/reyes/set_manager.rb
 - lib/reyes/utils.rb
 - lib/reyes/version.rb
 - reyes.gemspec