reyes 0.2.4 → 0.3.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NmVjNDViYWE0OTViNWYwNTkzZWFlZjE2MjA1YzA3ZTQwM2RjZDU2Yg==
+    NzRmNzQ2MDEyZGUzZDZlNDZmNzlkMTM5MDkxN2UwZGYwYWI2MjFmOA==
   data.tar.gz: !binary |-
-    YTcxMGIwMDU4NWJlNzljMmE5MmYyNTdmZjY5YmFlMmQ3YzM2NjY2Yw==
+    YjFkZTIyM2JhNTM4ODgxMDgwN2JjYjNlZWEyYjM1OWQxN2E3ZmRlOA==
 SHA512:
   metadata.gz: !binary |-
-    ZjFiNWI1NDg4MGFhYjAzOTJjZTI4M2MxYTNhYTExYzgzOWVlNGNmYWY1OTM4
-    NjdlZGRjZTdiYTM0YmZmZDY2ZjYwMWE2NGNkMmJmOWRiZjA4ODAzM2E3MGUx
-    OTVhYzY0OGFlY2VhMjNhMDMyOTEwNGZkMGVhODFiY2Y2NWVjZjc=
+    MzYzNGY5ZThkMTJjYTI5NDM0ODgwZThlZGQwNTJlMDFjNzkwZjU3ZDM5Mjkw
+    NjE4ZGJlOTE0MGM4NmI0NjIyNjFkZDE5ZDY3YzBlOWY3Y2RkNGQxMDliMDdi
+    NDk3MDlkYmM1YzYxOWU3NGUxODNiYjc0YWFkNGIzMThiMzAwM2U=
   data.tar.gz: !binary |-
-    YmFkMGZlZWZjOTg1NWU0MmNlNDBkMTI2MzIxNmZmYTJlODVkOTBlYWUzYzIz
-    NzkzNjI4NzI3ODNlNDNjNjM4ZjNhMDdiNDQ4YzE3ZGIwNTExZjZkMjRlMTUw
-    YjQwYmQ5MWY1ZjU3ZjdhYjA1NDViOTMzMzVjNmJhODIzNzUxOTY=
+    NDVjZGM1NWFlODI2YzE4OTI2YjgwMDBiNTg1Yjg0MzY4YTc4NmFlOTRhODEy
+    Yjk3MWQyZmQ4YjkyMTk0OGUwZjczZTRhNGYyMjJkNDU0M2MyN2VkZmQxMDZk
+    ZGE3OTY0MjQxNjczYzRiMjkyNDA2YjgzMDUwNjVhMGRjMDc4M2U=

data/bin/reyes

@@ -7,8 +7,7 @@ def command_dump(output_file, options)
   aws.dump_fake_data(output_file)
 end
 
-def command_fetch(instance_id, options)
-  region = options.fetch(:region)
+def command_fetch(region, instance_id, options)
   aws = Reyes::AwsManager.new(options[:config])
   s3 = Reyes::S3Loader.new(aws, options[:config])
   wrapper = Reyes::PgpWrapper.new(options[:config])
@@ -28,9 +27,7 @@ def command_fetch(instance_id, options)
   end
 end
 
-def command_install(json_file, instance_id, options)
-  region = options.fetch(:region)
-
+def command_install(json_file, region, instance_id, options)
   if options[:splay]
     Reyes::Utils.sleep_random(options[:splay])
   end
@@ -75,7 +72,6 @@ end
 
 def parse_args
   options = {
-    :region => 'us-west-1', # TODO: make required
     :gen_options => {},
     :apply_options => {},
     :fetch_options => {},
@@ -91,7 +87,7 @@ Commands:
 
   install: load AWS data from JSON_FILE and install rules for INSTANCE_ID
 
-    #{File.basename($0)} [options] install JSON_FILE INSTANCE_ID
+    #{File.basename($0)} [options] install JSON_FILE EC2_REGION INSTANCE_ID
 
   dump: generate JSON from AWS and serialize data to JSON_FILE
 
@@ -99,20 +95,16 @@ Commands:
 
   fetch: load new rules from s3 and install them on the local system
 
-    #{File.basename($0)} [options] fetch INSTANCE_ID
+    #{File.basename($0)} [options] fetch EC2_REGION INSTANCE_ID
 
   upload: generate and sign a full dump of the rule inputs and upload them to s3
 
     #{File.basename($0)} [options] upload
 
 
-Defaults:
-  region: #{options.fetch(:region)}
-
-
 For example:
 
-  #{File.basename($0)} --prune install data.json $(facter -p ec2_instance_id)
+  #{File.basename($0)} --prune install data.json us-west-1 $(facter -p ec2_instance_id)
 
 
 Options:
@@ -122,10 +114,6 @@ Options:
       options[:config] = config
     end
 
-    opts.on('--region REGION', 'Set EC2 region') do |region|
-      options[:region] = region
-    end
-
     opts.on('--env ENV', 'Set SRFC4 env for dump') do |env|
       options[:tag_filters] = {'env' => env}
     end
@@ -163,6 +151,11 @@ Options:
       options[:apply_options][:log_drop] = arg
     end
 
+    opts.on('-v', '--version', 'Display version number and exit') do
+      puts "reyes version #{Reyes::VERSION}"
+      exit 0
+    end
+
     opts.on('-h', '--help', 'Display this help message') do
       STDERR.puts opts
       exit 0
@@ -180,18 +173,18 @@ Options:
     end
     command_dump(ARGV.fetch(0), options)
   when 'install'
-    unless ARGV.length == 2
+    unless ARGV.length == 3
       STDERR.puts optparse
       exit 2
     end
-    command_install(ARGV.fetch(0), ARGV.fetch(1), options)
+    command_install(ARGV.fetch(0), ARGV.fetch(1), ARGV.fetch(2), options)
   when 'fetch'
-    unless ARGV.length == 1
+    unless ARGV.length == 2
       STDERR.puts optparse
       exit 2
     end
 
-    command_fetch(ARGV.fetch(0), options)
+    command_fetch(ARGV.fetch(0), ARGV.fetch(1), options)
   when 'upload'
     unless ARGV.length == 0
       STDERR.puts optparse

data/config.yaml.example

@@ -11,10 +11,18 @@ aws:
   vpcs:
     - [us-west-2, vpc-abcdef12]
 
+  classic_cidr_blocks:
+    - 10.160.0.0/11
+    - 10.192.0.0/10
+
   s3:
     bucket: reyes-config
     path: rules.json
 
+  excluded_group_names:
+    - reyes
+    - qa-reyes
+
 reyes:
   pgp:
     signing_key: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

data/lib/reyes/aws_manager.rb

@@ -8,6 +8,21 @@ module Reyes
 
     include Chalk::Log
 
+    # Short names for AWS regions to save space in ipset names
+    RegionShortNames = {
+      'us-east-1' => 'VA',
+      'us-west-2' => 'OR',
+      'us-west-1' => 'CA',
+      'eu-west-1' => 'IE',
+      'eu-central-1' => 'DE',
+      'ap-southeast-1' => 'SG',
+      'ap-southeast-2' => 'AU',
+      'ap-northeast-1' => 'JP',
+      'sa-east-1' => 'BR',
+      'us-gov-west-1' => 'GV',
+      'cn-north-1' => 'CN',
+    }
+
     def self.with_retries(retries=5, delay=2)
       raise ArgumentError.new('Block is required') unless block_given?
       begin
@@ -115,21 +130,107 @@ module Reyes
       end
     end
 
+    # Generate AWS data suitable for offline rule processing.
+    #
+    # The data will include EC2 instance and security group information for
+    # each region. (see `#regions`)
+    #
+    # Generated DATA will be a hash including these keys:
+    #
+    # DATA:
+    #
+    #     'metadata' => hash including information about the generation process
+    #     'vpcs' => A mapping of VPC ID => VPC_DATA
+    #     'classic_cidr_blocks' => from config: list of EC2 classic CIDR blocks
+    #     'excluded_group_names' => from config: list of security group names
+    #       to ignore
+    #     'security_groups_by_name' => An index of global security groups by
+    #       name. See SG_GROUPS_BY_NAME below.
+    #     'regions' => A hash of {region_name => REGION_DATA}
+    #
+    #
+    # VPC_DATA: information about a VPC, including:
+    #
+    #     - region
+    #     - cidr_block
+    #
+    #
+    # SG_GROUPS_BY_NAME: a mapping of {security_group_name => SG_REFS}
+    #
+    #     some_group_name:
+    #       - region: us-east-1
+    #         group_id: sg-1234eeee
+    #         vpc: vpc-eeeeeeee
+    #       - region: us-west-1
+    #         group_id: sg-1234cccc
+    #         vpc: null
+    #
+    #
+    # REGION_DATA
+    #
+    #     'instances' => mapping of {instance_id => INSTANCE_DATA}
+    #     'security_groups' => mapping of {group_id => SG_DATA}
+    #
+    #
+    # INSTANCE_DATA: information about an instance, including:
+    #
+    #     - tags
+    #     - region
+    #     - vpc (ID)
+    #     - availability_zone
+    #     - private_ip_address
+    #     - public_ip_address
+    #     - security_groups (IDs)
+    #     - security_group_names
+    #
+    #
+    # SG_DATA: information about a security group, including:
+    #
+    #     - name
+    #     - description
+    #     - vpc (ID)
+    #     - region
+    #     - ipset_suffix (a name appropriate for an IPset)
+    #     - ingress_ip_permissions => list of IP_PERMISSION_DATA
+    #     - instances (IDs)
+    #
+    #
+    # IP_PERMISSION_DATA: information about an IP Permission, including:
+    #
+    #     - protocol
+    #     - port_start
+    #     - port_end
+    #     - ip_ranges => list of CIDR block strings
+    #     - group_names => list of security group names
+    #
+    #
+    # @return [Hash]
+    #
     def generate_fake_data
       log.info('Generating AWS data for serialization')
       start = Time.now.utc
       data = {
         'metadata' => {
+          'format_version' => Reyes::JSON_FORMAT_VERSION,
           'generated' => start,
           'generated_stamp' => start.to_i,
           'hostname' => Socket.gethostname,
           'pid' => Process.pid,
         },
-        'vpcs' => vpcs.map(&:id),
+        'vpcs' => {},
+        'classic_cidr_blocks' => aws_config.fetch('classic_cidr_blocks'),
+        'excluded_group_names' => aws_config.fetch('excluded_group_names'),
         'security_groups_by_name' => {},
         'regions' => {},
       }
 
+      vpcs.each do |vpc|
+        data['vpcs'][vpc.vpc_id] = {
+          'region' => vpc.client.config.ec2_region,
+          'cidr_block' => vpc.cidr_block,
+        }
+      end
+
       regions.each do |region|
         data['regions'][region] = fake_data_for_region(region)
       end
@@ -189,6 +290,7 @@ module Reyes
           data['instances'][i.instance_id] = {
             'tags' => i.tags.to_h.to_hash,
             'region' => region,
+            'vpc' => i.vpc_id,
             'availability_zone' => i.availability_zone,
             'private_ip_address' => i.private_ip_address,
             'public_ip_address' => i.public_ip_address,
@@ -248,7 +350,7 @@ module Reyes
     #
     def ipset_suffix_for_group(region, group)
       [
-        Reyes::GroupManager::RegionShortNames.fetch(region),
+        RegionShortNames.fetch(region),
         group.group_id,
         group.name,
       ].join(':')[0...31]

data/lib/reyes/fake_aws.rb

@@ -1,5 +1,4 @@
 require 'json'
-require 'set'
 
 module Reyes
   class FakeAws
@@ -9,6 +8,13 @@ module Reyes
     def initialize(data)
       @data = data
       log.info("Initialized FakeAws with metadata: #{metadata.inspect}")
+
+      version = metadata['format_version']
+      if version != Reyes::JSON_FORMAT_VERSION
+        log.error("WARNING: JSON format_version #{version.inspect} " \
+                  "differs from our version #{Reyes::JSON_FORMAT_VERSION}")
+        log.error('Proceeding anyway...')
+      end
     end
 
     def region_data(region)
@@ -47,11 +53,31 @@ module Reyes
       security_group(region, security_group_id).fetch('instances')
     end
 
-    def foreign_groups_by_name(group_name)
+    # Look up remote security group data. If `vpc_id` is nil, all VPC security
+    # groups will be returned. If `vpc_id` is given, all EC2 classic and all
+    # VPC security groups that aren't in the given VPC will be returned.
+    #
+    # @param group_name [String] The name of the groups to fetch.
+    # @param vpc_id [String, nil] The String VPC ID or nil if EC2 classic of
+    #   the current cluster.
+    #
+    # @return [Hash<Hash>] A mapping of group_id => group_data for relevant
+    #   security groups.
+    #
+    def foreign_groups_by_name(group_name, vpc_id)
       vpc_set = vpc_ids.to_set
       groups = {}
       @data.fetch('security_groups_by_name').fetch(group_name).each do |g|
-        next unless vpc_set.include?(g.fetch('vpc'))
+
+        group_vpc_id = g.fetch('vpc')
+
+        # skip local groups that are not foreign
+        # (groups with same VPC ID / VPCness)
+        next if group_vpc_id == vpc_id
+
+        # skip VPC groups that are not listed in our config
+        next if group_vpc_id && !vpc_set.include?(group_vpc_id)
+
         groups[g.fetch('group_id')] = security_group(g.fetch('region'),
                                                      g.fetch('group_id'))
       end
@@ -59,12 +85,45 @@ module Reyes
       groups
     end
 
-    def vpc_ids
+    def vpcs
       @data.fetch('vpcs')
     end
 
+    def vpcs_except(vpc_id)
+      unless vpc_id.is_a?(String)
+        raise ArgumentError.new("#{vpc_id.inspect} must be a String")
+      end
+      vpcs.find_all {|v, _| v != vpc_id }
+    end
+
+    def vpc_ids
+      @data.fetch('vpcs').keys
+    end
+
+    # We could actually calculate this data, but it makes assigning security
+    # group rules a little bit trickier, so these will be generated statically
+    # from config.
+    def ec2_classic_cidr_blocks
+      @data.fetch('classic_cidr_blocks')
+    end
+
+    # Return all EC2 classic and VPC CIDR blocks that are not in `self_vpc_id`.
+    #
+    # @param [String] self_vpc_id
+    #
+    # @return [Array<String>] A list of CIDR block strings.
+    #
+    def remote_cidr_blocks(self_vpc_id)
+      nets = vpcs_except(self_vpc_id).map {|v, data| data.fetch('cidr_block') }
+      return nets + ec2_classic_cidr_blocks
+    end
+
     def metadata
       @data.fetch('metadata')
     end
+
+    def excluded_group_names
+      @data.fetch('excluded_group_names')
+    end
   end
 end

data/lib/reyes/group_manager.rb

@@ -1,24 +1,11 @@
+require 'ipaddr'
+
 module Reyes
 
   # TODO: use a more precise name
   class GroupManager
     include Chalk::Log
 
-    # Short names for AWS regions to save space in ipset names
-    RegionShortNames = {
-      'us-east-1' => 'VA',
-      'us-west-2' => 'OR',
-      'us-west-1' => 'CA',
-      'eu-west-1' => 'IE',
-      'eu-central-1' => 'DE',
-      'ap-southeast-1' => 'SG',
-      'ap-southeast-2' => 'AU',
-      'ap-northeast-1' => 'JP',
-      'sa-east-1' => 'BR',
-      'us-gov-west-1' => 'GV',
-      'cn-north-1' => 'CN',
-    }
-
     ReyesInputChain = 'reyes-ipsec-input'
 
     attr_reader :fake_aws, :instance_id
@@ -36,9 +23,39 @@ module Reyes
       @instance_id = instance_id
     end
 
+    # Whether the self EC2 instance is in VPC.
+    #
+    # @return [Boolean]
+    #
+    def vpc?
+      !!vpc_id
+    end
+
+    # The VPC ID (or nil) of the self EC2 instance.
+    #
+    # @return [String, nil]
+    #
+    def vpc_id
+      our_instance.fetch('vpc')
+    end
+
+    # Look up data for this instance from FakeAws data.
+    #
     # @return [Hash]
-    def our_groups
-      fake_aws.security_groups_for_instance(@region, @instance_id)
+    #
+    def our_instance
+      fake_aws.instance(@region, @instance_id)
+    end
+
+    # @return [Hash]
+    def our_groups(skip_excluded=true)
+      data = fake_aws.security_groups_for_instance(@region, @instance_id)
+      if skip_excluded
+        exclude = fake_aws.excluded_group_names.to_set
+        data.reject {|g_id, g_data| exclude.include?(g_data.fetch('name')) }
+      else
+        data
+      end
     end
 
     def generate_rules_empty
@@ -50,7 +67,7 @@ module Reyes
 
     def load_from_s3(aws, config)
       s3 = S3Loader.new(aws, config)
-      data = s3.latest
+      s3.latest
     end
 
     # Given our instance ID and security group rules, generate IPTables rules
@@ -133,20 +150,6 @@ module Reyes
       end
     end
 
-    # TODO: delurk
-    def create_iptables_rules(data)
-      data.fetch(:groups).each do |cluster, items|
-        log.info "Creating rules for #{cluster}"
-        items.each do |item|
-          rules = Reyes::IPTables.generate_rules_from_hash(item)
-          rules.each do |rule|
-            log.info("  #{rule.cmd.join(" ")}")
-            rule.materialize
-          end
-        end
-      end
-    end
-
     # @return [Integer]
     def run_generation
       @generation.value
@@ -162,20 +165,6 @@ module Reyes
       @generation.increment!
     end
 
-    # TODO: delurk
-    # @param group [AWS::EC2::SecurityGroup]
-    #
-    # @return [String] A string title, at most 31 characters long
-    #
-    def ipset_name_for_group(group)
-      [
-        run_generation.to_s,
-        RegionShortNames.fetch(group.client.instance_variable_get(:@region)),
-        group.group_id,
-        group.name,
-      ].join(':')[0...31]
-    end
-
     # @param group_hash [Hash]
     #
     # @return [String] A string title, at most 31 characters long
@@ -209,17 +198,56 @@ module Reyes
     # @return [Array<String>] A list of private instance IP addresses
     #
     def addresses_for_group(region, group_id)
-      fake_aws.addresses_for_security_group(region, group_id)
+      fake_aws.addresses_for_security_group(region, group_id).reject(&:nil?)
     end
 
-    # Look up remote VPC security groups by name.
+    # Look up remote VPC / EC2 classic security groups by name.
     #
     # @param name [String]
     #
     # @return [Hash]
     #
+    # @see FakeAws#foreign_groups_by_name
+    #
     def foreign_groups_by_name(name)
-      fake_aws.foreign_groups_by_name(name)
+      fake_aws.foreign_groups_by_name(name, vpc_id)
+    end
+
+    # Generate a list of IPTables script lines that will inject traffic into
+    # the Reyes processing chain.
+    #
+    # In EC2 classic, traffic relevant to Reyes will be arriving directly
+    # through IPsec, so these rules will filter all IPsec traffic.
+    #
+    # In VPC, traffic relevant to Reyes may be forwarded by VPN servers and
+    # arrive from VPC CIDR blocks or EC2 classic CIDR blocks. All of this CIDR
+    # block information will be fetched from FakeAws data.
+    #
+    # @return [Array<String>]
+    #
+    def input_chain_rules
+      if vpc?
+        # filter all remote CIDR blocks through reyes
+        fake_aws.remote_cidr_blocks(vpc_id).map do |cidr|
+
+          # make sure cidr block can be parsed as an IPAddr
+          IPAddr.new(cidr)
+
+          # safeguard against accidentally including our own CIDR block
+          if cidr == fake_aws.vpcs.fetch(vpc_id).fetch('cidr_block')
+            log.error('Somehow remote_cidr_blocks includes our cidr_block')
+            log.error("Our VPC: #{vpc_id.inspect}")
+            log.error("Our CIDR block: #{cidr.inspect}")
+            raise ArgumentError.new(
+              "Refusing to filter CIDR block for self VPC")
+          end
+
+          "-A INPUT -s #{cidr} -j #{ReyesInputChain}"
+        end
+      else
+        # filter all ipsec tunneled traffic through reyes
+        ["-A INPUT -m policy --pol ipsec --dir in -j #{ReyesInputChain}"]
+      end
     end
 
     # @param [Hash] data
@@ -269,8 +297,13 @@ module Reyes
       lines << '-A reyes-accept -j reyes-log-accept' if log_accept
       lines << '-A reyes-accept -j ACCEPT'
 
-      # filter all ipsec tunneled traffic through reyes
-      lines << "-A INPUT -m policy --pol ipsec --dir in -j #{ReyesInputChain}"
+      # add rules to direct appropriate traffic into reyes
+      lines << ''
+      lines << '# input chain rules'
+      lines.concat(input_chain_rules)
+
+      lines << ''
+      lines << '# static global rules'
 
       # allow normal ICMP traffic
       IPTables.innocuous_icmp_rules(ReyesInputChain).each do |r|

data/lib/reyes/pgp_wrapper.rb

@@ -102,10 +102,7 @@ module Reyes
 
     def keyring_args
       [
-        '--no-default-keyring',
-        '--keyring', keyring_file('pubring.gpg'),
-        '--secret-keyring', keyring_file('secring.gpg'),
-        '--trustdb-name', keyring_file('trustdb.gpg'),
+        '--homedir', @keyring_directory
       ]
     end
 

data/lib/reyes/version.rb

@@ -1,3 +1,7 @@
 module Reyes
-  VERSION = '0.2.4' unless defined?(self::VERSION)
+  # The Reyes version number
+  VERSION = '0.3.1' unless defined?(self::VERSION)
+
+  # Number defining the JSON serialization format
+  JSON_FORMAT_VERSION = 2 unless defined?(self::JSON_FORMAT_VERSION)
 end

data/reyes.gemspec

@@ -41,4 +41,6 @@ Gem::Specification.new do |gem|
   gem.add_development_dependency 'pry'
   gem.add_development_dependency 'rake'
   gem.add_development_dependency 'rubocop'
+
+  gem.required_ruby_version = '>= 1.9.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reyes
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.1
 platform: ruby
 authors:
 - Andy Brody
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-26 00:00:00.000000000 Z
+date: 2015-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -142,7 +142,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='