reyes 0.1.1 → 0.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NGE0MzE5OTBmNzQ2YWI0YmMxYmJlZGMxMDgxMGEwNDJiOTgzNTg0Mg==
+    ZTZhZTI3NWU5Yjk3ZWE1NWI4MzlkYjVjMzg1NDg0NGYwNTAxMzU0OA==
   data.tar.gz: !binary |-
-    ZTY4MWFiMmNmMzg5MzEwNDBlYTc4MWUwZGVmYWZhMWYyOGMyZjI3YQ==
+    NzE1OGRiMzYxMWE2OWQwNDdhNDc4ZGFhYThlMTZmZDMyMGI1MGQ4NQ==
 SHA512:
   metadata.gz: !binary |-
-    ZTczYjRhNGIzZjZjMDViZjRlZDg4ODc2ZTNhYTQ5YjA4MjBiMDFmMjY5NWIw
-    NDE5M2ZmYzZkNDgzMDlmNjVlZTcwYjIyMGI2ODdiNzQ0MDg0ZWJmN2M3Y2Qy
-    MWJhYzljYTM5YzljNDYyMDI2MmQyYTZjODExZDUwZWM1YTFjNjg=
+    Y2NhNTZjZDI3NTIzOGFhMzJmY2EyY2JhNjdhZWE1OTQ1OTZlM2Q2ZjBhOGUy
+    MzQ5NWRjYzAzNjk2NGZiZTFkYjI5MDczOGZhODA0NTZmYmZmZTZjN2E3ZjM4
+    NmIyZGU4YzBjZTI5ZDY0MTVjODI4NTY4ZGJkNjc0ZjYyYWVhZDE=
   data.tar.gz: !binary |-
-    OWRhNjJjMWI4YTFjN2UwYmNkNWQxZjllZTRkMDM5ZGM5MWM0YzA0MzQyMDVm
-    ZDMzZDE2ZTc0ZjE2NzEyOWM1NzhmZjYyNzlmNmYwZWZkMzYzZWMyMjE4OWM5
-    M2MzM2FiYTRiMDZmZjUwZmQyYjk4MjQyOGFkOThjZGVmYjYwZTk=
+    YjE1MjliYjFmMWJhMmI2YTdlZmQzMGMwYmIyOTg0NWZkMjIxYTE3ZjRhNTky
+    YTNjYWU5OGYyZDhiZjU1ZDcyOGEzNDZlMjUyNWU5MTEzNjI5MzYwNmE3MmFi
+    NmMwOWNmNTQ1NjVhMjRlYmQyYmYwN2I1ZmNiNjlmYjllMWM4OGU=

data/bin/reyes

@@ -7,6 +7,27 @@ def command_dump(output_file, options)
   aws.dump_fake_data(output_file)
 end
 
+def command_fetch(instance_id, options)
+  region = options.fetch(:region)
+  aws = Reyes::AwsManager.new(options[:config])
+  s3 = Reyes::S3Loader.new(aws, options[:config])
+  wrapper = Reyes::PgpWrapper.new(options[:config])
+
+  armoured_rules = s3.fetch_rules
+  cleartext_rules = wrapper.verify!(armoured_rules)
+
+  fake = Reyes::FakeAws.new(JSON.load cleartext_rules)
+  g = Reyes::GroupManager.new(fake, region, instance_id)
+  r = Reyes::RunManager.new(g)
+
+  data = r.generate_data!(options.fetch(:gen_options))
+  r.apply_data!(data, options.fetch(:apply_options))
+
+  if options[:prune]
+    r.prune_ipsets
+  end
+end
+
 def command_install(json_file, instance_id, options)
   region = options.fetch(:region)
 
@@ -14,8 +35,16 @@ def command_install(json_file, instance_id, options)
     Reyes::Utils.sleep_random(options[:splay])
   end
 
-  fake = Reyes::FakeAws.new(File.open(json_file, 'r'))
+  data = JSON.load(File.open(json_file, 'r'))
+  generated_time = data.fetch("metadata").fetch("generated_stamp")
+  fake = Reyes::FakeAws.new(data)
   g = Reyes::GroupManager.new(fake, region, instance_id)
+
+  if Time.new(generated_time) > g.run_generation_time
+    # FIXME Deal gracefully with this case
+    # Should bail out, exiting nonzero if there were changes to commit
+  end
+
   r = Reyes::RunManager.new(g)
 
   data = r.generate_data!(options.fetch(:gen_options))
@@ -26,6 +55,18 @@ def command_install(json_file, instance_id, options)
   end
 end
 
+def command_upload(options)
+  aws = Reyes::AwsManager.new(options[:config])
+  wrapper = Reyes::PgpWrapper.new(options[:config])
+
+  data = aws.generate_fake_data_json
+
+  armoured_data = wrapper.clearsign(data)
+
+  s3 = Reyes::S3Loader.new(aws, options[:config])
+  s3.upload_rules(armoured_data)
+end
+
 def parse_args
   options = {
     :region => 'us-west-1', # TODO: make required
@@ -33,6 +74,7 @@ def parse_args
     :apply_options => {
       :log_accept => true, # TODO remove this default
     },
+    :fetch_options => {},
   }
 
   optparse = OptionParser.new do |opts|
@@ -51,6 +93,14 @@ Commands:
 
     #{File.basename($0)} [options] dump JSON_FILE
 
+  fetch: load new rules from s3 and install them on the local system
+
+    #{File.basename($0)} [options] fetch INSTANCE_ID
+
+  upload: generate and sign a full dump of the rule inputs and upload them to s3
+
+    #{File.basename($0)} [options] upload
+
 
 Defaults:
   region: #{options.fetch(:region)}
@@ -127,6 +177,21 @@ Options:
       exit 2
     end
     command_install(ARGV.fetch(0), ARGV.fetch(1), options)
+  when 'fetch'
+    unless ARGV.length == 1
+      STDERR.puts optparse
+      exit 2
+    end
+
+    command_fetch(ARGV.fetch(0), options)
+  when 'upload'
+    unless ARGV.length == 0
+      STDERR.puts optparse
+      exit 2
+    end
+
+    command_upload(options)
+
   else
     STDERR.puts optparse
     STDERR.puts "\nError: Must provide a command"

data/config.yaml.example

@@ -11,3 +11,10 @@ aws:
   vpcs:
     - [us-west-2, vpc-abcdef12]
 
+  s3:
+    bucket: reyes-config
+    path: rules.json
+
+reyes:
+  pgp:
+    signing_key: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA

data/lib/reyes.rb

@@ -22,3 +22,5 @@ require_relative './reyes/iptables'
 require_relative './reyes/run_generation'
 require_relative './reyes/run_manager'
 require_relative './reyes/utils'
+require_relative './reyes/s3_loader'
+require_relative './reyes/pgp_wrapper'

data/lib/reyes/aws_manager.rb

@@ -32,6 +32,14 @@ module Reyes
       connections.fetch(:ec2).fetch(region)
     end
 
+    def s3
+      @s3 ||= AWS::S3.new({
+        access_key_id: @config.aws_credentials.fetch(:access_key_id),
+        secret_access_key: @config.aws_credentials.fetch(:secret_access_key),
+        logger: Chalk::Log::Logger.new("s3"),
+      })
+    end
+
     def connections
       @connections ||= connect!
     end
@@ -145,6 +153,10 @@ module Reyes
       data
     end
 
+    def generate_fake_data_json
+      JSON.pretty_generate(generate_fake_data)
+    end
+
     def dump_fake_data(filename)
       log.info("Dumping AWS data to #{filename.inspect}")
       data = generate_fake_data

data/lib/reyes/config.rb

@@ -14,6 +14,10 @@ module Reyes
       aws_config.fetch('credentials')
     end
 
+    def reyes_config
+      config.fetch('reyes')
+    end
+
     def config
       @config ||= YAML.load_file(@path)
     end

data/lib/reyes/fake_aws.rb

@@ -5,11 +5,10 @@ module Reyes
   class FakeAws
     include Chalk::Log
 
-    # @param json_source
-    def initialize(json_source)
-      log.info('Loading JSON data')
-      @data = JSON.load(json_source)
-      log.info("Loaded JSON with metadata: #{metadata.inspect}")
+    # @param data [Hash]
+    def initialize(data)
+      @data = data
+      log.info("Initialized FakeAws with metadata: #{metadata.inspect}")
     end
 
     def region_data(region)

data/lib/reyes/group_manager.rb

@@ -48,6 +48,11 @@ module Reyes
       }
     end
 
+    def load_from_s3(aws, config)
+      s3 = S3Loader.new(aws, config)
+      data = s3.latest
+    end
+
     # Given our instance ID and security group rules, generate IPTables rules
     # needed to emulate security group behavior for foreign VPCs.
     #
@@ -147,6 +152,11 @@ module Reyes
       @generation.value
     end
 
+    # @return [Time]
+    def run_generation_time
+      @generation.mtime
+    end
+
     # Increment the run generation and persist it to disk
     def run_generation_increment!
       @generation.increment!

data/lib/reyes/pgp_wrapper.rb

@@ -0,0 +1,73 @@
+module Reyes
+  class PgpWrapper
+    class VerificationFailed < StandardError
+    end
+
+    attr_reader :key_id, :keyring_directory
+
+    # Create a PgpVerifier
+    #
+    # @param key_id [String] 40 digit key fingerprint
+    def initialize(config_path=nil)
+      @config ||= Reyes::Config.new(config_path)
+      @key_id = @config.reyes_config.fetch("signing_key").upcase
+      @keyring_directory = @config.reyes_config.fetch("keyring_directory")
+    end
+
+    # Pattern for parsing gpg --status-fd signature output
+    PATTERN = /^\[GNUPG:\] VALIDSIG ([A-F0-9]{40}) .+ ([A-F0-9]{40})$/
+
+    # Verifies +data+ against +@keyid+
+    #
+    # @param data [String] the data to verify
+    #
+    # @raise [VerificationFailed]
+    #
+    # @return [String] the stripped cleartext data
+    #
+    def verify!(data)
+      gpg_cmd = %w{gpg --batch --decrypt --status-fd 2} + keyring_args + ['-']
+      Subprocess.check_call(gpg_cmd,
+                            :stdin => Subprocess::PIPE,
+                            :stdout => Subprocess::PIPE,
+                            :stderr => Subprocess::PIPE) do |child|
+        out, err = child.communicate(data)
+
+        if err =~ PATTERN
+          raise VerificationFailed.new("Bad key match") unless $1 == $2
+          raise VerificationFailed.new("Bad Key ID") unless $1 == key_id
+        else
+          raise VerificationFailed.new("Pattern does not match")
+        end
+
+        # Sig looks ok
+       return out
+      end
+    end
+
+    def clearsign(data)
+      gpg_cmd = %W{gpg --batch --clearsign -u #{key_id}} + keyring_args + ['-']
+      Subprocess.check_call(gpg_cmd,
+                            :stdin => Subprocess::PIPE,
+                            :stdout => Subprocess::PIPE) do |child|
+        out, _ = child.communicate(data)
+        return out
+      end
+    end
+
+    private
+
+    def keyring_args
+      [
+        '--no-default-keyring',
+        '--keyring', keyring_file('pubring.gpg'),
+        '--secret-keyring', keyring_file('secring.gpg'),
+        '--trustdb-name', keyring_file('trustdb.gpg'),
+      ]
+    end
+
+    def keyring_file(name)
+      File.join(@keyring_directory, name)
+    end
+  end
+end

data/lib/reyes/run_generation.rb

@@ -8,6 +8,7 @@ module Reyes
     DefaultGenerationPath = "/tmp/reyes.u#{Process.euid}.generation"
 
     attr_reader :value
+    attr_reader :mtime
 
     # @param path [String] Path to generation file
     #
@@ -52,8 +53,10 @@ module Reyes
       @fh = open(@path)
       if @fh.size == 0
         @value = 0
+        @mtime = Time.at(0)
       else
         @value = load_value
+        @mtime = @fh.mtime
       end
       log.debug("Loaded generation value #{@value.inspect}")
     end

data/lib/reyes/s3_loader.rb

@@ -0,0 +1,32 @@
+module Reyes
+  class S3Loader
+
+    include Chalk::Log
+
+    def initialize(aws, config)
+      @aws = aws
+      @config = Reyes::Config.new(config)
+
+      log.info("Initialized S3Loader: #{bucket.inspect}/#{path.inspect}")
+    end
+
+    def fetch_rules
+      @aws.s3.buckets[bucket].objects[path].read
+    end
+
+    def upload_rules(data)
+      @aws.s3.buckets[bucket].objects[path].write(data)
+    end
+
+  private
+
+    def bucket
+      @config.aws_config.fetch("s3").fetch("bucket")
+    end
+
+    def path
+      @config.aws_config.fetch("s3").fetch("path")
+    end
+
+  end
+end

data/lib/reyes/version.rb

@@ -1,3 +1,3 @@
 module Reyes
-  VERSION = '0.1.1' unless defined?(self::VERSION)
+  VERSION = '0.2.0' unless defined?(self::VERSION)
 end

data/reyes.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |gem|
 
     TO DO
   EOM
-  gem.homepage = ""
+  gem.homepage = "https://github.com/stripe/reyes/"
 
   gem.files         = list_files
   gem.executables   = find_binaries

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reyes
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Andy Brody
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-19 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -122,13 +122,15 @@ files:
 - lib/reyes/group_tools.rb
 - lib/reyes/ipset.rb
 - lib/reyes/iptables.rb
+- lib/reyes/pgp_wrapper.rb
 - lib/reyes/run_generation.rb
 - lib/reyes/run_manager.rb
+- lib/reyes/s3_loader.rb
 - lib/reyes/set_manager.rb
 - lib/reyes/utils.rb
 - lib/reyes/version.rb
 - reyes.gemspec
-homepage: ''
+homepage: https://github.com/stripe/reyes/
 licenses: []
 metadata: {}
 post_install_message: