clusterfsck 0.1.5.0

data/lib/clusterfsck/commands/environments.rb

@@ -0,0 +1,14 @@
+require 'commander'
+
+module ClusterFsck
+  module Commands
+    class Environments
+      include ClusterFsck::S3Methods
+
+      def run_command(args, options = {})
+        $stdout.puts(all_environments.join("\n"))
+      end
+
+    end
+  end
+end

data/lib/clusterfsck/commands/list.rb

@@ -0,0 +1,20 @@
+require 'commander'
+
+module ClusterFsck
+  module Commands
+    class List
+      include ClusterFsck::S3Methods
+
+      def run_command(args, options = {})
+        @cluster_fsck_env = if args.length > 0
+                        args.first
+                      else
+                        ClusterFsck.cluster_fsck_env
+                      end
+
+        $stdout.puts(all_files.join("\n"))
+      end
+
+    end
+  end
+end

data/lib/clusterfsck/commands/override.rb

@@ -0,0 +1,27 @@
+require 'commander'
+require 'fileutils'
+
+module ClusterFsck
+  module Commands
+    class Override
+      include ClusterFsckEnvArgumentParser
+
+      def run_command(args, options = {})
+        set_cluster_fsck_env_and_key_from_args(args)
+
+        contents = reader.read(remote_only: true)
+        FileUtils.mkdir_p(File.dirname(reader.local_override_path))
+        File.open(reader.local_override_path, "w") do |f|
+          f << contents.to_yaml
+        end
+        $stdout.puts("wrote #{reader.local_override_path} with contents of #{reader.full_s3_path(key)}")
+      end
+
+    private
+      def reader
+        @reader ||= ClusterFsck::Reader.new(key, cluster_fsck_env: cluster_fsck_env)
+      end
+
+    end
+  end
+end

data/lib/clusterfsck/commands/setup.rb

@@ -0,0 +1,11 @@
+require 'commander'
+
+module ClusterFsck
+  module Commands
+    class Setup
+      def run_command(args, options = {})
+        ClusterFsck::Setup.config
+      end
+    end
+  end
+end

data/lib/clusterfsck/configuration.rb

@@ -0,0 +1,13 @@
+module ClusterFsck
+  class Configuration < Hashie::Mash
+
+    def self.from_yaml(yaml)
+      new(YAML.load(yaml))
+    end
+
+    def to_yaml
+      YAML.dump(to_hash)
+    end
+
+  end
+end

data/lib/clusterfsck/credential_grabber.rb

@@ -0,0 +1,53 @@
+module ClusterFsck
+
+  class CredentialGrabber
+    FOG_PATH = "~/.fog"
+    CF_PATH = "~/.clusterfsck"
+
+    def self.find
+      new.find
+    end
+
+    def find
+      from_env_vars || from_cluster_fsck_config || from_fog_file
+    end
+
+    def from_fog_file
+      if exists?(File.expand_path(FOG_PATH))
+        fog_credentials = YAML.load_file(File.expand_path(FOG_PATH))
+        {
+            access_key_id: fog_credentials[:default][:aws_access_key_id],
+            secret_access_key: fog_credentials[:default][:aws_secret_access_key],
+        }
+      end
+    rescue ArgumentError #when there is no HOME, File.expand_path above raises ArgumentError
+      nil
+    end
+
+    def from_env_vars
+      if ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
+        {
+            access_key_id: ENV['AWS_ACCESS_KEY_ID'],
+            secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
+        }
+      end
+    end
+
+    def from_cluster_fsck_config
+      if ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'] &&
+         ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key']
+        {
+            access_key_id: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'],
+            secret_access_key: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key'],
+        }
+      end
+    end
+
+  private
+    def exists?(path)
+      File.exist?(path)
+    end
+
+  end
+
+end

data/lib/clusterfsck/reader.rb

@@ -0,0 +1,56 @@
+module ClusterFsck
+  class Reader
+    include S3Methods
+
+    LOCAL_OVERRIDE_DIR = "clusterfsck"
+    SHARED_ENV = "shared"
+
+
+    attr_reader :cluster_fsck_env, :key, :version_count
+    def initialize(key, opts={})
+      @key = key
+      @cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
+      @ignore_local = opts[:ignore_local]
+    end
+
+    def set_cluster_fsck_env_to_shared_unless_key_found!
+      unless has_local_override? or stored_object.exists?
+        original_cluster_fsck_env = @cluster_fsck_env
+        @cluster_fsck_env = SHARED_ENV
+        raise KeyDoesNotExistError, "there was no #{key} in either #{original_cluster_fsck_env} or #{SHARED_ENV}" unless stored_object(true).exists?
+      end
+    end
+
+    def read(opts = {})
+      set_cluster_fsck_env_to_shared_unless_key_found!
+      yaml = data_for_key(opts)
+      if yaml
+        @version_count = stored_object.versions.count unless has_local_override?
+        Configuration.from_yaml(yaml)
+      end
+    end
+
+    def has_local_override?
+      File.exists?(local_override_path)
+    end
+
+    def local_override_path
+      File.join(LOCAL_OVERRIDE_DIR, full_s3_path(key))
+    end
+
+  private
+    def data_for_key(opts = {})
+      if has_local_override? and !opts[:remote_only]
+        File.read(local_override_path)
+      else
+        stored_object.read
+      end
+    end
+
+    def stored_object(reload = false)
+      return @stored_object if @stored_object and !reload
+      @stored_object = s3_object(key)
+    end
+
+  end
+end

data/lib/clusterfsck/s3_methods.rb

@@ -0,0 +1,52 @@
+module ClusterFsck
+  module S3Methods
+
+    class ConflictError < StandardError; end
+    class KeyDoesNotExistError < StandardError; end
+
+    def s3_object(object_name)
+      bucket.objects[full_s3_path(object_name)]
+    end
+
+    def bucket
+      @bucket ||= s3.buckets[ClusterFsck.config_bucket]
+    end
+
+    def credentials
+      S3Methods.credentials
+    end
+
+    def self.credentials
+      @credentials ||= CredentialGrabber.find
+    end
+
+    def s3
+      S3Methods.s3
+    end
+
+    def self.s3
+      if credentials
+        AWS::S3.new(credentials) #could be nil, especially if on EC2
+      else
+        AWS::S3.new
+      end
+    end
+
+    def all_files
+      bucket.objects.with_prefix(cluster_fsck_env).collect(&:key)
+    end
+
+    def all_environments
+      bucket.objects.map(&:key).map {|key| key.split('/').first }.uniq
+    end
+
+    def full_s3_path(key)
+      "#{cluster_fsck_env}/#{key}"
+    end
+
+    def cluster_fsck_env
+      @cluster_fsck_env || ClusterFsck.cluster_fsck_env
+    end
+
+  end
+end

data/lib/clusterfsck/setup.rb

@@ -0,0 +1,51 @@
+module ClusterFsck
+  module Setup
+    def self.config
+      puts <<-UI
+        ClusterFsck stores your configuration(s) in an S3 bucket, which must have a globally unique name.
+        The name will be stored in `#{ClusterFsck::CLUSTER_FSCK_PATHS[2]}` on this machine, and should be placed in
+        `#{ClusterFsck::CLUSTER_FSCK_PATHS[1]}` on your production box (with a different ENV setting if desired).
+        It may also be overridden on a per project basis by creating a `#{ClusterFsck::CLUSTER_FSCK_PATHS[0]}`
+        file in the project root. The bucket and ENV setting are first checked from environment
+        variables, so you can set the environment to override the files. The bucket is read from CLUSTER_FSCK_BUCKET
+        and the environment (production, staging, development, etc) is read from CLUSTER_FSCK_ENV.
+      UI
+      unless ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
+        set_bucket_name
+      end
+      unless ClusterFsck::CredentialGrabber.find
+        set_aws_keys
+      end
+      unless ClusterFsck::S3Methods.s3.buckets[ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']].exists?
+        warn_create_bucket
+      end
+      ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_env'] ||= ClusterFsck.default_env
+      File.open(File.expand_path(ClusterFsck::CLUSTER_FSCK_PATHS[2]), 'w') do |f|
+        f.write(YAML.dump(ClusterFsck.config_hash))
+      end
+      ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
+    end
+
+    def self.set_bucket_name
+      random_name = "clusterfsck_#{RandomWord.adjs.next}_#{RandomWord.nouns.next}"
+      puts <<-UI
+        Enter a name for your bucket, or press enter to accept the randomly generated name:
+        #{random_name}
+      UI
+      input_name = ask("bucket name: ")
+      CLUSTER_FSCK_CONFIG['cluster_fsck_bucket'] = input_name.empty? ? random_name : input_name
+    end
+
+    def self.set_aws_keys
+      CLUSTER_FSCK_CONFIG['aws_access_key_id'] = ask("Enter your AWS access key: ")
+      CLUSTER_FSCK_CONFIG['aws_secret_access_key'] = ask("Enter your AWS secret access key: ")
+    end
+
+    def self.warn_create_bucket
+      puts "WARNING: #{ClusterFsck.config_bucket} does not exist.  Type yes below to have it created for you, or return to abort."
+      input = ask("Create bucket #{ClusterFsck.config_bucket}?: ")
+      S3Methods.s3.buckets.create(ClusterFsck.config_bucket) unless input.empty?
+      raise "No bucket present, not created" if input.empty?
+    end
+  end
+end

data/lib/clusterfsck/version.rb

@@ -0,0 +1,3 @@
+module ClusterFsck
+  VERSION = "0.1.5.0"
+end

data/lib/clusterfsck/writer.rb

@@ -0,0 +1,30 @@
+module ClusterFsck
+  class Writer
+    include S3Methods
+
+    attr_reader :cluster_fsck_env, :key
+    def initialize(key, opts = {})
+      @key = key
+      @cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
+    end
+
+    #todo, thread safety and process safety please
+    def set(val, version_count = nil)
+      raise_unless_version_count_is_good(version_count)
+      stored_object.write(val.to_yaml)
+      raise_unless_version_count_is_good(version_count + 1) if version_count
+    end
+
+    def raise_unless_version_count_is_good(version_count)
+      if version_count and stored_object.versions.count > version_count
+        raise ConflictError, "File #{key} changed underneath you, version_count expected to be #{version_count} but was #{stored_object.versions.count}"
+      end
+    end
+
+    def stored_object
+      @stored_object ||= s3_object(key)
+    end
+
+
+  end
+end

data/spec/lib/clusterfsck/commands/amicus_env_argument_parser_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+require 'commander'
+require 'clusterfsck/cli'
+
+module ClusterFsck::Commands
+  class DummyClass
+    include ClusterFsckEnvArgumentParser
+  end
+
+  describe ClusterFsckEnvArgumentParser do
+    subject { DummyClass.new }
+
+    it "should use the key when only one arg" do
+      subject.set_cluster_fsck_env_and_key_from_args(["test-key"])
+      subject.key.should == "test-key"
+      subject.cluster_fsck_env.should == 'test'
+    end
+
+    it "should set cluster_fsck_env and key when two args" do
+      subject.set_cluster_fsck_env_and_key_from_args(["a_different_env", "test-key"])
+      subject.key.should == "test-key"
+      subject.cluster_fsck_env.should == "a_different_env"
+    end
+
+  end
+end

data/spec/lib/clusterfsck/commands/create_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+require 'commander'
+require 'clusterfsck/cli'
+
+module ClusterFsck::Commands
+  describe Create do
+
+    let(:cluster_fsck_env) { "test" }
+    let(:args) { ["test-key"] }
+    let(:mock_s3_object) do
+      mock("s3_obj",
+        :"exists?" => false,
+        :write => true
+      )
+    end
+
+    before do
+      ClusterFsck::Commands::Edit.any_instance.stub(:run_command).and_return(true)
+      subject.stub(:s3_object).and_return(mock_s3_object)
+    end
+
+    it "should create the file" do
+      mock_s3_object.should_receive(:write).with("")
+      subject.run_command(args)
+    end
+
+    it "should open the editor with the file" do
+      ClusterFsck::Commands::Edit.any_instance.should_receive(:run_command).with(args).and_return(true)
+      subject.run_command(args)
+    end
+
+    it "should error and not write if the file already exists" do
+      mock_s3_object.stub(:exists?).and_return(true)
+      mock_s3_object.should_not_receive(:write)
+      ->() { subject.run_command(args) }.should raise_error(ClusterFsck::S3Methods::ConflictError)
+    end
+
+  end
+
+
+end

data/spec/lib/clusterfsck/commands/edit_spec.rb

@@ -0,0 +1,49 @@
+require 'spec_helper'
+require 'commander'
+require 'clusterfsck/cli'
+
+module ClusterFsck::Commands
+  describe Edit do
+    let(:key) { "test-key" }
+    let(:mock_writer) { mock("writer", set: true) }
+
+    let(:dummy_val) do
+      {
+          key => {
+            'test' => true
+          }
+      }
+    end
+
+    let(:dummy_yaml) { YAML.dump(dummy_val) }
+    let(:mock_s3_obj) do
+      mock(:s3_object, {
+          read: dummy_yaml,
+          versions: mock('versions', count: 3),
+          :exists? => true
+      })
+    end
+
+    before do
+      ClusterFsck::Reader.any_instance.stub(:s3_object).and_return(mock_s3_obj)
+    end
+
+    describe "with a key" do
+      let(:args) { [key] }
+
+      it "should open the yaml version of the key in an editor" do
+        subject.should_receive(:ask_editor).with(dummy_yaml).and_return(dummy_yaml)
+        subject.stub(:writer).and_return(mock_writer)
+        mock_writer.should_receive(:set).with(dummy_val, mock_s3_obj.versions.count).once
+
+        subject.run_command(args)
+      end
+
+      it "should error when it is locally overriden" do
+        subject.reader.stub(:has_local_override?).and_return(true)
+        ->() { subject.run_command(args) }.should raise_error(ArgumentError)
+      end
+
+    end
+  end
+end