clusterfsck 0.1.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. data/.gitignore +19 -0
  2. data/.rspec +2 -0
  3. data/.rvmrc +1 -0
  4. data/Gemfile +12 -0
  5. data/Guardfile +24 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +121 -0
  8. data/Rakefile +10 -0
  9. data/bin/clusterfsck +60 -0
  10. data/clusterfsck.gemspec +34 -0
  11. data/lib/clusterfsck.rb +48 -0
  12. data/lib/clusterfsck/cli.rb +8 -0
  13. data/lib/clusterfsck/commands/cluster_fsck_env_argument_parser.rb +21 -0
  14. data/lib/clusterfsck/commands/create.rb +20 -0
  15. data/lib/clusterfsck/commands/edit.rb +30 -0
  16. data/lib/clusterfsck/commands/environments.rb +14 -0
  17. data/lib/clusterfsck/commands/list.rb +20 -0
  18. data/lib/clusterfsck/commands/override.rb +27 -0
  19. data/lib/clusterfsck/commands/setup.rb +11 -0
  20. data/lib/clusterfsck/configuration.rb +13 -0
  21. data/lib/clusterfsck/credential_grabber.rb +53 -0
  22. data/lib/clusterfsck/reader.rb +56 -0
  23. data/lib/clusterfsck/s3_methods.rb +52 -0
  24. data/lib/clusterfsck/setup.rb +51 -0
  25. data/lib/clusterfsck/version.rb +3 -0
  26. data/lib/clusterfsck/writer.rb +30 -0
  27. data/spec/lib/clusterfsck/commands/amicus_env_argument_parser_spec.rb +26 -0
  28. data/spec/lib/clusterfsck/commands/create_spec.rb +41 -0
  29. data/spec/lib/clusterfsck/commands/edit_spec.rb +49 -0
  30. data/spec/lib/clusterfsck/commands/override_spec.rb +26 -0
  31. data/spec/lib/clusterfsck/configuration_spec.rb +25 -0
  32. data/spec/lib/clusterfsck/credential_grabber_spec.rb +100 -0
  33. data/spec/lib/clusterfsck/reader_spec.rb +60 -0
  34. data/spec/lib/clusterfsck/s3_methods_spec.rb +39 -0
  35. data/spec/lib/clusterfsck/writer_spec.rb +84 -0
  36. data/spec/spec_helper.rb +24 -0
  37. data/spec/support/cluster_fsck_env_setter.rb +11 -0
  38. metadata +177 -0
@@ -0,0 +1,14 @@
1
+ require 'commander'
2
+
3
+ module ClusterFsck
4
+ module Commands
5
+ class Environments
6
+ include ClusterFsck::S3Methods
7
+
8
+ def run_command(args, options = {})
9
+ $stdout.puts(all_environments.join("\n"))
10
+ end
11
+
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,20 @@
1
+ require 'commander'
2
+
3
+ module ClusterFsck
4
+ module Commands
5
+ class List
6
+ include ClusterFsck::S3Methods
7
+
8
+ def run_command(args, options = {})
9
+ @cluster_fsck_env = if args.length > 0
10
+ args.first
11
+ else
12
+ ClusterFsck.cluster_fsck_env
13
+ end
14
+
15
+ $stdout.puts(all_files.join("\n"))
16
+ end
17
+
18
+ end
19
+ end
20
+ end
@@ -0,0 +1,27 @@
1
+ require 'commander'
2
+ require 'fileutils'
3
+
4
+ module ClusterFsck
5
+ module Commands
6
+ class Override
7
+ include ClusterFsckEnvArgumentParser
8
+
9
+ def run_command(args, options = {})
10
+ set_cluster_fsck_env_and_key_from_args(args)
11
+
12
+ contents = reader.read(remote_only: true)
13
+ FileUtils.mkdir_p(File.dirname(reader.local_override_path))
14
+ File.open(reader.local_override_path, "w") do |f|
15
+ f << contents.to_yaml
16
+ end
17
+ $stdout.puts("wrote #{reader.local_override_path} with contents of #{reader.full_s3_path(key)}")
18
+ end
19
+
20
+ private
21
+ def reader
22
+ @reader ||= ClusterFsck::Reader.new(key, cluster_fsck_env: cluster_fsck_env)
23
+ end
24
+
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,11 @@
1
+ require 'commander'
2
+
3
+ module ClusterFsck
4
+ module Commands
5
+ class Setup
6
+ def run_command(args, options = {})
7
+ ClusterFsck::Setup.config
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,13 @@
1
+ module ClusterFsck
2
+ class Configuration < Hashie::Mash
3
+
4
+ def self.from_yaml(yaml)
5
+ new(YAML.load(yaml))
6
+ end
7
+
8
+ def to_yaml
9
+ YAML.dump(to_hash)
10
+ end
11
+
12
+ end
13
+ end
@@ -0,0 +1,53 @@
1
+ module ClusterFsck
2
+
3
+ class CredentialGrabber
4
+ FOG_PATH = "~/.fog"
5
+ CF_PATH = "~/.clusterfsck"
6
+
7
+ def self.find
8
+ new.find
9
+ end
10
+
11
+ def find
12
+ from_env_vars || from_cluster_fsck_config || from_fog_file
13
+ end
14
+
15
+ def from_fog_file
16
+ if exists?(File.expand_path(FOG_PATH))
17
+ fog_credentials = YAML.load_file(File.expand_path(FOG_PATH))
18
+ {
19
+ access_key_id: fog_credentials[:default][:aws_access_key_id],
20
+ secret_access_key: fog_credentials[:default][:aws_secret_access_key],
21
+ }
22
+ end
23
+ rescue ArgumentError #when there is no HOME, File.expand_path above raises ArgumentError
24
+ nil
25
+ end
26
+
27
+ def from_env_vars
28
+ if ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
29
+ {
30
+ access_key_id: ENV['AWS_ACCESS_KEY_ID'],
31
+ secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
32
+ }
33
+ end
34
+ end
35
+
36
+ def from_cluster_fsck_config
37
+ if ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'] &&
38
+ ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key']
39
+ {
40
+ access_key_id: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'],
41
+ secret_access_key: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key'],
42
+ }
43
+ end
44
+ end
45
+
46
+ private
47
+ def exists?(path)
48
+ File.exist?(path)
49
+ end
50
+
51
+ end
52
+
53
+ end
@@ -0,0 +1,56 @@
1
+ module ClusterFsck
2
+ class Reader
3
+ include S3Methods
4
+
5
+ LOCAL_OVERRIDE_DIR = "clusterfsck"
6
+ SHARED_ENV = "shared"
7
+
8
+
9
+ attr_reader :cluster_fsck_env, :key, :version_count
10
+ def initialize(key, opts={})
11
+ @key = key
12
+ @cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
13
+ @ignore_local = opts[:ignore_local]
14
+ end
15
+
16
+ def set_cluster_fsck_env_to_shared_unless_key_found!
17
+ unless has_local_override? or stored_object.exists?
18
+ original_cluster_fsck_env = @cluster_fsck_env
19
+ @cluster_fsck_env = SHARED_ENV
20
+ raise KeyDoesNotExistError, "there was no #{key} in either #{original_cluster_fsck_env} or #{SHARED_ENV}" unless stored_object(true).exists?
21
+ end
22
+ end
23
+
24
+ def read(opts = {})
25
+ set_cluster_fsck_env_to_shared_unless_key_found!
26
+ yaml = data_for_key(opts)
27
+ if yaml
28
+ @version_count = stored_object.versions.count unless has_local_override?
29
+ Configuration.from_yaml(yaml)
30
+ end
31
+ end
32
+
33
+ def has_local_override?
34
+ File.exists?(local_override_path)
35
+ end
36
+
37
+ def local_override_path
38
+ File.join(LOCAL_OVERRIDE_DIR, full_s3_path(key))
39
+ end
40
+
41
+ private
42
+ def data_for_key(opts = {})
43
+ if has_local_override? and !opts[:remote_only]
44
+ File.read(local_override_path)
45
+ else
46
+ stored_object.read
47
+ end
48
+ end
49
+
50
+ def stored_object(reload = false)
51
+ return @stored_object if @stored_object and !reload
52
+ @stored_object = s3_object(key)
53
+ end
54
+
55
+ end
56
+ end
@@ -0,0 +1,52 @@
1
+ module ClusterFsck
2
+ module S3Methods
3
+
4
+ class ConflictError < StandardError; end
5
+ class KeyDoesNotExistError < StandardError; end
6
+
7
+ def s3_object(object_name)
8
+ bucket.objects[full_s3_path(object_name)]
9
+ end
10
+
11
+ def bucket
12
+ @bucket ||= s3.buckets[ClusterFsck.config_bucket]
13
+ end
14
+
15
+ def credentials
16
+ S3Methods.credentials
17
+ end
18
+
19
+ def self.credentials
20
+ @credentials ||= CredentialGrabber.find
21
+ end
22
+
23
+ def s3
24
+ S3Methods.s3
25
+ end
26
+
27
+ def self.s3
28
+ if credentials
29
+ AWS::S3.new(credentials) #could be nil, especially if on EC2
30
+ else
31
+ AWS::S3.new
32
+ end
33
+ end
34
+
35
+ def all_files
36
+ bucket.objects.with_prefix(cluster_fsck_env).collect(&:key)
37
+ end
38
+
39
+ def all_environments
40
+ bucket.objects.map(&:key).map {|key| key.split('/').first }.uniq
41
+ end
42
+
43
+ def full_s3_path(key)
44
+ "#{cluster_fsck_env}/#{key}"
45
+ end
46
+
47
+ def cluster_fsck_env
48
+ @cluster_fsck_env || ClusterFsck.cluster_fsck_env
49
+ end
50
+
51
+ end
52
+ end
@@ -0,0 +1,51 @@
1
+ module ClusterFsck
2
+ module Setup
3
+ def self.config
4
+ puts <<-UI
5
+ ClusterFsck stores your configuration(s) in an S3 bucket, which must have a globally unique name.
6
+ The name will be stored in `#{ClusterFsck::CLUSTER_FSCK_PATHS[2]}` on this machine, and should be placed in
7
+ `#{ClusterFsck::CLUSTER_FSCK_PATHS[1]}` on your production box (with a different ENV setting if desired).
8
+ It may also be overridden on a per project basis by creating a `#{ClusterFsck::CLUSTER_FSCK_PATHS[0]}`
9
+ file in the project root. The bucket and ENV setting are first checked from environment
10
+ variables, so you can set the environment to override the files. The bucket is read from CLUSTER_FSCK_BUCKET
11
+ and the environment (production, staging, development, etc) is read from CLUSTER_FSCK_ENV.
12
+ UI
13
+ unless ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
14
+ set_bucket_name
15
+ end
16
+ unless ClusterFsck::CredentialGrabber.find
17
+ set_aws_keys
18
+ end
19
+ unless ClusterFsck::S3Methods.s3.buckets[ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']].exists?
20
+ warn_create_bucket
21
+ end
22
+ ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_env'] ||= ClusterFsck.default_env
23
+ File.open(File.expand_path(ClusterFsck::CLUSTER_FSCK_PATHS[2]), 'w') do |f|
24
+ f.write(YAML.dump(ClusterFsck.config_hash))
25
+ end
26
+ ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
27
+ end
28
+
29
+ def self.set_bucket_name
30
+ random_name = "clusterfsck_#{RandomWord.adjs.next}_#{RandomWord.nouns.next}"
31
+ puts <<-UI
32
+ Enter a name for your bucket, or press enter to accept the randomly generated name:
33
+ #{random_name}
34
+ UI
35
+ input_name = ask("bucket name: ")
36
+ CLUSTER_FSCK_CONFIG['cluster_fsck_bucket'] = input_name.empty? ? random_name : input_name
37
+ end
38
+
39
+ def self.set_aws_keys
40
+ CLUSTER_FSCK_CONFIG['aws_access_key_id'] = ask("Enter your AWS access key: ")
41
+ CLUSTER_FSCK_CONFIG['aws_secret_access_key'] = ask("Enter your AWS secret access key: ")
42
+ end
43
+
44
+ def self.warn_create_bucket
45
+ puts "WARNING: #{ClusterFsck.config_bucket} does not exist. Type yes below to have it created for you, or return to abort."
46
+ input = ask("Create bucket #{ClusterFsck.config_bucket}?: ")
47
+ S3Methods.s3.buckets.create(ClusterFsck.config_bucket) unless input.empty?
48
+ raise "No bucket present, not created" if input.empty?
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,3 @@
1
+ module ClusterFsck
2
+ VERSION = "0.1.5.0"
3
+ end
@@ -0,0 +1,30 @@
1
+ module ClusterFsck
2
+ class Writer
3
+ include S3Methods
4
+
5
+ attr_reader :cluster_fsck_env, :key
6
+ def initialize(key, opts = {})
7
+ @key = key
8
+ @cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
9
+ end
10
+
11
+ #todo, thread safety and process safety please
12
+ def set(val, version_count = nil)
13
+ raise_unless_version_count_is_good(version_count)
14
+ stored_object.write(val.to_yaml)
15
+ raise_unless_version_count_is_good(version_count + 1) if version_count
16
+ end
17
+
18
+ def raise_unless_version_count_is_good(version_count)
19
+ if version_count and stored_object.versions.count > version_count
20
+ raise ConflictError, "File #{key} changed underneath you, version_count expected to be #{version_count} but was #{stored_object.versions.count}"
21
+ end
22
+ end
23
+
24
+ def stored_object
25
+ @stored_object ||= s3_object(key)
26
+ end
27
+
28
+
29
+ end
30
+ end
@@ -0,0 +1,26 @@
1
+ require 'spec_helper'
2
+ require 'commander'
3
+ require 'clusterfsck/cli'
4
+
5
+ module ClusterFsck::Commands
6
+ class DummyClass
7
+ include ClusterFsckEnvArgumentParser
8
+ end
9
+
10
+ describe ClusterFsckEnvArgumentParser do
11
+ subject { DummyClass.new }
12
+
13
+ it "should use the key when only one arg" do
14
+ subject.set_cluster_fsck_env_and_key_from_args(["test-key"])
15
+ subject.key.should == "test-key"
16
+ subject.cluster_fsck_env.should == 'test'
17
+ end
18
+
19
+ it "should set cluster_fsck_env and key when two args" do
20
+ subject.set_cluster_fsck_env_and_key_from_args(["a_different_env", "test-key"])
21
+ subject.key.should == "test-key"
22
+ subject.cluster_fsck_env.should == "a_different_env"
23
+ end
24
+
25
+ end
26
+ end
@@ -0,0 +1,41 @@
1
+ require 'spec_helper'
2
+ require 'commander'
3
+ require 'clusterfsck/cli'
4
+
5
+ module ClusterFsck::Commands
6
+ describe Create do
7
+
8
+ let(:cluster_fsck_env) { "test" }
9
+ let(:args) { ["test-key"] }
10
+ let(:mock_s3_object) do
11
+ mock("s3_obj",
12
+ :"exists?" => false,
13
+ :write => true
14
+ )
15
+ end
16
+
17
+ before do
18
+ ClusterFsck::Commands::Edit.any_instance.stub(:run_command).and_return(true)
19
+ subject.stub(:s3_object).and_return(mock_s3_object)
20
+ end
21
+
22
+ it "should create the file" do
23
+ mock_s3_object.should_receive(:write).with("")
24
+ subject.run_command(args)
25
+ end
26
+
27
+ it "should open the editor with the file" do
28
+ ClusterFsck::Commands::Edit.any_instance.should_receive(:run_command).with(args).and_return(true)
29
+ subject.run_command(args)
30
+ end
31
+
32
+ it "should error and not write if the file already exists" do
33
+ mock_s3_object.stub(:exists?).and_return(true)
34
+ mock_s3_object.should_not_receive(:write)
35
+ ->() { subject.run_command(args) }.should raise_error(ClusterFsck::S3Methods::ConflictError)
36
+ end
37
+
38
+ end
39
+
40
+
41
+ end
@@ -0,0 +1,49 @@
1
+ require 'spec_helper'
2
+ require 'commander'
3
+ require 'clusterfsck/cli'
4
+
5
+ module ClusterFsck::Commands
6
+ describe Edit do
7
+ let(:key) { "test-key" }
8
+ let(:mock_writer) { mock("writer", set: true) }
9
+
10
+ let(:dummy_val) do
11
+ {
12
+ key => {
13
+ 'test' => true
14
+ }
15
+ }
16
+ end
17
+
18
+ let(:dummy_yaml) { YAML.dump(dummy_val) }
19
+ let(:mock_s3_obj) do
20
+ mock(:s3_object, {
21
+ read: dummy_yaml,
22
+ versions: mock('versions', count: 3),
23
+ :exists? => true
24
+ })
25
+ end
26
+
27
+ before do
28
+ ClusterFsck::Reader.any_instance.stub(:s3_object).and_return(mock_s3_obj)
29
+ end
30
+
31
+ describe "with a key" do
32
+ let(:args) { [key] }
33
+
34
+ it "should open the yaml version of the key in an editor" do
35
+ subject.should_receive(:ask_editor).with(dummy_yaml).and_return(dummy_yaml)
36
+ subject.stub(:writer).and_return(mock_writer)
37
+ mock_writer.should_receive(:set).with(dummy_val, mock_s3_obj.versions.count).once
38
+
39
+ subject.run_command(args)
40
+ end
41
+
42
+ it "should error when it is locally overriden" do
43
+ subject.reader.stub(:has_local_override?).and_return(true)
44
+ ->() { subject.run_command(args) }.should raise_error(ArgumentError)
45
+ end
46
+
47
+ end
48
+ end
49
+ end