clusterfsck 0.1.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +19 -0
- data/.rspec +2 -0
- data/.rvmrc +1 -0
- data/Gemfile +12 -0
- data/Guardfile +24 -0
- data/LICENSE.txt +22 -0
- data/README.md +121 -0
- data/Rakefile +10 -0
- data/bin/clusterfsck +60 -0
- data/clusterfsck.gemspec +34 -0
- data/lib/clusterfsck.rb +48 -0
- data/lib/clusterfsck/cli.rb +8 -0
- data/lib/clusterfsck/commands/cluster_fsck_env_argument_parser.rb +21 -0
- data/lib/clusterfsck/commands/create.rb +20 -0
- data/lib/clusterfsck/commands/edit.rb +30 -0
- data/lib/clusterfsck/commands/environments.rb +14 -0
- data/lib/clusterfsck/commands/list.rb +20 -0
- data/lib/clusterfsck/commands/override.rb +27 -0
- data/lib/clusterfsck/commands/setup.rb +11 -0
- data/lib/clusterfsck/configuration.rb +13 -0
- data/lib/clusterfsck/credential_grabber.rb +53 -0
- data/lib/clusterfsck/reader.rb +56 -0
- data/lib/clusterfsck/s3_methods.rb +52 -0
- data/lib/clusterfsck/setup.rb +51 -0
- data/lib/clusterfsck/version.rb +3 -0
- data/lib/clusterfsck/writer.rb +30 -0
- data/spec/lib/clusterfsck/commands/amicus_env_argument_parser_spec.rb +26 -0
- data/spec/lib/clusterfsck/commands/create_spec.rb +41 -0
- data/spec/lib/clusterfsck/commands/edit_spec.rb +49 -0
- data/spec/lib/clusterfsck/commands/override_spec.rb +26 -0
- data/spec/lib/clusterfsck/configuration_spec.rb +25 -0
- data/spec/lib/clusterfsck/credential_grabber_spec.rb +100 -0
- data/spec/lib/clusterfsck/reader_spec.rb +60 -0
- data/spec/lib/clusterfsck/s3_methods_spec.rb +39 -0
- data/spec/lib/clusterfsck/writer_spec.rb +84 -0
- data/spec/spec_helper.rb +24 -0
- data/spec/support/cluster_fsck_env_setter.rb +11 -0
- metadata +177 -0
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'commander'
|
2
|
+
|
3
|
+
module ClusterFsck
|
4
|
+
module Commands
|
5
|
+
class List
|
6
|
+
include ClusterFsck::S3Methods
|
7
|
+
|
8
|
+
def run_command(args, options = {})
|
9
|
+
@cluster_fsck_env = if args.length > 0
|
10
|
+
args.first
|
11
|
+
else
|
12
|
+
ClusterFsck.cluster_fsck_env
|
13
|
+
end
|
14
|
+
|
15
|
+
$stdout.puts(all_files.join("\n"))
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'commander'
|
2
|
+
require 'fileutils'
|
3
|
+
|
4
|
+
module ClusterFsck
|
5
|
+
module Commands
|
6
|
+
class Override
|
7
|
+
include ClusterFsckEnvArgumentParser
|
8
|
+
|
9
|
+
def run_command(args, options = {})
|
10
|
+
set_cluster_fsck_env_and_key_from_args(args)
|
11
|
+
|
12
|
+
contents = reader.read(remote_only: true)
|
13
|
+
FileUtils.mkdir_p(File.dirname(reader.local_override_path))
|
14
|
+
File.open(reader.local_override_path, "w") do |f|
|
15
|
+
f << contents.to_yaml
|
16
|
+
end
|
17
|
+
$stdout.puts("wrote #{reader.local_override_path} with contents of #{reader.full_s3_path(key)}")
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
def reader
|
22
|
+
@reader ||= ClusterFsck::Reader.new(key, cluster_fsck_env: cluster_fsck_env)
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
module ClusterFsck
|
2
|
+
|
3
|
+
class CredentialGrabber
|
4
|
+
FOG_PATH = "~/.fog"
|
5
|
+
CF_PATH = "~/.clusterfsck"
|
6
|
+
|
7
|
+
def self.find
|
8
|
+
new.find
|
9
|
+
end
|
10
|
+
|
11
|
+
def find
|
12
|
+
from_env_vars || from_cluster_fsck_config || from_fog_file
|
13
|
+
end
|
14
|
+
|
15
|
+
def from_fog_file
|
16
|
+
if exists?(File.expand_path(FOG_PATH))
|
17
|
+
fog_credentials = YAML.load_file(File.expand_path(FOG_PATH))
|
18
|
+
{
|
19
|
+
access_key_id: fog_credentials[:default][:aws_access_key_id],
|
20
|
+
secret_access_key: fog_credentials[:default][:aws_secret_access_key],
|
21
|
+
}
|
22
|
+
end
|
23
|
+
rescue ArgumentError #when there is no HOME, File.expand_path above raises ArgumentError
|
24
|
+
nil
|
25
|
+
end
|
26
|
+
|
27
|
+
def from_env_vars
|
28
|
+
if ENV['AWS_ACCESS_KEY_ID'] && ENV['AWS_SECRET_ACCESS_KEY']
|
29
|
+
{
|
30
|
+
access_key_id: ENV['AWS_ACCESS_KEY_ID'],
|
31
|
+
secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
|
32
|
+
}
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def from_cluster_fsck_config
|
37
|
+
if ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'] &&
|
38
|
+
ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key']
|
39
|
+
{
|
40
|
+
access_key_id: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_access_key_id'],
|
41
|
+
secret_access_key: ClusterFsck::CLUSTER_FSCK_CONFIG['aws_secret_access_key'],
|
42
|
+
}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
def exists?(path)
|
48
|
+
File.exist?(path)
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module ClusterFsck
|
2
|
+
class Reader
|
3
|
+
include S3Methods
|
4
|
+
|
5
|
+
LOCAL_OVERRIDE_DIR = "clusterfsck"
|
6
|
+
SHARED_ENV = "shared"
|
7
|
+
|
8
|
+
|
9
|
+
attr_reader :cluster_fsck_env, :key, :version_count
|
10
|
+
def initialize(key, opts={})
|
11
|
+
@key = key
|
12
|
+
@cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
|
13
|
+
@ignore_local = opts[:ignore_local]
|
14
|
+
end
|
15
|
+
|
16
|
+
def set_cluster_fsck_env_to_shared_unless_key_found!
|
17
|
+
unless has_local_override? or stored_object.exists?
|
18
|
+
original_cluster_fsck_env = @cluster_fsck_env
|
19
|
+
@cluster_fsck_env = SHARED_ENV
|
20
|
+
raise KeyDoesNotExistError, "there was no #{key} in either #{original_cluster_fsck_env} or #{SHARED_ENV}" unless stored_object(true).exists?
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def read(opts = {})
|
25
|
+
set_cluster_fsck_env_to_shared_unless_key_found!
|
26
|
+
yaml = data_for_key(opts)
|
27
|
+
if yaml
|
28
|
+
@version_count = stored_object.versions.count unless has_local_override?
|
29
|
+
Configuration.from_yaml(yaml)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def has_local_override?
|
34
|
+
File.exists?(local_override_path)
|
35
|
+
end
|
36
|
+
|
37
|
+
def local_override_path
|
38
|
+
File.join(LOCAL_OVERRIDE_DIR, full_s3_path(key))
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
def data_for_key(opts = {})
|
43
|
+
if has_local_override? and !opts[:remote_only]
|
44
|
+
File.read(local_override_path)
|
45
|
+
else
|
46
|
+
stored_object.read
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def stored_object(reload = false)
|
51
|
+
return @stored_object if @stored_object and !reload
|
52
|
+
@stored_object = s3_object(key)
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module ClusterFsck
|
2
|
+
module S3Methods
|
3
|
+
|
4
|
+
class ConflictError < StandardError; end
|
5
|
+
class KeyDoesNotExistError < StandardError; end
|
6
|
+
|
7
|
+
def s3_object(object_name)
|
8
|
+
bucket.objects[full_s3_path(object_name)]
|
9
|
+
end
|
10
|
+
|
11
|
+
def bucket
|
12
|
+
@bucket ||= s3.buckets[ClusterFsck.config_bucket]
|
13
|
+
end
|
14
|
+
|
15
|
+
def credentials
|
16
|
+
S3Methods.credentials
|
17
|
+
end
|
18
|
+
|
19
|
+
def self.credentials
|
20
|
+
@credentials ||= CredentialGrabber.find
|
21
|
+
end
|
22
|
+
|
23
|
+
def s3
|
24
|
+
S3Methods.s3
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.s3
|
28
|
+
if credentials
|
29
|
+
AWS::S3.new(credentials) #could be nil, especially if on EC2
|
30
|
+
else
|
31
|
+
AWS::S3.new
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def all_files
|
36
|
+
bucket.objects.with_prefix(cluster_fsck_env).collect(&:key)
|
37
|
+
end
|
38
|
+
|
39
|
+
def all_environments
|
40
|
+
bucket.objects.map(&:key).map {|key| key.split('/').first }.uniq
|
41
|
+
end
|
42
|
+
|
43
|
+
def full_s3_path(key)
|
44
|
+
"#{cluster_fsck_env}/#{key}"
|
45
|
+
end
|
46
|
+
|
47
|
+
def cluster_fsck_env
|
48
|
+
@cluster_fsck_env || ClusterFsck.cluster_fsck_env
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module ClusterFsck
|
2
|
+
module Setup
|
3
|
+
def self.config
|
4
|
+
puts <<-UI
|
5
|
+
ClusterFsck stores your configuration(s) in an S3 bucket, which must have a globally unique name.
|
6
|
+
The name will be stored in `#{ClusterFsck::CLUSTER_FSCK_PATHS[2]}` on this machine, and should be placed in
|
7
|
+
`#{ClusterFsck::CLUSTER_FSCK_PATHS[1]}` on your production box (with a different ENV setting if desired).
|
8
|
+
It may also be overridden on a per project basis by creating a `#{ClusterFsck::CLUSTER_FSCK_PATHS[0]}`
|
9
|
+
file in the project root. The bucket and ENV setting are first checked from environment
|
10
|
+
variables, so you can set the environment to override the files. The bucket is read from CLUSTER_FSCK_BUCKET
|
11
|
+
and the environment (production, staging, development, etc) is read from CLUSTER_FSCK_ENV.
|
12
|
+
UI
|
13
|
+
unless ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
|
14
|
+
set_bucket_name
|
15
|
+
end
|
16
|
+
unless ClusterFsck::CredentialGrabber.find
|
17
|
+
set_aws_keys
|
18
|
+
end
|
19
|
+
unless ClusterFsck::S3Methods.s3.buckets[ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']].exists?
|
20
|
+
warn_create_bucket
|
21
|
+
end
|
22
|
+
ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_env'] ||= ClusterFsck.default_env
|
23
|
+
File.open(File.expand_path(ClusterFsck::CLUSTER_FSCK_PATHS[2]), 'w') do |f|
|
24
|
+
f.write(YAML.dump(ClusterFsck.config_hash))
|
25
|
+
end
|
26
|
+
ClusterFsck::CLUSTER_FSCK_CONFIG['cluster_fsck_bucket']
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.set_bucket_name
|
30
|
+
random_name = "clusterfsck_#{RandomWord.adjs.next}_#{RandomWord.nouns.next}"
|
31
|
+
puts <<-UI
|
32
|
+
Enter a name for your bucket, or press enter to accept the randomly generated name:
|
33
|
+
#{random_name}
|
34
|
+
UI
|
35
|
+
input_name = ask("bucket name: ")
|
36
|
+
CLUSTER_FSCK_CONFIG['cluster_fsck_bucket'] = input_name.empty? ? random_name : input_name
|
37
|
+
end
|
38
|
+
|
39
|
+
def self.set_aws_keys
|
40
|
+
CLUSTER_FSCK_CONFIG['aws_access_key_id'] = ask("Enter your AWS access key: ")
|
41
|
+
CLUSTER_FSCK_CONFIG['aws_secret_access_key'] = ask("Enter your AWS secret access key: ")
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.warn_create_bucket
|
45
|
+
puts "WARNING: #{ClusterFsck.config_bucket} does not exist. Type yes below to have it created for you, or return to abort."
|
46
|
+
input = ask("Create bucket #{ClusterFsck.config_bucket}?: ")
|
47
|
+
S3Methods.s3.buckets.create(ClusterFsck.config_bucket) unless input.empty?
|
48
|
+
raise "No bucket present, not created" if input.empty?
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module ClusterFsck
|
2
|
+
class Writer
|
3
|
+
include S3Methods
|
4
|
+
|
5
|
+
attr_reader :cluster_fsck_env, :key
|
6
|
+
def initialize(key, opts = {})
|
7
|
+
@key = key
|
8
|
+
@cluster_fsck_env = opts[:cluster_fsck_env] || ClusterFsck.cluster_fsck_env
|
9
|
+
end
|
10
|
+
|
11
|
+
#todo, thread safety and process safety please
|
12
|
+
def set(val, version_count = nil)
|
13
|
+
raise_unless_version_count_is_good(version_count)
|
14
|
+
stored_object.write(val.to_yaml)
|
15
|
+
raise_unless_version_count_is_good(version_count + 1) if version_count
|
16
|
+
end
|
17
|
+
|
18
|
+
def raise_unless_version_count_is_good(version_count)
|
19
|
+
if version_count and stored_object.versions.count > version_count
|
20
|
+
raise ConflictError, "File #{key} changed underneath you, version_count expected to be #{version_count} but was #{stored_object.versions.count}"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def stored_object
|
25
|
+
@stored_object ||= s3_object(key)
|
26
|
+
end
|
27
|
+
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'commander'
|
3
|
+
require 'clusterfsck/cli'
|
4
|
+
|
5
|
+
module ClusterFsck::Commands
|
6
|
+
class DummyClass
|
7
|
+
include ClusterFsckEnvArgumentParser
|
8
|
+
end
|
9
|
+
|
10
|
+
describe ClusterFsckEnvArgumentParser do
|
11
|
+
subject { DummyClass.new }
|
12
|
+
|
13
|
+
it "should use the key when only one arg" do
|
14
|
+
subject.set_cluster_fsck_env_and_key_from_args(["test-key"])
|
15
|
+
subject.key.should == "test-key"
|
16
|
+
subject.cluster_fsck_env.should == 'test'
|
17
|
+
end
|
18
|
+
|
19
|
+
it "should set cluster_fsck_env and key when two args" do
|
20
|
+
subject.set_cluster_fsck_env_and_key_from_args(["a_different_env", "test-key"])
|
21
|
+
subject.key.should == "test-key"
|
22
|
+
subject.cluster_fsck_env.should == "a_different_env"
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'commander'
|
3
|
+
require 'clusterfsck/cli'
|
4
|
+
|
5
|
+
module ClusterFsck::Commands
|
6
|
+
describe Create do
|
7
|
+
|
8
|
+
let(:cluster_fsck_env) { "test" }
|
9
|
+
let(:args) { ["test-key"] }
|
10
|
+
let(:mock_s3_object) do
|
11
|
+
mock("s3_obj",
|
12
|
+
:"exists?" => false,
|
13
|
+
:write => true
|
14
|
+
)
|
15
|
+
end
|
16
|
+
|
17
|
+
before do
|
18
|
+
ClusterFsck::Commands::Edit.any_instance.stub(:run_command).and_return(true)
|
19
|
+
subject.stub(:s3_object).and_return(mock_s3_object)
|
20
|
+
end
|
21
|
+
|
22
|
+
it "should create the file" do
|
23
|
+
mock_s3_object.should_receive(:write).with("")
|
24
|
+
subject.run_command(args)
|
25
|
+
end
|
26
|
+
|
27
|
+
it "should open the editor with the file" do
|
28
|
+
ClusterFsck::Commands::Edit.any_instance.should_receive(:run_command).with(args).and_return(true)
|
29
|
+
subject.run_command(args)
|
30
|
+
end
|
31
|
+
|
32
|
+
it "should error and not write if the file already exists" do
|
33
|
+
mock_s3_object.stub(:exists?).and_return(true)
|
34
|
+
mock_s3_object.should_not_receive(:write)
|
35
|
+
->() { subject.run_command(args) }.should raise_error(ClusterFsck::S3Methods::ConflictError)
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
|
41
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'commander'
|
3
|
+
require 'clusterfsck/cli'
|
4
|
+
|
5
|
+
module ClusterFsck::Commands
|
6
|
+
describe Edit do
|
7
|
+
let(:key) { "test-key" }
|
8
|
+
let(:mock_writer) { mock("writer", set: true) }
|
9
|
+
|
10
|
+
let(:dummy_val) do
|
11
|
+
{
|
12
|
+
key => {
|
13
|
+
'test' => true
|
14
|
+
}
|
15
|
+
}
|
16
|
+
end
|
17
|
+
|
18
|
+
let(:dummy_yaml) { YAML.dump(dummy_val) }
|
19
|
+
let(:mock_s3_obj) do
|
20
|
+
mock(:s3_object, {
|
21
|
+
read: dummy_yaml,
|
22
|
+
versions: mock('versions', count: 3),
|
23
|
+
:exists? => true
|
24
|
+
})
|
25
|
+
end
|
26
|
+
|
27
|
+
before do
|
28
|
+
ClusterFsck::Reader.any_instance.stub(:s3_object).and_return(mock_s3_obj)
|
29
|
+
end
|
30
|
+
|
31
|
+
describe "with a key" do
|
32
|
+
let(:args) { [key] }
|
33
|
+
|
34
|
+
it "should open the yaml version of the key in an editor" do
|
35
|
+
subject.should_receive(:ask_editor).with(dummy_yaml).and_return(dummy_yaml)
|
36
|
+
subject.stub(:writer).and_return(mock_writer)
|
37
|
+
mock_writer.should_receive(:set).with(dummy_val, mock_s3_obj.versions.count).once
|
38
|
+
|
39
|
+
subject.run_command(args)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should error when it is locally overriden" do
|
43
|
+
subject.reader.stub(:has_local_override?).and_return(true)
|
44
|
+
->() { subject.run_command(args) }.should raise_error(ArgumentError)
|
45
|
+
end
|
46
|
+
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|