clusterfsck 0.1.5.0

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea
+.DS_Store
+

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.rvmrc

@@ -0,0 +1 @@
+rvm 1.9.3@clusterfsck --create

data/Gemfile

@@ -0,0 +1,12 @@
+source 'http://rubygems.org'
+
+gemspec
+
+group :test do
+  gem 'guard-rspec'
+  gem 'rb-fsevent', '~> 0.9.1'
+  gem 'rake'
+  gem 'fakes3'
+  gem 'pry'
+  gem 'ci_reporter', '~> 1.7.3'
+end

data/Guardfile

@@ -0,0 +1,24 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+
+  # Rails example
+  watch(%r{^app/(.+)\.rb$})                           { |m| "spec/#{m[1]}_spec.rb" }
+  watch(%r{^app/(.*)(\.erb|\.haml)$})                 { |m| "spec/#{m[1]}#{m[2]}_spec.rb" }
+  watch(%r{^app/controllers/(.+)_(controller)\.rb$})  { |m| ["spec/routing/#{m[1]}_routing_spec.rb", "spec/#{m[2]}s/#{m[1]}_#{m[2]}_spec.rb", "spec/acceptance/#{m[1]}_spec.rb"] }
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('config/routes.rb')                           { "spec/routing" }
+  watch('app/controllers/application_controller.rb')  { "spec/controllers" }
+
+  # Capybara features specs
+  watch(%r{^app/views/(.+)/.*\.(erb|haml)$})          { |m| "spec/features/#{m[1]}_spec.rb" }
+
+  # Turnip features and steps
+  watch(%r{^spec/acceptance/(.+)\.feature$})
+  watch(%r{^spec/acceptance/steps/(.+)_steps\.rb$})   { |m| Dir[File.join("**/#{m[1]}.feature")][0] || 'spec/acceptance' }
+end
+

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Topper Bowers
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,121 @@
+# ClusterFsck
+
+ClusterFsck manages configuration across any number of projects or servers by reading and writing simple YAML stored on Amazon S3. It also allows overriding these configs locally through files (e.g. so your tests don't need network).
+
+(not tested on windows, patches accepted)
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'clusterfsck'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install clusterfsck
+
+
+## Configuration & Setup
+
+#### From the command line
+Type `clusterfsck init` to initiate ClusterFsck's setup assistant, or manually create a YAML file called `.clusterfsck` in your user's home directory or the root of your project with a `cluster_fsck_bucket` key set to the name of the bucket you want to use to store configuration. ClusterFsck will also need your aws credentials. Store those in the .clusterfsck file or pass those in using environment variables. In production, we recommend you use [IAM role-based access](http://aws.typepad.com/aws/2012/06/iam-roles-for-ec2-instances-simplified-secure-access-to-aws-service-apis-from-ec2.html).
+
+Now, you can run `clusterfsck new <config name>` to create your first clusterfsck managed configuration.  It will automatically call the clusterfsck edit command for you. Future edits should be done with `clusterfsck edit <config name>`, using the editor defined in `$EDITOR`.  It raises an error if no config name is provided. See Usage below for accessing the config from code.
+
+Like Rails, clusterfsck defaults `CLUSTER_FSCK_ENV` to `development` but the environment can be set to any string. There is one 'special' environment `shared` that is where default configs are stored. If a config does not exist in the environment you request it, then clusterfsck will look in the shared environment and only if it does *not* exist, will it error.
+
+For example:
+`$> clusterfsck edit shared myconfig`
+edit... edit... edit...
+```ruby
+ENV['CLUSTERFSCK_ENV'] = 'production'
+ClusterFsck::Reader.new(:myconfig).read #will be anything you set in the shared myconfig
+```
+
+The command line client also provides several additional commands, including list, which lists config files in the current or specified `list`, and `override` which will copy down the files for the environment, and use those local files in place of the S3 based configuration.  You can use this option to develop locally without internet access. In general, the command line uses the default environment, but allows specifying an environment as the first argument.
+
+You can run `clusterfsck --help` to get usage, currently that outputs as follows:
+
+`$> clusterfsck --help`
+```bash
+  NAME:
+
+    clusterfsck
+
+  DESCRIPTION:
+
+    Filebased S3 Cofiguration Kontrol for Clusters, by Amicus
+
+  COMMANDS:
+
+    edit                 Bring up the YAML for key specified in the current CLUSTER_FSCK_ENV or specified CLUSTER_FSCK_ENV in your $editor
+    environments         List all environments defined in the bucket ClusterFsck is setup using
+    help                 Display global or [command] help documentation.
+    init                 Create ClusterFsck configuration file - called automatically from other commands if no config found.
+    list                 List all keys in the current or specified CLUSTER_FSCK_ENV
+    new                  Create a yaml file for your current CLUSTER_FSCK_ENV or specified CLUSTER_FSCK_ENV
+    override             Will copy down the remote config and create a directory clusterfsck/:CLUSTER_FSCK_ENV/:key that will be used by default over the remote
+
+  GLOBAL OPTIONS:
+
+    -h, --help
+        Display help documentation
+
+    -v, --version
+        Display version information
+
+    -t, --trace
+        Display backtrace when an error occurs
+```
+
+Please help us improve this documentation. Pull requests or feedback are very appreciated!
+
+## Usage
+
+### setup your credential file
+You should probably run automated setup as a first step, but ClusterFsck checks for all its settings first in
+ENV variables as below.  For AWS keys, both access and secret keys must be defined or neither will be used (which is awesome in production, use IAM roles!):
+
+You may also define ClusterFsck's configuration using environment variables:
+```bash
+  ENV['CLUSTER_FSCK_BUCKET']
+  ENV['CLUSTER_FSCK_ENV']
+  ENV['AWS_ACCESS_KEY_ID']
+  ENV['AWS_SECRET_ACCESS_KEY']
+```
+
+After checking ENV variables, clusterfsck will look in './.clusterfsck','/usr/clusterfsck', or '~/.clusterfsck'] for the following yaml.
+
+```yaml
+:aws_access_key_id: access_key
+:aws_secret_access_key: secret_key
+:cluster_fsck_bucket: bucket_name
+:cluster_fsck_env: environment //probably development, production, staging or shared
+```
+
+If you do not define AWS in the environment variables, or the .clusterfsck file, then clusterfsck will check in a ~/.fog file for something that looks like this:
+
+```yaml
+:default:
+  :aws_access_key_id: access_key
+  :aws_secret_access_key: secret_key
+```
+
+### From Code
+
+```ruby
+reader = ClusterFsck::Reader.new(:stripe)
+reader.read[:api_key] # loads config_bucket/cluster_fsck_env/stripe and returns the api_key from the hash
+```
+
+The ClusterFsck::Reader instance will load the configuration for the environment stored in the CLUSTER_FSCK_ENV.
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,10 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+begin
+  require 'ci/reporter/rake/rspec'
+rescue LoadError => ex
+  puts "Failed to load the ci_reporter gem. Make sure it is available in your loadpaths"
+end

data/bin/clusterfsck

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+require 'commander/import'
+
+$LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), "../lib"))
+require "clusterfsck/cli"
+
+program :version, ClusterFsck::VERSION
+program :description, 'Filebased S3 Cofiguration Kontrol for Clusters, by Amicus'
+
+#TODO: (topper) - fill in the help stuff
+
+command :'init' do |c|
+  c.syntax = 'clusterfsck init'
+  c.summary = 'Create ClusterFsck configuration file - called automatically from other commands if no config found.'
+  c.description = 'clusterfsck init'
+  c.when_called ClusterFsck::Commands::Setup, :run_command
+end
+
+command :edit do |c|
+  c.syntax = 'clusterfsck edit [options]'
+  c.summary = 'Bring up the YAML for key specified in the current CLUSTER_FSCK_ENV or specified CLUSTER_FSCK_ENV in your $editor'
+  c.description = 'clusterfsck edit [cluster_fsck_env] key'
+  c.example 'edit stripe in current CLUSTER_FSCK_ENV', 'clusterfsck edit stripe'
+  c.example 'edit stripe in staging CLUSTER_FSCK_ENV', 'clusterfsck edit staging stripe'
+  c.option '--force', 'force edit even if local file override present'
+  c.when_called ClusterFsck::Commands::Edit, :run_command
+end
+
+command :new do |c|
+  c.syntax = 'clusterfsck new [options]'
+  c.summary = 'Create a yaml file for your current CLUSTER_FSCK_ENV or specified CLUSTER_FSCK_ENV'
+  c.description = 'clusterfsck new [cluster_fsck_env] key'
+  c.example 'create stripe key in current CLUSTER_FSCK_ENV', 'clusterfsck new stripe'
+  c.example 'create stripe key in staging CLUSTER_FSCK_ENV', 'clusterfsck new staging stripe'
+  c.when_called ClusterFsck::Commands::Create, :run_command
+end
+
+command :list do |c|
+  c.syntax = 'clusterfsck list'
+  c.summary = 'List all keys in the current or specified CLUSTER_FSCK_ENV'
+  c.description = 'clusterfsck list [cluster_fsck_env]'
+  c.example 'list keys in staging CLUSTER_FSCK_ENV', 'clusterfsck list staging'
+  c.when_called ClusterFsck::Commands::List, :run_command
+end
+
+command :environments do |c|
+  c.syntax = 'clusterfsck environments'
+  c.summary = 'List all environments defined in the bucket ClusterFsck is setup using'
+  c.description = 'clusterfsck environments [cluster_fsck_env]'
+  c.example 'list all environments defined', 'clusterfsck environments'
+  c.when_called ClusterFsck::Commands::Environments, :run_command
+end
+
+command :override do |c|
+  c.syntax = 'clusterfsck override [options]'
+  c.summary = 'Will copy down the remote config and create a directory clusterfsck/:CLUSTER_FSCK_ENV/:key that will be used by default over the remote'
+  c.description = ''
+  c.example 'override the staging stripe key in this working directory', 'clusterfsck override staging stripe'
+  c.when_called ClusterFsck::Commands::Override, :run_command
+end

data/clusterfsck.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'clusterfsck/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "clusterfsck"
+  gem.version       = ClusterFsck::VERSION
+  gem.authors       = ["Topper Bowers, Brian Glusman"]
+  gem.email         = ["topper@amicushq.com, brian@amicushq.com"]
+  gem.description   = "cluster-aware S3 based config getter/setter"
+  gem.summary       = %Q{
+    ClusterFsck manages configurable settings and sensitive login information
+    across any number of separate or related projects by reading writing simple
+    values or arrays/hashes of data to and from YAML files stored on Amazon S3.
+    It also allows centrally overriding or changing these on a per environment
+    basis.
+  }
+  gem.homepage      = "http://github.com/amicus/clusterfsck"
+
+  # we need a big version here, because we rely on a change made in january
+  # that was breaking EC2 provider credentials
+  gem.add_dependency "aws-sdk", "~> 1.8.1.2"
+  gem.add_dependency "hashie"
+  gem.add_dependency "random-word"
+  gem.add_dependency "commander", "~> 4.1.0"
+
+  gem.add_development_dependency "rspec"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/lib/clusterfsck.rb

@@ -0,0 +1,48 @@
+require 'clusterfsck/version'
+require 'clusterfsck/setup'
+require 'yaml'
+require 'aws-sdk'
+require 'hashie'
+require 'random-word'
+require 'commander'
+
+module ClusterFsck
+  CLUSTER_FSCK_PATHS = ['./.clusterfsck','/usr/clusterfsck','~/.clusterfsck']
+
+  CLUSTER_FSCK_CONFIG = CLUSTER_FSCK_PATHS.map do |path_string|
+    path = File.expand_path(path_string)
+    YAML.load_file(path) if File.exists?(path)
+  end.compact.first || {}
+
+  def self.logger=(logger)
+    @logger = logger
+  end
+
+  def self.logger
+    @logger ||= Logger.new($stdout)
+  end
+
+  def self.cluster_fsck_env
+    raise "Configuration failure, check ~/.clusterfsck or other config values" unless config_bucket
+    @env ||= ENV['CLUSTER_FSCK_ENV'] || CLUSTER_FSCK_CONFIG['cluster_fsck_env'] || default_env
+  end
+
+  def self.config_bucket
+    @config_bucket ||= ENV['CLUSTER_FSCK_BUCKET'] || CLUSTER_FSCK_CONFIG['cluster_fsck_bucket'] || Setup.config
+  end
+
+  def self.default_env
+    ENV['CLUSTER_FSCK_ENV'] || 'development'
+  end
+
+  def self.config_hash
+    CLUSTER_FSCK_CONFIG
+  end
+
+end
+
+require 'clusterfsck/credential_grabber'
+require 'clusterfsck/s3_methods'
+require 'clusterfsck/configuration'
+require 'clusterfsck/reader'
+require 'clusterfsck/writer'

data/lib/clusterfsck/cli.rb

@@ -0,0 +1,8 @@
+require_relative "../clusterfsck"
+require_relative "commands/cluster_fsck_env_argument_parser"
+require_relative "commands/edit"
+require_relative "commands/create"
+require_relative "commands/list"
+require_relative "commands/override"
+require_relative "commands/setup"
+require_relative "commands/environments"

data/lib/clusterfsck/commands/cluster_fsck_env_argument_parser.rb

@@ -0,0 +1,21 @@
+module ClusterFsck
+  module Commands
+    module ClusterFsckEnvArgumentParser
+
+      def self.included(base)
+        base.send(:attr_reader, :key) unless base.respond_to?(:reader_key)
+        base.send(:include, ClusterFsck::S3Methods)
+      end
+
+      def set_cluster_fsck_env_and_key_from_args(args)
+        if args.length > 1
+          @cluster_fsck_env, @key = args
+        else
+          @key = args.first
+        end
+      end
+
+
+    end
+  end
+end

data/lib/clusterfsck/commands/create.rb

@@ -0,0 +1,20 @@
+require 'commander'
+require_relative "edit"
+
+module ClusterFsck
+  module Commands
+    class Create
+      include ClusterFsckEnvArgumentParser
+
+      def run_command(args, options = {})
+        raise ArgumentError, "must provide a project name" if args.empty?
+        set_cluster_fsck_env_and_key_from_args(args)
+        obj = s3_object(key)
+        raise ConflictError, "#{key} already exists!" if obj.exists?
+        obj.write('')
+        Edit.new.run_command(args)
+      end
+
+    end
+  end
+end

data/lib/clusterfsck/commands/edit.rb

@@ -0,0 +1,30 @@
+require 'commander'
+
+module ClusterFsck
+  module Commands
+    class Edit
+      include Commander::UI
+      include ClusterFsckEnvArgumentParser
+
+      def run_command(args, options = Hashie::Mash.new)
+        raise ArgumentError, "must provide a project name" if args.empty?
+        set_cluster_fsck_env_and_key_from_args(args)
+
+        @options = options
+        raise ArgumentError, "File #{key} is overridden locally! use --force to force" if reader.has_local_override? and !options.force
+
+        new_yaml = ask_editor(YAML.dump(reader.read(remote_only: true).to_hash))
+        writer.set(Configuration.from_yaml(new_yaml), reader.version_count)
+      end
+
+      def writer
+        @writer ||= ClusterFsck::Writer.new(key, cluster_fsck_env: reader.cluster_fsck_env)
+      end
+
+      def reader
+        @reader ||= ClusterFsck::Reader.new(key, cluster_fsck_env: cluster_fsck_env)
+      end
+
+    end
+  end
+end