etcdist 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 31d1efd6790453a5cc15c0c2400d7e3d6c629865
+  data.tar.gz: ad13049f8d7a5c640a1be8e52db5888af619c870
+SHA512:
+  metadata.gz: 7f638b9482640267f0b4f0e5172568705d7c1fea319613c981f737afc3d8c11adcf786782d1d5a2b578fde43708a78f906a6aa88455d6cbfef16e44c427e89e2
+  data.tar.gz: a8f872b5fd43654733b24db3b4e76e7e7e3d9f210ad21f309c6ef01b2ece0251aae412fa1d5c8e04602bb4cce461acf5605a48a76cf73a2e5192f052d9aa48bc

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rubocop.yml

@@ -0,0 +1,15 @@
+AllCops:
+  Include:
+    - lib/**/*.rb
+    - spec/**/*.rb
+
+Documentation:
+  Exclude:
+    - lib/**/version.rb
+    - spec/**/*
+
+Style/EachWithObject:
+  Enabled: false
+
+LineLength:
+  Max: 120

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in etcdist.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,6 @@
+# More info at https://github.com/guard/guard#readme
+guard :rspec, cmd: 'bundle exec rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { 'spec' }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Springer, Part of Springer Science+Business Media
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+# Etcdist
+
+This is a small gem that helps you manage your etcd keys and values in a way that's easy to version control.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'etcdist'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install etcdist
+
+## Usage
+
+### Create the data
+
+Create your desired directory structure. This will be replicated in etcd. For example:
+
+```bash
+mkdir -p data/foo/bar
+```
+
+Then create a file in the leaf directory containing the keys and values that you want to go into etcd. For example:
+
+```bash
+cat <<EOT > data/foo/bar/food.data
+fish=plankton
+cows=grass
+EOT
+```
+
+The name of the file containing the keys and values doesn't matter. In fact, you can split the data into multiple files, if you want. An example would look like:
+
+```text
+./data
+└── ./foo
+    └── ./bar
+        ├── food.data    # contains fish=plankton and cows=grass
+        └── more.data    # could contain more keys and values
+```
+
+### Populate etcd
+
+Then pass the path to your data directory to Etcdist. For example:
+
+```ruby
+#!/usr/bin/env ruby
+require 'etcdist'
+Etcdist.execute('data')
+```
+
+If you want Etcdist to remove data from etcd that is no longer in your data files, use `dangerous` mode:
+
+```ruby
+Etcdist.execute('data', dangerous: true)
+```
+
+If you just want to perform a trial run without making any changes, then use the `dry_run` mode:
+
+```ruby
+Etcdist.execute('data', dangerous: true, dry_run: true)
+```
+
+## Configuration
+
+### Etcd host
+
+```ruby
+Etcdist.execute(dir) # defaults to localhost:4001
+Etcdist.execute(dir, host: '127.0.0.1', port: 4003)
+```
+
+### Log level
+
+You can control the log level, as follows:
+
+```ruby
+Etcdist::Log.level = :info
+```
+
+## Developing
+
+Clone the source code. To see what's possible, run:
+
+    $ rake -T
+
+To get the acceptance test to pass, make sure you've got etcd running locally:
+
+    $ docker run -d -p 4001:4001 coreos/etcd
+
+To continuously run tests, run:
+
+    $ guard
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,23 @@
+require 'bundler/gem_tasks'
+require 'rdoc/task'
+require 'rubocop/rake_task'
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+task default: :test
+
+RuboCop::RakeTask.new do |task|
+  task.fail_on_error = true
+end
+
+RDoc::Task.new do |rdoc|
+  rdoc.main = 'README.rdoc'
+  rdoc.rdoc_files.include('lib   /*.rb')
+end
+
+desc 'Run specs and code checks.'
+task :test do
+  Rake::Task['spec'].invoke
+  Rake::Task['rubocop'].invoke
+end

data/TODO

@@ -0,0 +1,9 @@
+# TODO
+* think about / improve error handling (for example: write operation is not atomic)
+* gemspec: specify required ruby version
+
+## Lower priority
+* feature: **optionally** remove directories from etcd if no longer present
+* feature: do not process directories that don't have any modified files
+* feature: provide etcdist as a binary so that it can be executed from the command line.
+

data/etcdist.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'etcdist/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'etcdist'
+  spec.version       = Etcdist::VERSION
+  spec.authors       = ['Springer, Part of Springer Science+Business Media']
+  spec.summary       = 'Populate etcd in a reproducable way.  '
+  spec.homepage      = 'https://github.com/springersbm/etcdist'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 2.0'
+
+  spec.add_runtime_dependency 'etcd', '~> 0.2.4'
+  spec.add_runtime_dependency 'mixlib-log'
+
+  spec.add_development_dependency 'bundler', '~> 1.7'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec-mocks', '~> 3.0'
+  spec.add_development_dependency 'guard-rspec'
+  spec.add_development_dependency 'rubocop'
+end

data/lib/etcdist/log.rb

@@ -0,0 +1,8 @@
+require 'mixlib/log'
+
+module Etcdist
+  # A wrapper class that extends Mixlib::Log
+  class Log
+    extend Mixlib::Log
+  end
+end

data/lib/etcdist/reader.rb

@@ -0,0 +1,26 @@
+require 'etcdist/log'
+
+module Etcdist
+  # Reads data from file system into directories, keys and values.
+  class Reader
+    # @param [String] dir The path to the data directory
+    def initialize(dir)
+      @dir = dir
+    end
+
+    # Returns a hash of type { directory => { key => val } }
+    def read
+      @dir = File.expand_path(@dir)
+      files = Dir[File.join(@dir, '**', '*')].reject { |p| File.directory? p }
+      Log.info("found #{files.length} files in #{@dir}")
+
+      files.reduce(Hash.new { |h, k| h[k] = {} }) do |h, f|
+        directory = File.dirname(f).gsub(@dir, '')
+        entries = Hash[IO.readlines(f).map { |e| e.chomp.split('=') }]
+        Log.debug("found #{entries.length} entries in #{f.gsub(@dir, '')}: #{entries}")
+        h[directory].merge!(entries)
+        h
+      end
+    end
+  end
+end

data/lib/etcdist/version.rb

@@ -0,0 +1,3 @@
+module Etcdist
+  VERSION = '0.0.3'
+end

data/lib/etcdist/writer.rb

@@ -0,0 +1,47 @@
+require 'etcdist/log'
+
+module Etcdist
+  # Writes data into etcd.
+  class Writer
+    # @param [Hash] opts Options
+    # @opts [String] :dangerous remove data from etcd (default false)
+    # @opts [String] :dry_run perform a trial run with no changes made (default false)
+    def initialize(etcd, opts = {})
+      @etcd = etcd
+      @dangerous = opts[:dangerous] || false
+      @dry_run = opts[:dry_run] || false
+    end
+
+    # Writes data into etcd and optionally removes extra data from etcd
+    # @param [Hash] data of type { directory => { key => val } }
+    def write(data)
+      count = { put: 0, del: 0 }
+      data.each do |dir, entries|
+        count[:put] += put(dir, entries)
+        count[:del] += delete(dir, entries) if @dangerous
+      end
+      Log.info("#{count[:put]} entries added/modified. #{count[:del]} entries deleted.")
+    end
+
+    private
+
+    def put(dir, entries)
+      existing = entries_in(dir)
+      to_put = entries.select { |k, v| existing[k] != v }
+      to_put.each { |k, v| @etcd.set([dir, '/', k].join, value: v) } unless @dry_run
+      Log.debug("wrote #{to_put.length} entries to #{dir}: #{to_put}")
+      to_put.length
+    end
+
+    def delete(dir, entries)
+      to_delete = entries_in(dir).keys - entries.keys
+      to_delete.each { |k| @etcd.delete([dir, '/', k].join) } unless @dry_run
+      Log.debug("deleted #{to_delete.length} entries from #{dir}: #{to_delete}")
+      to_delete.length
+    end
+
+    def entries_in(dir)
+      @etcd.exists?(dir) ? Hash[@etcd.get(dir).children.map { |n| [n.key.sub(/.*\//, ''), n.value] }] : {}
+    end
+  end
+end

data/lib/etcdist.rb

@@ -0,0 +1,25 @@
+require 'etcd'
+require 'etcdist/log'
+require 'etcdist/reader'
+require 'etcdist/writer'
+
+# The Etcdist name space
+module Etcdist
+  Log.level = :info
+
+  # Main entry point to read data from F/S and write into etcd.
+  #
+  # @param [String] dir The path to the data directory
+  # @param [Hash] opts Options
+  # @opts [String] :host IP address of the etcd server (default 127.0.0.1)
+  # @opts [Fixnum] :port Port number of the etcd server (default 4001)
+  def self.execute(dir, opts = {})
+    etcd = Etcd::Client.new(opts)
+    reader = Etcdist::Reader.new(dir)
+    writer = Etcdist::Writer.new(etcd, opts)
+
+    Log.info("using etcd host at: #{etcd.host}:#{etcd.port}")
+    writer.write(reader.read)
+    Log.info('finished successfully.')
+  end
+end

data/spec/data/etcdist/foo/one.config

@@ -0,0 +1,2 @@
+fish=plankton
+cows=grass

data/spec/data/etcdist/foo/two.config

@@ -0,0 +1 @@
+dogs=bones

data/spec/etcdist/reader_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Etcdist::Reader do
+
+  let(:reader) do
+    dir = File.join(File.dirname(__FILE__), '../data')
+    Etcdist::Reader.new(dir)
+  end
+
+  it 'should return directories pointing to key/values' do
+    expect(reader.read).to match('/etcdist/foo' => { 'fish' => 'plankton', 'cows' => 'grass', 'dogs' => 'bones' })
+  end
+
+end

data/spec/etcdist/writer_spec.rb

@@ -0,0 +1,84 @@
+require 'spec_helper'
+
+describe Etcdist::Writer do
+
+  let(:etcd) do
+    double('etcd').as_null_object
+  end
+
+  let(:writer) do
+    Etcdist::Writer.new(etcd)
+  end
+
+  describe 'PUTs' do
+
+    it 'should put entries in etcd' do
+      pretend_etcd_contains(nothing)
+      expect(etcd).to receive(:set).with('/foo/bar/fish', value: 'plankton')
+      writer.write('/foo/bar' => { 'fish' => 'plankton' })
+    end
+
+    it 'should put entries in correct directory' do
+      pretend_etcd_contains(nothing)
+      expect(etcd).to receive(:set).with('/water/fish', value: 'plankton')
+      expect(etcd).to receive(:set).with('/land/cows', value: 'grass')
+      writer.write('/water' => { 'fish' => 'plankton' }, '/land' => { 'cows' => 'grass' })
+    end
+
+    it 'should not put unchanged entries' do
+      pretend_etcd_contains('/foo' => { 'fish' => 'plankton', 'cows' => 'grass' })
+      expect(etcd).not_to receive(:set)
+      writer.write('/foo' => { 'fish' => 'plankton' })
+    end
+
+    context 'dry run mode' do
+      let(:writer) do
+        Etcdist::Writer.new(etcd, dry_run: true)
+      end
+
+      it 'should not put entries in etcd' do
+        pretend_etcd_contains(nothing)
+        expect(etcd).not_to receive(:set)
+        writer.write( '/foo/bar' => { 'fish' => 'plankton' } )
+      end
+    end
+  end
+
+  describe 'DELETEs' do
+
+    it 'should not delete extra entries by default' do
+      pretend_etcd_contains('/foo' => { 'fish' => 'plankton' })
+      expect(etcd).not_to receive(:delete)
+      writer.write('/foo' => {})
+    end
+
+    context 'dangerous mode' do
+
+      let(:writer) do
+        Etcdist::Writer.new(etcd, dangerous: true)
+      end
+
+      it 'should delete extra entries' do
+        pretend_etcd_contains('/foo' => { 'fish' => 'plankton' })
+        expect(etcd).to receive(:delete).with('/foo/fish')
+        writer.write('/foo' => { 'cows' => 'grass' })
+      end
+
+    end
+
+    context 'dry run mode' do
+      let(:writer) do
+        Etcdist::Writer.new(etcd, dangerous: true, dry_run: true)
+      end
+
+      it 'should not delete entries' do
+        pretend_etcd_contains('/foo' => { 'fish' => 'plankton' })
+        expect(etcd).not_to receive(:delete)
+        writer.write( '/foo' => { 'cows' => 'grass'} )
+      end
+
+    end
+
+  end
+
+end

data/spec/etcdist_spec.rb

@@ -0,0 +1,26 @@
+require 'spec_helper'
+
+describe Etcdist do
+
+  describe 'acceptance test - expects etcd running on localhost:4001' do
+
+    let(:etcd_opts) do
+      { host: 'localhost', port: 4001 }
+    end
+
+    let(:etcd) do
+      Etcd.client(etcd_opts)
+    end
+
+    let(:dir) do
+      File.join(File.dirname(__FILE__), 'data')
+    end
+
+    it 'should read data and put entries to etcd' do
+      Etcdist.execute(dir, etcd_opts)
+      expect(etcd.get('/etcdist/foo/fish').value).to eq('plankton')
+    end
+
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift(File.expand_path('../lib', __FILE__))
+$LOAD_PATH.unshift(File.expand_path('../spec', __FILE__))
+
+require 'etcdist'
+require 'etcdist/log'
+
+Etcdist::Log.level = :warn
+
+module Etcdist
+  module SpecHelper
+    def nothing
+      {}
+    end
+
+    def pretend_etcd_contains(data = nothing)
+      allow(etcd).to receive(:exists?).with(any_args).and_return(!data.empty?)
+      data.each do |dir, entries|
+        nodes = entries.map { |k, v| { 'key' => k, 'value' => v } }
+        opts = { 'node' => { 'dir' => true, 'nodes' => nodes } }
+        allow(etcd).to receive(:get).with(dir).and_return(Etcd::Response.new(opts))
+      end
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Etcdist::SpecHelper
+  config.color = true
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

metadata

@@ -0,0 +1,181 @@
+--- !ruby/object:Gem::Specification
+name: etcdist
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Springer, Part of Springer Science+Business Media
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: etcd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.2.4
+- !ruby/object:Gem::Dependency
+  name: mixlib-log
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-mocks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rubocop.yml
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- TODO
+- etcdist.gemspec
+- lib/etcdist.rb
+- lib/etcdist/log.rb
+- lib/etcdist/reader.rb
+- lib/etcdist/version.rb
+- lib/etcdist/writer.rb
+- spec/data/etcdist/foo/one.config
+- spec/data/etcdist/foo/two.config
+- spec/etcdist/reader_spec.rb
+- spec/etcdist/writer_spec.rb
+- spec/etcdist_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/springersbm/etcdist
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Populate etcd in a reproducable way.
+test_files:
+- spec/data/etcdist/foo/one.config
+- spec/data/etcdist/foo/two.config
+- spec/etcdist/reader_spec.rb
+- spec/etcdist/writer_spec.rb
+- spec/etcdist_spec.rb
+- spec/spec_helper.rb