leda 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 432c8ae222a8f9c8802fadf82f52794c756c3860
+  data.tar.gz: 48a6bdb44438075b512f8b6bf86a1e80ed265b48
+SHA512:
+  metadata.gz: ade3e8d94b1e082b218f459b70df6885c2de8811468fc373fd6d129ce72ed1f5e5e24ae34c660b775c6da731a33185afee39cfd81bab4590fbea13039fef44cf
+  data.tar.gz: 8fe10d3609461f86d1cd25c617e032398bbfd999a7dab98b6e3134916cf38ff4782408b4491743173d7d1d91dc4b010c70f4457021615396295aa0c5eaf20643

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/spec/tmp
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in leda.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Rhett Sutphin
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,91 @@
+# Leda
+
+Rake and capistrano tasks for clone data between Rails application environments.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'leda'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install leda
+
+## Usage
+
+### 60 second version
+
+#### Define config/leda.rb
+
+```
+Leda.configure do |leda|
+  leda.data_unit 'providers' do |du|
+    du.postgresql tables: %w(practices offices practitioners)
+  end
+end
+```
+
+#### Create rake tasks
+
+In `lib/tasks/data.rake`:
+
+```
+require 'leda'
+
+load File.expand_path('../../../config/leda.rb', __FILE__)
+
+namespace :data do
+  Leda.define_rake_tasks(:environment)
+end
+```
+
+#### Create cap tasks
+
+(For Capistrano 3 only.)
+
+In `lib/capistrano/tasks/data.cap`:
+
+```
+require 'leda'
+
+load File.expand_path('../../../../config/leda.rb', __FILE__)
+
+namespace :data do
+  Leda.define_capistrano_tasks('data')
+end
+```
+
+#### Dump data via rake or capistrano
+
+```
+$ bin/rake data:providers:dump
+$ bin/cap production data:providers:dump
+```
+
+#### Restore data via rake
+
+```
+bin/rake data:providers:restore_from[production]
+```
+
+## Limitations
+
+The current version of this library is tied to Rails (to discover the current
+environment name) and particular libraries (for database configuration). A goal
+for 1.0 would be to factor these things out so that Leda can be used in
+standalone scripts also.
+
+## Contributing
+
+1. Fork it ( https://github.com/humanpractice/leda/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/leda.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'leda/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "leda"
+  spec.version       = Leda::VERSION
+  spec.authors       = ["Rhett Sutphin"]
+  spec.email         = ["rhett@detailedbalance.net"]
+  spec.summary       = %q{Rake & capistrano tasks to clone partial data between Rails application environments.}
+  # spec.description   = %q{TODO: Write a longer description. Optional.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+
+  spec.add_dependency 'activesupport', '>= 3.2'
+
+  # For ES
+  spec.add_dependency 'oj'
+end

data/lib/leda.rb

@@ -0,0 +1,37 @@
+require "leda/version"
+
+module Leda
+  autoload :Capistrano, 'leda/capistrano'
+  autoload :Configuration, 'leda/configuration'
+  autoload :DataUnit, 'leda/data_unit'
+  autoload :Rake, 'leda/rake'
+  autoload :Runner, 'leda/runner'
+  autoload :Store, 'leda/store'
+
+  class << self
+    def configuration
+      @configuration || reset_configuration
+    end
+
+    def reset_configuration
+      @configuration = ::Leda::Configuration.new
+    end
+
+    ##
+    # Builds up the global Leda configuration using the configuration DSL.
+    #
+    # Multiple invocations will add to the existing configuration. Call
+    # {#reset_configuration} to clear if desired.
+    def configure(&dsl)
+      configuration.update(&dsl)
+    end
+
+    def define_rake_tasks(*prerequisites)
+      ::Leda::Rake.define_tasks(configuration, prerequisites)
+    end
+
+    def define_capistrano_tasks(rake_task_namespace)
+      ::Leda::Capistrano.define_tasks(configuration, rake_task_namespace)
+    end
+  end
+end

data/lib/leda/capistrano.rb

@@ -0,0 +1,84 @@
+module Leda
+  module Capistrano
+    extend self
+
+    ##
+    # Defines capstrano tasks which run a dump remotely, compress it, download
+    # it, and expand it locally.
+    #
+    # As with {{Rake.define_tasks}}, it's probably best to run this inside a
+    # namespace.
+    def define_tasks(configuration, rake_task_namespace='data')
+      configuration.data_units.each do |data_unit|
+        t = ::Rake::Task.define_task [data_unit.name, 'dump'].join(':') do
+          current_env = fetch(:stage).to_s
+          runner = Runner.new(current_env, configuration)
+          tarball_directory = runner.relative_directory(current_env, data_unit)
+          tarball_path = tarball_directory.to_s + ".tar.bz2"
+          paths = runner.dump_relative_paths(data_unit.name)
+
+          # Create local target directory
+          run_locally do
+            execute :mkdir, '-p', tarball_directory.to_s
+          end
+
+          # dump & compress on server
+          on roles(:app) do
+            within(current_path) do
+              execute :rake, [rake_task_namespace, data_unit.name, 'dump'].compact.join(':')
+              execute :tar, 'cjvf', tarball_path, paths
+
+              expected_remote_filename = "#{current_path}/#{tarball_path}"
+              download! expected_remote_filename, tarball_path
+
+              execute :rm, tarball_path
+            end
+          end
+
+          # decompress locally
+          run_locally do
+            execute :tar, 'xjvf', tarball_path
+          end
+        end
+        t.add_description "Dump & download the remote #{data_unit.name} data"
+      end
+
+      t = ::Rake::Task.define_task 'dump' => configuration.data_units.map { |du| [rake_task_namespace, du.name, 'dump'].compact.join(':') }
+      t.add_description "Dump and download all remote data units"
+    end
+  end
+end
+
+__END__
+
+namespace :data do
+  def execute_remote_dump(key, data_types)
+    paths_in_project = data_types.map { |ext| "db/data/#{key}.#{fetch :stage}.#{ext}" }
+    tarball_path = "db/data/#{key}.#{fetch :stage}.tar.bz2"
+
+    on roles(:app) do
+      within(current_path) do
+        execute :rake, "data:#{key}:dump"
+        execute :tar, 'cjvf', tarball_path, paths_in_project
+
+        expected_remote_filename = "#{current_path}/#{tarball_path}"
+        download! expected_remote_filename, tarball_path
+
+        execute :rm, tarball_path
+      end
+    end
+
+    # decompress locally
+    sh 'tar', 'xjvf', tarball_path
+  end
+
+  {
+    providers: %w(psql elasticsearch),
+    groups: %w(psql)
+  }.each do |key, data_types|
+    desc "Dump & download the remote #{key} data"
+    task "#{key}:dump" do
+      execute_remote_dump(key, data_types)
+    end
+  end
+end

data/lib/leda/configuration.rb

@@ -0,0 +1,91 @@
+require 'leda'
+
+require 'pathname'
+
+module Leda
+  class Configuration
+    attr_reader :base_path
+
+    def initialize(&dsl)
+      @data_units_map = {}
+      @base_path = Pathname.new('db/leda')
+
+      if block_given?
+        update(&dsl)
+      end
+    end
+
+    def update(&dsl)
+      dsl.call(self)
+      self
+    end
+
+    def data_unit(name, &dsl)
+      data_unit = (@data_units_map[name] ||= DataUnit.new(name))
+
+      if block_given?
+        dsl.call(DataUnitConfigurator.new(data_unit))
+      end
+
+      data_unit
+    end
+
+    def data_units
+      @data_units_map.values
+    end
+
+    def project_root_dir
+      @project_root_dir or fail "Please set project_root_dir in your Leda configuration"
+    end
+
+    def project_root_dir=(path)
+      @project_root_dir = ensure_pathname(path)
+    end
+
+    def base_path=(path)
+      @base_path = ensure_pathname(path)
+    end
+
+    def base_dir
+      project_root_dir + base_path
+    end
+
+    private
+
+    def ensure_pathname(path)
+      case path
+      when Pathname
+        path
+      when nil
+        nil
+      else
+        Pathname.new(path.to_s)
+      end
+    end
+
+    ##
+    # @private
+    class DataUnitConfigurator
+      attr_reader :target
+
+      def initialize(data_unit)
+        @target = data_unit
+      end
+
+      def add_store(store_class, options)
+        options ||= {}
+
+        target.stores << store_class.new(options)
+      end
+
+      def method_missing(name, *args)
+        store_class = Store.find(name)
+        if store_class
+          add_store(store_class, args.first)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/leda/data_unit.rb

@@ -0,0 +1,13 @@
+require 'leda'
+
+module Leda
+  class DataUnit
+    attr_reader :name
+    attr_reader :stores
+
+    def initialize(name)
+      @name = name
+      @stores = []
+    end
+  end
+end

data/lib/leda/rake.rb

@@ -0,0 +1,67 @@
+require 'leda'
+require 'rake'
+
+module Leda
+  module Rake
+    extend self
+
+    ##
+    # Defines aggregate and individual dump & restore tasks based on the given
+    # {{Configuration}}. In real life, you'll probably want to call this from
+    # inside a `namespace` block:
+    #
+    #     # In Rakefile
+    #     namespace 'data' do
+    #       Leda::Rake.define_tasks(Leda.configuration, [:environment])
+    #     end
+    def define_tasks(configuration, outside_prerequisites=[])
+      define_dump_task(configuration, outside_prerequisites,
+        "Dump all Leda-configured data", nil, nil)
+      define_restore_from_task(configuration, outside_prerequisites,
+        "Restore all Leda-configured data from the specified env", nil, nil)
+
+      configuration.data_units.each do |data_unit|
+        define_dump_task(configuration, outside_prerequisites,
+          "Dump all data for #{data_unit.name} from the current env",
+          data_unit.name, nil)
+        define_restore_from_task(configuration, outside_prerequisites,
+          "Restore all data for #{data_unit.name} from the specified env into the current env",
+          data_unit.name, nil)
+
+        data_unit.stores.each do |store|
+          define_dump_task(configuration, outside_prerequisites,
+            "Dump all data from #{store.name} for #{data_unit.name} from the current env",
+            data_unit.name, store.name)
+          define_restore_from_task(configuration, outside_prerequisites,
+            "Restore all data into #{store.name} for #{data_unit.name} from the specified env into the current env",
+            data_unit.name, store.name)
+        end
+      end
+    end
+
+    def create_runner(configuration)
+      # TODO: externalize configuration
+      ::Leda::Runner.new(Rails.env, configuration)
+    end
+
+    def define_dump_task(configuration, outside_prerequisites, description, data_unit_name, store_name)
+      task_name = [data_unit_name, store_name, 'dump'].compact.join(':')
+
+      t = ::Rake::Task.define_task(task_name => outside_prerequisites) do
+        create_runner(configuration).dump(data_unit_name, store_name)
+      end
+      t.add_description description
+    end
+
+    def define_restore_from_task(configuration, outside_prerequisites, description, data_unit_name, store_name)
+      task_name = [data_unit_name, store_name, 'restore_from'].compact.join(':')
+
+      t = ::Rake::Task.define_task(task_name, [:source_env] => outside_prerequisites) do |t, args|
+        source_env = args[:source_env] or fail "Please specify the source env name"
+
+        create_runner(configuration).restore_from(source_env, data_unit_name, store_name)
+      end
+      t.add_description description
+    end
+  end
+end