chef_backup 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 3aaeca0a04fa4fe4ee8443b47d5f9295929d201d
-  data.tar.gz: 82f06bbc30bff3a57c235d98012f0a251498af9c
+SHA256:
+  metadata.gz: b4e824870ac3e10eb6ffaa5bedd55483721a1af6b15cb61c7cb58cc2ccbcf7d9
+  data.tar.gz: 73be0c4d4911cd7f2f238aa0c9bd93b67a5fe7fdbfeb90294a8bfbffdcc42d7d
 SHA512:
-  metadata.gz: 688f33e0046368ac2c35dcf7dce9ae9f474cf5e8d8628651c105af4c709769fe7728e5e4a8fd66d530f0e2ad82577039bd42874ffdb34b4a168b563045460b1f
-  data.tar.gz: 3c965dbc203fd45876c38d051cc2427ba67e9a8a97c38db7b947f85d2d4e277de620a06dd0fa9a7bf29f04eec327787006e17679e2b6140e84445f36e1e9dac0
+  metadata.gz: 718ef11067a9e995f10bef3ccee17ac176b42a9a08fd9c3eebad64856dd5fda41f63c3e75e82d5e22fbb37dff521c0dd8e231436a580ace946366fb7dc9f1bc3
+  data.tar.gz: 03d05f2b1fd933cbd1842fc3563392b63f91127697ab17ae066eaf7c19f094c9c14aaeb083b9bbcbd580db0eb6ed0a71a67b20371821466308a2e1103a8fcbfb

data/lib/chef_backup.rb

@@ -2,11 +2,11 @@
 #
 # All Rights Reserved
 
-require 'chef_backup/version'
-require 'chef_backup/exceptions'
-require 'chef_backup/config'
-require 'chef_backup/logger'
-require 'chef_backup/data_map'
-require 'chef_backup/helpers'
-require 'chef_backup/runner'
-require 'chef_backup/strategy'
+require "chef_backup/version"
+require "chef_backup/exceptions"
+require "chef_backup/config"
+require "chef_backup/logger"
+require "chef_backup/data_map"
+require "chef_backup/helpers"
+require "chef_backup/runner"
+require "chef_backup/strategy"

data/lib/chef_backup/config.rb

@@ -1,23 +1,23 @@
-require 'fileutils'
-require 'json'
-require 'forwardable'
+require "fileutils"
+require "json"
+require "forwardable"
 
 module ChefBackup
   # ChefBackup Global Config
   class Config
     extend Forwardable
 
-    DEFAULT_BASE = 'private_chef'.freeze
+    DEFAULT_BASE = "private_chef".freeze
     DEFAULT_CONFIG = {
-      'backup' => {
-        'always_dump_db' => true,
-        'strategy' => 'none',
-        'export_dir' => '/var/opt/chef-backup',
-        'project_name' => 'opscode',
-        'ctl-command' => 'chef-server-ctl',
-        'running_filepath' => '/etc/opscode/chef-server-running.json',
-        'database_name' => 'opscode_chef'
-      }
+      "backup" => {
+        "always_dump_db" => true,
+        "strategy" => "none",
+        "export_dir" => "/var/opt/chef-backup",
+        "project_name" => "opscode",
+        "ctl-command" => "chef-server-ctl",
+        "running_filepath" => "/etc/opscode/chef-server-running.json",
+        "database_name" => "opscode_chef",
+      },
     }.freeze
 
     class << self
@@ -50,11 +50,11 @@ module ChefBackup
     # @param config [Hash] a Hash of the private-chef-running.json
     #
     def initialize(config = {})
-      config['config_base'] ||= DEFAULT_BASE
-      base = config['config_base']
+      config["config_base"] ||= DEFAULT_BASE
+      base = config["config_base"]
       config[base] ||= {}
-      config[base]['backup'] ||= {}
-      config[base]['backup'] = DEFAULT_CONFIG['backup'].merge(config[base]['backup'])
+      config[base]["backup"] ||= {}
+      config[base]["backup"] = DEFAULT_CONFIG["backup"].merge(config[base]["backup"])
       @config = config
     end
 

data/lib/chef_backup/data_map.rb

@@ -1,4 +1,4 @@
-require 'time'
+require "time"
 
 module ChefBackup
   # DataMap class to store data about the data we're backing up
@@ -11,27 +11,32 @@ module ChefBackup
       attr_writer :data_map
     end
 
-    attr_accessor :strategy, :backup_time, :topology, :configs, :services, :ha
+    attr_accessor :strategy, :backup_time, :topology, :configs, :services, :ha, :versions
 
     def initialize
       @services = {}
       @configs = {}
+      @versions = {}
       @ha = {}
       yield self if block_given?
 
       @backup_time ||= Time.now.iso8601
-      @strategy ||= 'none'
-      @toplogy ||= 'idontknow'
+      @strategy ||= "none"
+      @toplogy ||= "idontknow"
     end
 
     def add_service(service, data_dir)
       @services[service] ||= {}
-      @services[service]['data_dir'] = data_dir
+      @services[service]["data_dir"] = data_dir
     end
 
     def add_config(config, path)
       @configs[config] ||= {}
-      @configs[config]['data_dir'] = path
+      @configs[config]["data_dir"] = path
+    end
+
+    def add_version(project_name, data)
+      @versions[project_name] = data
     end
 
     def add_ha_info(k, v)
@@ -40,12 +45,13 @@ module ChefBackup
 
     def manifest
       {
-        'strategy' => strategy,
-        'backup_time' => backup_time,
-        'topology' => topology,
-        'ha' => ha,
-        'services' => services,
-        'configs' => configs
+        "strategy" => strategy,
+        "backup_time" => backup_time,
+        "topology" => topology,
+        "ha" => ha,
+        "services" => services,
+        "configs" => configs,
+        "versions" => versions,
       }
     end
   end

data/lib/chef_backup/deep_merge.rb

@@ -0,0 +1,145 @@
+#
+# Author:: Adam Jacob (<adam@chef.io>)
+# Author:: Steve Midgley (http://www.misuse.org/science)
+# Copyright:: Copyright 2009-2016, Chef Software Inc.
+# Copyright:: Copyright 2008-2016, Steve Midgley
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'chef_backup/mash'
+
+#
+# DANGER! THIS FILE WAS VENDORDED FROM CHEF. IF YOU ARE
+# MAKING A CHANGE HERE, CONSIDER WHETHER IT IS REQUIRED IN CHEF ALSO?
+#
+module ChefBackup
+  module Mixin
+    # == Chef::Mixin::DeepMerge
+    # Implements a deep merging algorithm for nested data structures.
+    # ==== Notice:
+    #   This code was originally imported from deep_merge by Steve Midgley.
+    #   deep_merge is available under the MIT license from
+    #   http://trac.misuse.org/science/wiki/DeepMerge
+    module DeepMerge
+
+      extend self
+
+      def merge(first, second)
+        first  = Mash.new(first)  unless first.kind_of?(Mash)
+        second = Mash.new(second) unless second.kind_of?(Mash)
+
+        DeepMerge.deep_merge(second, first)
+      end
+
+      class InvalidParameter < StandardError; end
+
+      # Deep Merge core documentation.
+      # deep_merge! method permits merging of arbitrary child elements. The two top level
+      # elements must be hashes. These hashes can contain unlimited (to stack limit) levels
+      # of child elements. These child elements to not have to be of the same types.
+      # Where child elements are of the same type, deep_merge will attempt to merge them together.
+      # Where child elements are not of the same type, deep_merge will skip or optionally overwrite
+      # the destination element with the contents of the source element at that level.
+      # So if you have two hashes like this:
+      #   source = {:x => [1,2,3], :y => 2}
+      #   dest =   {:x => [4,5,'6'], :y => [7,8,9]}
+      #   dest.deep_merge!(source)
+      #   Results: {:x => [1,2,3,4,5,'6'], :y => 2}
+      # By default, "deep_merge!" will overwrite any unmergeables and merge everything else.
+      # To avoid this, use "deep_merge" (no bang/exclamation mark)
+      def deep_merge!(source, dest)
+        # if dest doesn't exist, then simply copy source to it
+        if dest.nil?
+          dest = source; return dest
+        end
+
+        case source
+        when nil
+          dest
+        when Hash
+          if dest.kind_of?(Hash)
+            source.each do |src_key, src_value|
+              if dest[src_key]
+                dest[src_key] = deep_merge!(src_value, dest[src_key])
+              else # dest[src_key] doesn't exist so we take whatever source has
+                dest[src_key] = src_value
+              end
+            end
+          else # dest isn't a hash, so we overwrite it completely
+            dest = source
+          end
+        when Array
+          if dest.kind_of?(Array)
+            dest = dest | source
+          else
+            dest = source
+          end
+        when String
+          dest = source
+        else # src_hash is not an array or hash, so we'll have to overwrite dest
+          dest = source
+        end
+        dest
+      end # deep_merge!
+
+      def hash_only_merge(merge_onto, merge_with)
+        hash_only_merge!(safe_dup(merge_onto), safe_dup(merge_with))
+      end
+
+      def safe_dup(thing)
+        thing.dup
+      rescue TypeError
+        thing
+      end
+
+      # Deep merge without Array merge.
+      # `merge_onto` is the object that will "lose" in case of conflict.
+      # `merge_with` is the object whose values will replace `merge_onto`s
+      # values when there is a conflict.
+      def hash_only_merge!(merge_onto, merge_with)
+        # If there are two Hashes, recursively merge.
+        if merge_onto.kind_of?(Hash) && merge_with.kind_of?(Hash)
+          merge_with.each do |key, merge_with_value|
+            value =
+              if merge_onto.has_key?(key)
+                hash_only_merge(merge_onto[key], merge_with_value)
+              else
+                merge_with_value
+              end
+
+            if merge_onto.respond_to?(:public_method_that_only_deep_merge_should_use)
+              # we can't call ImmutableMash#[]= because its immutable, but we need to mutate it to build it in-place
+              merge_onto.public_method_that_only_deep_merge_should_use(key, value)
+            else
+              merge_onto[key] = value
+            end
+          end
+          merge_onto
+
+        # If merge_with is nil, don't replace merge_onto
+        elsif merge_with.nil?
+          merge_onto
+
+        # In all other cases, replace merge_onto with merge_with
+        else
+          merge_with
+        end
+      end
+
+      def deep_merge(source, dest)
+        deep_merge!(safe_dup(source), safe_dup(dest))
+      end
+
+    end
+  end
+end

data/lib/chef_backup/helpers.rb

@@ -1,8 +1,10 @@
 require 'fileutils'
+require 'json'
 require 'mixlib/shellout'
 require 'chef_backup/config'
 require 'chef_backup/logger'
 
+# rubocop:disable ModuleLength
 # rubocop:disable IndentationWidth
 module ChefBackup
 # Common helper methods that are usefull in many classes
@@ -27,7 +29,7 @@ module Helpers
       'ctl_command' => 'opscode-analytics-ctl'
     },
     'chef-ha' => {
-      'config_file' => 'etc/opscode/chef-server.rb'
+      'config_file' => '/etc/opscode/chef-server.rb'
     },
     'chef-sync' => {
       'config_file' => '/etc/chef-sync/chef-sync.rb',
@@ -69,6 +71,19 @@ module Helpers
     ChefBackup::Logger.logger.log(message, level)
   end
 
+  # Note that when we are in the backup codepath, we have access to a running
+  # chef server and hence, the ctl command puts all our flags under the current
+  # running service namespace. The lets the default configuration of the server
+  # provide flags that the user doesn't necessarily provide on the command line.
+  #
+  # During the restore codepath, there may be no running chef server. This means
+  # that we need to be paranoid about the existence of the service_config hash.
+  def shell_timeout
+    option = config['shell_out_timeout'] ||
+             (service_config && service_config['backup']['shell_out_timeout'])
+    option.to_f unless option.nil?
+  end
+
   #
   # @param file [String] A path to a file on disk
   # @param exception [Exception] An exception to raise if file is not present
@@ -81,7 +96,9 @@ module Helpers
   end
 
   def shell_out(*command)
-    cmd = Mixlib::ShellOut.new(*command)
+    options = command.last.is_a?(Hash) ? command.pop : {}
+    opts_with_defaults = { 'timeout' => shell_timeout }.merge(options)
+    cmd = Mixlib::ShellOut.new(*command, opts_with_defaults)
     cmd.live_stream ||= $stdout.tty? ? $stdout : nil
     cmd.run_command
     cmd
@@ -101,6 +118,11 @@ module Helpers
     "/opt/#{project_name}"
   end
 
+  def addon_install_dir(name)
+    # can use extra field in SERVER_ADD_ONS to extend if someone isn't following this pattern.
+    "/opt/#{name}"
+  end
+
   def base_config_dir
     "/etc/#{project_name}"
   end
@@ -114,7 +136,9 @@ module Helpers
   end
 
   def pg_options
-    config['pg_options'] || DEFAULT_PG_OPTIONS
+    config['pg_options'] ||
+      (service_config && service_config['backup']['pg_options']) ||
+      DEFAULT_PG_OPTIONS
   end
 
   def all_services
@@ -126,7 +150,7 @@ module Helpers
   end
 
   def disabled_services
-    all_services.select { |sv| !service_enabled?(sv) }
+    all_services.reject { |sv| service_enabled?(sv) }
   end
 
   def service_enabled?(service)
@@ -177,12 +201,10 @@ module Helpers
   end
 
   def enabled_addons
-    SERVER_ADD_ONS.select do |_name, config|
-      begin
-        File.directory?(File.dirname(config['config_file']))
-      rescue
-        false
-      end
+    SERVER_ADD_ONS.select do |name, config|
+      !config['config_file'].nil? &&
+        File.directory?(File.dirname(config['config_file'])) &&
+        File.directory?(addon_install_dir(name))
     end
   end
 
@@ -242,6 +264,20 @@ module Helpers
     true
   end
 
+  def version_from_manifest_file(file)
+    return :no_version if file.nil?
+
+    path = File.expand_path(file)
+    if File.exist?(path)
+      config = JSON.parse(File.read(path))
+      { 'version' => config['build_version'],
+        'revision' => config['build_git_revision'],
+        'path' => path }
+    else
+      :no_version
+    end
+  end
+
   private
 
   def safe_key

data/lib/chef_backup/logger.rb

@@ -1,4 +1,4 @@
-require 'highline'
+require "highline"
 
 module ChefBackup
   # Basic Logging Class
@@ -15,7 +15,7 @@ module ChefBackup
     attr_accessor :stdout
 
     def initialize(logfile = nil)
-      $stdout = logfile ? File.open(logfile, 'ab') : $stdout
+      $stdout = logfile ? File.open(logfile, "ab") : $stdout
       @highline = HighLine.new($stdin, $stdout)
     end
 

data/lib/chef_backup/mash.rb

@@ -0,0 +1,226 @@
+# Copyright 2009-2016, Dan Kubb
+
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# ---
+# ---
+
+# Some portions of blank.rb and mash.rb are verbatim copies of software
+# licensed under the MIT license. That license is included below:
+
+# Copyright 2005-2016, David Heinemeier Hansson
+
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This class has dubious semantics and we only have it so that people can write
+# params[:key] instead of params['key'].
+class Mash < Hash
+
+  # @param constructor<Object>
+  #   The default value for the mash. Defaults to an empty hash.
+  #
+  # @details [Alternatives]
+  #   If constructor is a Hash, a new mash will be created based on the keys of
+  #   the hash and no default value will be set.
+  def initialize(constructor = {})
+    if constructor.is_a?(Hash)
+      super()
+      update(constructor)
+    else
+      super(constructor)
+    end
+  end
+
+  # @param orig<Object> Mash being copied
+  #
+  # @return [Object] A new copied Mash
+  def initialize_copy(orig)
+    super
+    # Handle nested values
+    each do |k, v|
+      if v.kind_of?(Mash) || v.is_a?(Array)
+        self[k] = v.dup
+      end
+    end
+    self
+  end
+
+  # @param key<Object> The default value for the mash. Defaults to nil.
+  #
+  # @details [Alternatives]
+  #   If key is a Symbol and it is a key in the mash, then the default value will
+  #   be set to the value matching the key.
+  def default(key = nil)
+    if key.is_a?(Symbol) && include?(key = key.to_s)
+      self[key]
+    else
+      super
+    end
+  end
+
+  alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+  alias_method :regular_update, :update unless method_defined?(:regular_update)
+
+  # @param key<Object> The key to set.
+  # @param value<Object>
+  #   The value to set the key to.
+  #
+  # @see Mash#convert_key
+  # @see Mash#convert_value
+  def []=(key, value)
+    regular_writer(convert_key(key), convert_value(value))
+  end
+
+  # @param other_hash<Hash>
+  #   A hash to update values in the mash with. The keys and the values will be
+  #   converted to Mash format.
+  #
+  # @return [Mash] The updated mash.
+  def update(other_hash)
+    other_hash.each_pair { |key, value| regular_writer(convert_key(key), convert_value(value)) }
+    self
+  end
+
+  alias_method :merge!, :update
+
+  # @param key<Object> The key to check for. This will be run through convert_key.
+  #
+  # @return [Boolean] True if the key exists in the mash.
+  def key?(key)
+    super(convert_key(key))
+  end
+
+  # def include? def has_key? def member?
+  alias_method :include?, :key?
+  alias_method :has_key?, :key?
+  alias_method :member?, :key?
+
+  # @param key<Object> The key to fetch. This will be run through convert_key.
+  # @param *extras<Array> Default value.
+  #
+  # @return [Object] The value at key or the default value.
+  def fetch(key, *extras)
+    super(convert_key(key), *extras)
+  end
+
+  # @param *indices<Array>
+  #   The keys to retrieve values for. These will be run through +convert_key+.
+  #
+  # @return [Array] The values at each of the provided keys
+  def values_at(*indices)
+    indices.collect { |key| self[convert_key(key)] }
+  end
+
+  # @param hash<Hash> The hash to merge with the mash.
+  #
+  # @return [Mash] A new mash with the hash values merged in.
+  def merge(hash)
+    self.dup.update(hash)
+  end
+
+  # @param key<Object>
+  #   The key to delete from the mash.\
+  def delete(key)
+    super(convert_key(key))
+  end
+
+  # @param *rejected<Array[(String, Symbol)] The mash keys to exclude.
+  #
+  # @return [Mash] A new mash without the selected keys.
+  #
+  # @example
+  #   { :one => 1, :two => 2, :three => 3 }.except(:one)
+  #     #=> { "two" => 2, "three" => 3 }
+  def except(*keys)
+    super(*keys.map { |k| convert_key(k) })
+  end
+
+  # Used to provide the same interface as Hash.
+  #
+  # @return [Mash] This mash unchanged.
+  def stringify_keys!; self end
+
+  # @return [Hash] The mash as a Hash with symbolized keys.
+  def symbolize_keys
+    h = Hash.new(default)
+    each { |key, val| h[key.to_sym] = val }
+    h
+  end
+
+  # @return [Hash] The mash as a Hash with string keys.
+  def to_hash
+    Hash.new(default).merge(self)
+  end
+
+  # @return [Mash] Convert a Hash into a Mash
+  # The input Hash's default value is maintained
+  def self.from_hash(hash)
+    mash = Mash.new(hash)
+    mash.default = hash.default
+    mash
+  end
+
+  protected
+
+  # @param key<Object> The key to convert.
+  #
+  # @param [Object]
+  #   The converted key. If the key was a symbol, it will be converted to a
+  #   string.
+  #
+  # @api private
+  def convert_key(key)
+    key.kind_of?(Symbol) ? key.to_s : key
+  end
+
+  # @param value<Object> The value to convert.
+  #
+  # @return [Object]
+  #   The converted value. A Hash or an Array of hashes, will be converted to
+  #   their Mash equivalents.
+  #
+  # @api private
+  def convert_value(value)
+    if value.class == Hash
+      Mash.from_hash(value)
+    elsif value.is_a?(Array)
+      value.collect { |e| convert_value(e) }
+    else
+      value
+    end
+  end
+end