chef_backup 0.0.1.dev.4 → 0.2.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 94cbd6726cb86fe838709bb6cd0102f18ff98a02
-  data.tar.gz: 7a38dd28ef497ae4b1ec8a59adb22634e28f2fc4
+SHA256:
+  metadata.gz: 51de6280ecd6ede16153afcfb70d6fcfa508e76dfa48dea7858d5d5fcc4be278
+  data.tar.gz: a358070ef671054bda4e8b21bc6e6bb51fb915b31bded0567d4c717731fff0c5
 SHA512:
-  metadata.gz: 4f1c58f55e246331ab7cc8c9919f6e9dafaba9922d4c597e5546bd59701634e3766461d4e58f3efffcea09cf1267724e52d3f11b4b9fc6b37be68eecdc1664b4
-  data.tar.gz: af600e295428765fb2e8bd0316039108a1133c5fd7a82ea8df83b1c518019c965a4239935cc9625010522b2bc82ff23aab588ab4afc9f0f24e06a0ebac846478
+  metadata.gz: cbc70201b088ac22f49d34cfb6114eebfe62f8b2f8cca7fd3beb5255b54df77dddcdc76287f3474f284532b387a9297c8659c4eaf5c5a50444536579c7e98354
+  data.tar.gz: 96e4e39b9d0083c81a0ba2dbf009f32545e027aa5a5a5f0f3beb91eb34f7fae0a17f9493e1c766979ec73719ee5104c28aae8fa59d739141ba9d49a64d6aa805

data/lib/chef_backup/config.rb

@@ -1,18 +1,23 @@
-require 'fileutils'
-require 'json'
-require 'forwardable'
+require "fileutils" unless defined?(FileUtils)
+require "json" unless defined?(JSON)
+require "forwardable" unless defined?(Forwardable)
 
 module ChefBackup
   # ChefBackup Global Config
   class Config
     extend Forwardable
 
+    DEFAULT_BASE = "private_chef".freeze
     DEFAULT_CONFIG = {
-      'backup' => {
-        'always_dump_db' => true,
-        'strategy' => 'none',
-        'export_dir' => '/var/opt/chef-backup'
-      }
+      "backup" => {
+        "always_dump_db" => true,
+        "strategy" => "none",
+        "export_dir" => "/var/opt/chef-backup",
+        "project_name" => "opscode",
+        "ctl-command" => "chef-server-ctl",
+        "running_filepath" => "/etc/opscode/chef-server-running.json",
+        "database_name" => "opscode_chef",
+      },
     }.freeze
 
     class << self
@@ -45,13 +50,18 @@ module ChefBackup
     # @param config [Hash] a Hash of the private-chef-running.json
     #
     def initialize(config = {})
-      config['private_chef'] ||= {}
-      config['private_chef']['backup'] ||= {}
-      config['private_chef']['backup'] =
-        DEFAULT_CONFIG['backup'].merge(config['private_chef']['backup'])
+      config["config_base"] ||= DEFAULT_BASE
+      base = config["config_base"]
+      config[base] ||= {}
+      config[base]["backup"] ||= {}
+      config[base]["backup"] = DEFAULT_CONFIG["backup"].merge(config[base]["backup"])
       @config = config
     end
 
+    def to_hash
+      @config
+    end
+
     def_delegators :@config, :[], :[]=
   end
 end

data/lib/chef_backup/data_map.rb

@@ -1,4 +1,4 @@
-require 'time'
+require "time" unless defined?(Time.zone_offset)
 
 module ChefBackup
   # DataMap class to store data about the data we're backing up
@@ -11,27 +11,32 @@ module ChefBackup
       attr_writer :data_map
     end
 
-    attr_accessor :strategy, :backup_time, :topology, :configs, :services, :ha
+    attr_accessor :strategy, :backup_time, :topology, :configs, :services, :ha, :versions
 
     def initialize
       @services = {}
       @configs = {}
+      @versions = {}
       @ha = {}
       yield self if block_given?
 
       @backup_time ||= Time.now.iso8601
-      @strategy ||= 'none'
-      @toplogy ||= 'idontknow'
+      @strategy ||= "none"
+      @toplogy ||= "idontknow"
     end
 
     def add_service(service, data_dir)
       @services[service] ||= {}
-      @services[service]['data_dir'] = data_dir
+      @services[service]["data_dir"] = data_dir
     end
 
     def add_config(config, path)
       @configs[config] ||= {}
-      @configs[config]['data_dir'] = path
+      @configs[config]["data_dir"] = path
+    end
+
+    def add_version(project_name, data)
+      @versions[project_name] = data
     end
 
     def add_ha_info(k, v)
@@ -40,12 +45,13 @@ module ChefBackup
 
     def manifest
       {
-        'strategy' => strategy,
-        'backup_time' => backup_time,
-        'topology' => topology,
-        'ha' => ha,
-        'services' => services,
-        'configs' => configs
+        "strategy" => strategy,
+        "backup_time" => backup_time,
+        "topology" => topology,
+        "ha" => ha,
+        "services" => services,
+        "configs" => configs,
+        "versions" => versions,
       }
     end
   end

data/lib/chef_backup/deep_merge.rb

@@ -0,0 +1,145 @@
+#
+# Author:: Adam Jacob (<adam@chef.io>)
+# Author:: Steve Midgley (http://www.misuse.org/science)
+# Copyright:: Copyright 2009-2016, Chef Software Inc.
+# Copyright:: Copyright 2008-2016, Steve Midgley
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'chef_backup/mash'
+
+#
+# DANGER! THIS FILE WAS VENDORDED FROM CHEF. IF YOU ARE
+# MAKING A CHANGE HERE, CONSIDER WHETHER IT IS REQUIRED IN CHEF ALSO?
+#
+module ChefBackup
+  module Mixin
+    # == Chef::Mixin::DeepMerge
+    # Implements a deep merging algorithm for nested data structures.
+    # ==== Notice:
+    #   This code was originally imported from deep_merge by Steve Midgley.
+    #   deep_merge is available under the MIT license from
+    #   http://trac.misuse.org/science/wiki/DeepMerge
+    module DeepMerge
+
+      extend self
+
+      def merge(first, second)
+        first  = Mash.new(first)  unless first.kind_of?(Mash)
+        second = Mash.new(second) unless second.kind_of?(Mash)
+
+        DeepMerge.deep_merge(second, first)
+      end
+
+      class InvalidParameter < StandardError; end
+
+      # Deep Merge core documentation.
+      # deep_merge! method permits merging of arbitrary child elements. The two top level
+      # elements must be hashes. These hashes can contain unlimited (to stack limit) levels
+      # of child elements. These child elements to not have to be of the same types.
+      # Where child elements are of the same type, deep_merge will attempt to merge them together.
+      # Where child elements are not of the same type, deep_merge will skip or optionally overwrite
+      # the destination element with the contents of the source element at that level.
+      # So if you have two hashes like this:
+      #   source = {:x => [1,2,3], :y => 2}
+      #   dest =   {:x => [4,5,'6'], :y => [7,8,9]}
+      #   dest.deep_merge!(source)
+      #   Results: {:x => [1,2,3,4,5,'6'], :y => 2}
+      # By default, "deep_merge!" will overwrite any unmergeables and merge everything else.
+      # To avoid this, use "deep_merge" (no bang/exclamation mark)
+      def deep_merge!(source, dest)
+        # if dest doesn't exist, then simply copy source to it
+        if dest.nil?
+          dest = source; return dest
+        end
+
+        case source
+        when nil
+          dest
+        when Hash
+          if dest.kind_of?(Hash)
+            source.each do |src_key, src_value|
+              if dest[src_key]
+                dest[src_key] = deep_merge!(src_value, dest[src_key])
+              else # dest[src_key] doesn't exist so we take whatever source has
+                dest[src_key] = src_value
+              end
+            end
+          else # dest isn't a hash, so we overwrite it completely
+            dest = source
+          end
+        when Array
+          if dest.kind_of?(Array)
+            dest = dest | source
+          else
+            dest = source
+          end
+        when String
+          dest = source
+        else # src_hash is not an array or hash, so we'll have to overwrite dest
+          dest = source
+        end
+        dest
+      end # deep_merge!
+
+      def hash_only_merge(merge_onto, merge_with)
+        hash_only_merge!(safe_dup(merge_onto), safe_dup(merge_with))
+      end
+
+      def safe_dup(thing)
+        thing.dup
+      rescue TypeError
+        thing
+      end
+
+      # Deep merge without Array merge.
+      # `merge_onto` is the object that will "lose" in case of conflict.
+      # `merge_with` is the object whose values will replace `merge_onto`s
+      # values when there is a conflict.
+      def hash_only_merge!(merge_onto, merge_with)
+        # If there are two Hashes, recursively merge.
+        if merge_onto.kind_of?(Hash) && merge_with.kind_of?(Hash)
+          merge_with.each do |key, merge_with_value|
+            value =
+              if merge_onto.has_key?(key)
+                hash_only_merge(merge_onto[key], merge_with_value)
+              else
+                merge_with_value
+              end
+
+            if merge_onto.respond_to?(:public_method_that_only_deep_merge_should_use)
+              # we can't call ImmutableMash#[]= because its immutable, but we need to mutate it to build it in-place
+              merge_onto.public_method_that_only_deep_merge_should_use(key, value)
+            else
+              merge_onto[key] = value
+            end
+          end
+          merge_onto
+
+        # If merge_with is nil, don't replace merge_onto
+        elsif merge_with.nil?
+          merge_onto
+
+        # In all other cases, replace merge_onto with merge_with
+        else
+          merge_with
+        end
+      end
+
+      def deep_merge(source, dest)
+        deep_merge!(safe_dup(source), safe_dup(dest))
+      end
+
+    end
+  end
+end

data/lib/chef_backup/helpers.rb

@@ -1,35 +1,89 @@
 require 'fileutils'
+require 'json'
 require 'mixlib/shellout'
 require 'chef_backup/config'
 require 'chef_backup/logger'
 
+# rubocop:disable ModuleLength
 # rubocop:disable IndentationWidth
 module ChefBackup
 # Common helper methods that are usefull in many classes
 module Helpers
   # rubocop:enable IndentationWidth
 
-  SERVER_ADD_ONS = %w(
-    opscode-manage
-    opscode-reporting
-    opscode-push-jobs-server
-    opscode-analytics
-    chef-ha
-    chef-sync
-  ).freeze
+  SERVER_ADD_ONS = {
+    'opscode-manage' => {
+      'config_file' => '/etc/opscode-manage/manage.rb',
+      'ctl_command' => 'opscode-manage-ctl'
+    },
+    'opscode-reporting' => {
+      'config_file' => '/etc/opscode-reporting/opscode-reporting.rb',
+      'ctl_command' => 'opscode-reporting-ctl'
+    },
+    'opscode-push-jobs-server' => {
+      'config_file' => '/etc/opscode-push-jobs-server/opscode-push-jobs-server.rb',
+      'ctl_command' => 'opscode-push-jobs-server-ctl'
+    },
+    'opscode-analytics' => {
+      'config_file' => '/etc/opscode-analytics/opscode-analytics.rb',
+      'ctl_command' => 'opscode-analytics-ctl'
+    },
+    'chef-ha' => {
+      'config_file' => '/etc/opscode/chef-server.rb'
+    },
+    'chef-sync' => {
+      'config_file' => '/etc/chef-sync/chef-sync.rb',
+      'ctl_command' => 'chef-sync-ctl'
+    },
+    'chef-marketplace' => {
+      'config_file' => '/etc/chef-marketplace/marketplace.rb',
+      'ctl_command' => 'chef-marketplace-ctl'
+    }
+  }.freeze
 
-  def private_chef
-    config['private_chef']
-  end
+  DEFAULT_PG_OPTIONS = '-c statement_timeout=3600000'.freeze
 
   def config
     ChefBackup::Config
   end
 
+  def config_base
+    ChefBackup::Config['config_base']
+  end
+
+  def service_config
+    ChefBackup::Config[config_base]
+  end
+
+  def ctl_command
+    service_config['backup']['ctl-command']
+  end
+
+  def running_filepath
+    service_config['backup']['running_filepath']
+  end
+
+  def database_name
+    service_config['backup']['database_name']
+  end
+
   def log(message, level = :info)
     ChefBackup::Logger.logger.log(message, level)
   end
 
+  # Note that when we are in the backup codepath, we have access to a running
+  # chef server and hence, the ctl command puts all our flags under the current
+  # running service namespace. The lets the default configuration of the server
+  # provide flags that the user doesn't necessarily provide on the command line.
+  #
+  # During the restore codepath, there may be no running chef server. This means
+  # that we need to be paranoid about the existence of the service_config hash.
+  def shell_timeout
+    option = config['shell_out_timeout'] ||
+             (service_config && service_config['backup']['shell_out_timeout'])
+    option.to_f unless option.nil?
+  end
+
   #
   # @param file [String] A path to a file on disk
   # @param exception [Exception] An exception to raise if file is not present
@@ -38,11 +92,13 @@ module Helpers
   # @return [TrueClass, FalseClass]
   #
   def ensure_file!(file, exception, message)
-    File.exist?(file) ? true : fail(exception, message)
+    File.exist?(file) ? true : raise(exception, message)
   end
 
   def shell_out(*command)
-    cmd = Mixlib::ShellOut.new(*command)
+    options = command.last.is_a?(Hash) ? command.pop : {}
+    opts_with_defaults = { 'timeout' => shell_timeout }.merge(options)
+    cmd = Mixlib::ShellOut.new(*command, opts_with_defaults)
     cmd.live_stream ||= $stdout.tty? ? $stdout : nil
     cmd.run_command
     cmd
@@ -54,8 +110,39 @@ module Helpers
     cmd
   end
 
+  def project_name
+    service_config['backup']['project_name']
+  end
+
+  def base_install_dir
+    "/opt/#{project_name}"
+  end
+
+  def addon_install_dir(name)
+    # can use extra field in SERVER_ADD_ONS to extend if someone isn't following this pattern.
+    "/opt/#{name}"
+  end
+
+  def base_config_dir
+    "/etc/#{project_name}"
+  end
+
+  def chpst
+    "#{base_install_dir}/embedded/bin/chpst"
+  end
+
+  def pgsql
+    "#{base_install_dir}/embedded/bin/psql"
+  end
+
+  def pg_options
+    config['pg_options'] ||
+      (service_config && service_config['backup']['pg_options']) ||
+      DEFAULT_PG_OPTIONS
+  end
+
   def all_services
-    Dir['/opt/opscode/sv/*'].map { |f| File.basename(f) }.sort
+    Dir["#{base_install_dir}/sv/*"].map { |f| File.basename(f) }.sort
   end
 
   def enabled_services
@@ -63,20 +150,20 @@ module Helpers
   end
 
   def disabled_services
-    all_services.select { |sv| !service_enabled?(sv) }
+    all_services.reject { |sv| service_enabled?(sv) }
   end
 
   def service_enabled?(service)
-    File.symlink?("/opt/opscode/service/#{service}")
+    File.symlink?("#{base_install_dir}/service/#{service}")
   end
 
   def stop_service(service)
-    res = shell_out("chef-server-ctl stop #{service}")
+    res = shell_out("#{ctl_command} stop #{service}")
     res
   end
 
   def start_service(service)
-    res = shell_out("chef-server-ctl start #{service}")
+    res = shell_out("#{ctl_command} start #{service}")
     res
   end
 
@@ -92,32 +179,53 @@ module Helpers
     enabled_services.each { |sv| start_service(sv) }
   end
 
-  def enabled_addons
-    SERVER_ADD_ONS.select { |service| addon?(service) }
+  def restart_chef_server
+    shell_out("#{ctl_command} restart #{service}")
+  end
+
+  def reconfigure_add_ons
+    enabled_addons.each do |_name, config|
+      shell_out("#{config['ctl_command']} reconfigure") if config.key?('ctl_command')
+    end
+  end
+
+  def restart_add_ons
+    enabled_addons.each do |_name, config|
+      shell_out("#{config['ctl_command']} restart") if config.key?('ctl_command')
+    end
+  end
+
+  def reconfigure_marketplace
+    log 'Setting up Chef Marketplace'
+    shell_out('chef-marketplace-ctl reconfigure')
   end
 
-  def addon?(service)
-    File.directory?("/etc/#{service}")
+  def enabled_addons
+    SERVER_ADD_ONS.select do |name, config|
+      !config['config_file'].nil? &&
+        File.directory?(File.dirname(config['config_file'])) &&
+        File.directory?(addon_install_dir(name))
+    end
   end
 
   def strategy
-    private_chef['backup']['strategy']
+    service_config['backup']['strategy']
   end
 
   def topology
-    private_chef['topology']
+    service_config['topology']
   end
 
   def frontend?
-    private_chef['role'] == 'frontend'
+    service_config['role'] == 'frontend'
   end
 
   def backend?
-    private_chef['role'] =~ /backend|standalone/
+    service_config['role'] =~ /backend|standalone/
   end
 
   def online?
-    private_chef['backup']['mode'] == 'online'
+    service_config['backup']['mode'] == 'online'
   end
 
   def ha?
@@ -132,10 +240,14 @@ module Helpers
     topology == 'standalone'
   end
 
+  def marketplace?
+    shell_out('which chef-marketplace-ctl').exitstatus == 0
+  end
+
   def tmp_dir
     @tmp_dir ||= begin
       dir = safe_key { config['tmp_dir'] } ||
-            safe_key { private_chef['backup']['tmp_dir'] }
+            safe_key { service_config['backup']['tmp_dir'] }
       if dir
         FileUtils.mkdir_p(dir) unless File.directory?(dir)
         dir
@@ -152,6 +264,20 @@ module Helpers
     true
   end
 
+  def version_from_manifest_file(file)
+    return :no_version if file.nil?
+
+    path = File.expand_path(file)
+    if File.exist?(path)
+      config = JSON.parse(File.read(path))
+      { 'version' => config['build_version'],
+        'revision' => config['build_git_revision'],
+        'path' => path }
+    else
+      :no_version
+    end
+  end
+
   private
 
   def safe_key

data/lib/chef_backup/logger.rb

@@ -1,5 +1,3 @@
-require 'highline'
-
 module ChefBackup
   # Basic Logging Class
   class Logger
@@ -15,18 +13,25 @@ module ChefBackup
     attr_accessor :stdout
 
     def initialize(logfile = nil)
-      $stdout = logfile ? File.open(logfile, 'ab') : $stdout
-      @highline = HighLine.new($stdin, $stdout)
+      $stdout = logfile ? File.open(logfile, "ab") : $stdout
+    end
+
+    # pastel.decorate is a lightweight replacement for highline.color
+    def pastel
+      @pastel ||= begin
+        require "pastel" unless defined?(Pastel)
+        Pastel.new
+      end
     end
 
     def log(msg, level = :info)
       case level
       when :warn
         msg = "WARNING: #{msg}"
-        $stdout.puts(color? ? @highline.color(msg, :yellow) : msg)
+        $stdout.puts(color? ? pastel.decorate(msg, :yellow) : msg)
       when :error
         msg = "ERROR: #{msg}"
-        $stdout.puts(color? ? @highline.color(msg, :red) : msg)
+        $stdout.puts(color? ? pastel.decorate(msg, :red) : msg)
       else
         $stdout.puts(msg)
       end