dopi 0.17.0

data/spec/fixtures/puppet/manifests/site.pp

@@ -0,0 +1,52 @@
+# This site.pp is the entry point for puppet in the testenv
+# It contains some workarounds to help with problems in the
+# used modules
+
+$role = hiera('role')
+
+class base {
+  hiera_include('classes')
+
+  file{'/usr/libexec/mcollective/':            ensure => directory}
+  file{'/usr/libexec/mcollective/mcollective': ensure => directory}
+}
+
+node default {
+  include base
+}
+
+node 'puppetmaster.example.com' {
+  include base
+
+   file{'/etc/puppet/autosign.conf':
+    content => "*.example.com\n",
+    mode    => '0664',
+  }
+
+  #file{'/etc/puppet/environments/production/manifests/': ensure => directory }
+  file{'/etc/puppet/environments/production/manifests/site.pp':
+    ensure => symlink,
+    target => '/etc/puppet/manifests/site.pp',
+  }
+
+}
+
+# This are some fixes because there is not realy a working
+# mcollective module for centos at the moment
+node 'broker.example.com' {
+  include base
+
+  # FIX for activemq module
+  file{'/etc/activemq/':                    ensure => directory }
+  file{'/etc/activemq/instances-available': ensure => directory }
+  file{'/etc/activemq/instances-enabled':   ensure => directory }
+
+  # This is probably needed because we use an debian module or something
+  file{'/etc/activemq/activemq.xml':
+    source  => 'file:///etc/activemq/instances-available/mcollective/activemq.xml',
+    require => File['/etc/activemq/instances-available/mcollective/activemq.xml'],
+    notify  => Service['activemq'],
+  }
+
+  Class['java'] -> Class['activemq']
+}

data/spec/fixtures/test_configuration.yaml

@@ -0,0 +1,54 @@
+configuration:
+  nodes:
+    'puppetmaster.example.com':
+      role: 'puppetmaster'
+    'broker.example.com':
+      role: 'mcollective_broker'
+    'windows01.example.com':
+      os: 'windows'
+  roles:
+    'puppetmaster':
+      classes:
+        - 'foreman'
+        - 'puppet'
+      foreman::foreman_url: "%{hiera('::fqdn')}"
+      foreman::unattended: 'false'
+      foreman::authentication: 'false'
+      puppet::server: true
+      puppet::autosign: '/usr/local/bin/dop-puppet-autosign'
+      puppet::autosign_mode: 'script'
+      puppet::server_foreman: true
+      puppet::server_rttendedeports: 'store'
+      puppet::server_external_nodes: ''
+    'mcollective_broker':
+      classes:
+        - 'java'
+        - 'activemq'
+      java::distribution: 'jdk'
+      java::version: 'latest'
+      activemq::instances:
+        'mcollective':
+          stomp_nio_port: "%{hiera('mcollective::broker_port')}"
+          user_name: "%{hiera('mcollective::broker_user')}"
+          user_password: "%{hiera('mcollective::broker_password')}"
+  defaults:
+    classes:
+      - 'firewall'
+      - 'puppet'
+      - 'mcollective'
+    role: 'base'
+    os: 'linux'
+    firewall::ensure: 'stopped'
+    puppet::puppetmaster: 'puppetmaster.example.com'
+    puppet::runmode: 'none'
+    mcollective::use_node: true
+    mcollective::connector: 'activemq'
+    mcollective::broker_host: 'broker.example.com'
+    mcollective::broker_port: 61614
+    mcollective::broker_ssl: 'false'
+    mcollective::broker_user: 'mcollective'
+    mcollective::broker_password: 'vagrant'
+    mcollective::security_provider: 'psk'
+    mcollective::security_secret: 'vagrant'
+
+

data/spec/fixtures/test_credentials.yaml

@@ -0,0 +1,11 @@
+credentials:
+  'linux_login':
+    type: 'username_password'
+    username: 'root'
+    password: 'puppet'
+  'windows_login':
+    type: 'username_password'
+    username: 'administrator'
+    password: 'vagrant'
+
+

data/spec/fixtures/test_deloyed_file.txt

@@ -0,0 +1,5 @@
+This is some multiline file
+Deployed from a file
+Which contains the sentence
+hello world
+and some other stuff

data/spec/fixtures/test_infrastructure.yaml

@@ -0,0 +1,12 @@
+infrastructures:
+  'vagrant':
+    type: 'baremetal'
+    networks:
+      'vboxnet0':
+        ip_pool:
+          from: 192.168.56.101
+          to: 192.168.56.254
+        ip_netmask: 255.255.255.0
+        ip_defgw: 192.168.56.1
+
+

data/spec/fixtures/test_nodes.yaml

@@ -0,0 +1,45 @@
+nodes:
+  'puppetmaster.example.com':
+    infrastructure: 'vagrant'
+    image: 'puppetlabs/centos-7.0-64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.101'
+  'broker.example.com':
+    infrastructure: 'vagrant'
+    image: 'puppetlabs/centos-7.0-64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.102'
+  'linux01.example.com':
+    infrastructure: 'vagrant'
+    image: 'puppetlabs/centos-7.0-64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.103'
+  'linux02.example.com':
+    infrastructure: 'vagrant'
+    image: 'puppetlabs/centos-7.0-64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.104'
+  'linux03.example.com':
+    infrastructure: 'vagrant'
+    image: 'puppetlabs/centos-7.0-64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.105'
+  'windows01.example.com':
+    infrastructure: 'vagrant'
+    image: 'opentable/win-2012r2-standard-amd64-nocm'
+    interfaces:
+      'eth0':
+        network: 'vboxnet0'
+        ip: '192.168.56.106'
+
+

data/spec/fixtures/testenv_plan.yaml

@@ -0,0 +1,159 @@
+# This dop plan will setup the test environment for DOPi
+#
+# For now we will just use a bermetal type and vagrant
+# this should one day be replaced with dopv and some
+# local endpoint to create the machines.
+#
+
+name: 'build_dop_test_environment'
+max_in_flight: -1
+
+include: 'test_infrastructure.yaml'
+include: 'test_nodes.yaml'
+include: 'test_credentials.yaml'
+include: 'test_configuration.yaml'
+
+steps:
+  - name: 'Configure ssh credentials and wait until all the linux machines are online'
+    nodes_by_config: {os: 'linux'}
+    set_plugin_defaults: [{plugins: '/^ssh/', credentials: 'linux_login'}]
+    command: 'ssh/wait_for_login'
+
+  - name: 'Configure winrm credentials and wait until all the windows machines are online'
+    nodes_by_config: {os: 'windows'}
+    set_plugin_defaults: [{plugins: '/^winrm/', credentials: 'windows_login'}]
+    command: 'winrm/wait_for_login'
+
+  - name: 'install the puppetlabs repository'
+    nodes_by_config: {os: 'linux'}
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/etc/yum.repos.d/puppetlabs.repo'
+      plugin: 'ssh/custom'
+      exec: 'rpm'
+      arguments: '-ivh https://yum.puppetlabs.com/puppetlabs-release-el-7.noarch.rpm'
+
+  - name: 'install puppet on all the nodes'
+    nodes_by_config: {os: 'linux'}
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/usr/bin/puppet'
+      plugin: 'ssh/custom'
+      exec: 'yum'
+      arguments: 'install -y puppet'
+
+  - name: 'copy the puppet configuration to the right place'
+    roles: 'puppetmaster'
+    command:
+      plugin: 'ssh/custom'
+      exec: 'cp'
+      arguments: '-r /vagrant/spec/fixtures/puppet/* /etc/puppet/'
+
+  - name: 'Install all the needed packages on the puppetmaster'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/usr/bin/rake'
+        - plugin: 'ssh/file_exists'
+          file: '/usr/bin/bundle'
+        - plugin: 'ssh/file_exists'
+          file: '/usr/bin/git'
+        - plugin: 'ssh/file_exists'
+          file: '/usr/include/ruby.h'
+      plugin: 'ssh/custom'
+      exec: 'yum'
+      arguments: 'install -y rubygem-rake rubygem-bundler git ruby-devel'
+
+  - name: 'Install librarian-puppet'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/usr/local/bin/librarian-puppet'
+      plugin: 'ssh/custom'
+      exec: 'gem'
+      arguments: 'install librarian-puppet'
+
+  - name: 'Install all the puppet modules'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/etc/puppet/.librarian/puppet/config'
+      plugin: 'ssh/custom'
+      exec: 'librarian-puppet'
+      arguments: 'install'
+      env: {LIBRARIAN_PUPPET_PUPPETFILE: '/etc/puppet/Puppetfile'}
+
+  - name: 'Install dop_common for the puppetmaster'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/custom'
+          exec: 'gem list | grep dop_common'
+      plugin: 'ssh/custom'
+      exec: 'cd /vagrant/vendor/cache/dop_common-*/ && gem build dop_common.gemspec && gem install dop_common*.gem'
+
+  - name: 'Install DOPi so we can add the plan'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/vagrant/.bundle/config'
+      plugin: 'ssh/custom'
+      exec: 'bundle'
+      arguments: 'install --local'
+      env: {BUNDLE_GEMFILE: '/vagrant/Gemfile'}
+
+  - name: 'Add deployment plan with hiera data to the plan cache'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/var/lib/dop/plans/build_dop_test_environment.yaml'
+      plugin: 'ssh/custom'
+      exec: 'bundle'
+      arguments: 'exec dopi add /vagrant/spec/fixtures/testenv_plan.yaml'
+      env: {BUNDLE_GEMFILE: '/vagrant/Gemfile'}
+
+  - name: 'Run puppet on puppetmaster'
+    roles: 'puppetmaster'
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_exists'
+          file: '/etc/httpd/conf.d/25-puppet.conf'
+      plugin: 'ssh/custom'
+      exec: 'puppet'
+      arguments: 'apply /etc/puppet/manifests/site.pp'
+
+  - name: 'Add puppetmaster to all hosts files'
+    nodes_by_config: {os: 'linux'}
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_contains'
+          file: '/etc/hosts'
+          pattern: 'puppetmaster.example.com'
+      plugin: 'ssh/custom'
+      exec: "echo"
+      arguments: '"192.168.56.101 puppetmaster.example.com puppetmaster" >> /etc/hosts'
+
+  - name: 'Add broker to all hosts files'
+    nodes_by_config: {os: 'linux'}
+    command:
+      verify_commands:
+        - plugin: 'ssh/file_contains'
+          file: '/etc/hosts'
+          pattern: 'broker.example.com'
+      plugin: 'ssh/custom'
+      exec: "echo"
+      arguments: '"192.168.56.102 broker.example.com broker" >> /etc/hosts'
+
+  - name: 'Run puppet on the rest of the linux nodes'
+    nodes_by_config: {os: 'linux'}
+    exclude_roles: 'puppetmaster'
+    command:
+      plugin: 'ssh/puppet_agent_run'
+      arguments: '--server puppetmaster.example.com'

data/spec/integration/dopi/addrun_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+# Try running the plan when it is added (persisted to disk) and loaded again
+# before running.
+
+DopCommon.config.mco_config = 'spec/fixtures/mco_client.cfg'
+DopCommon.config.hiera_yaml = 'spec/fixtures/puppet/hiera.yaml'
+
+plan_file = "spec/integration/dopi/plans/dummy.yaml"
+plan_name = nil
+plan = nil
+
+describe 'running an added plan' do
+  context 'in temp directory' do
+    Dir.mktmpdir do |tmp|
+      DopCommon.config.plan_store_dir = tmp
+      it 'add plan to store' do
+        plan_name = Dopi.add(plan_file)
+      end
+      it 'load plan from store' do
+        plan = Dopi.show(plan_name)
+      end
+      it 'run loaded plan' do
+        Dopi.run(plan_name)
+      end
+      it 'remove plan from store' do
+        Dopi.remove(plan_name)
+      end
+    end
+  end
+end

data/spec/integration/dopi/cli/command_run_spec.rb

@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+describe 'command_run' do
+  fixture_file 'puppet'
+  fixture_file 'plans'
+
+  describe 'run exit code' do
+    before :each do
+      Dopi.add("spec/fixtures/plans/#{plan_name}.yaml")
+    end
+    after :each do
+      Dopi.remove(plan_name, true)
+    end
+
+    context 'plan run will be successful' do
+      let(:plan_name) { 'hello_world' }
+      command 'dopi --verbosity INFO run hello_world'
+      its(:exitstatus) { is_expected.to eq 0 }
+    end
+    context 'plan will fail on command timeout' do
+      let(:plan_name) { 'fail_on_timeout' }
+      command 'dopi --verbosity INFO run fail_on_timeout', :allow_error => true
+      its(:exitstatus) { is_expected.to_not eq 0 }
+    end
+  end
+
+  describe 'oneshot exit code' do
+    context 'plan run will be successful' do
+      command "dopi --verbosity INFO oneshot hello_world.yaml"
+      its(:exitstatus) { is_expected.to eq 0 }
+    end
+    context 'plan will fail on command timeout' do
+      command "dopi --verbosity INFO oneshot fail_on_timeout.yaml", :allow_error => true
+      its(:exitstatus) { is_expected.to_not eq 0 }
+    end
+  end
+
+end

data/spec/integration/dopi/cli/global_options_spec.rb

@@ -0,0 +1,128 @@
+require 'spec_helper'
+
+describe 'global cli options' do
+  fixture_file 'puppet'
+  fixture_file 'plans'
+
+  describe 'connection_check_timeout' do
+    pending
+  end
+
+  describe 'facts_dir' do
+    pending # This option should be removed
+  end
+
+  describe 'hiera_yaml' do
+    context 'Invalid hiera.yaml' do
+      command 'dopi --hiera_yaml ./nothiera.yaml oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.to include 'hiera.yaml not found! Using empty config'}
+    end
+    context 'Valid hiera.yaml' do
+      command 'dopi --hiera_yaml ./hiera.yaml oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.not_to include 'hiera.yaml not found! Using empty config'}
+    end
+  end
+
+  describe 'load_facts' do
+    pending # This option should be removed
+  end
+
+  describe '--log_dir' do
+    before :each do
+      command "dopi --log_dir ./mylog oneshot hello_world.yaml"
+    end
+    it 'should create the directory' do
+      expect(Dir.entries(temp_path)).to include 'mylog'
+    end
+    it 'should write to the log file in the directory' do
+      log_file = File.join(temp_path, 'mylog', 'dopi.log')
+      expect(File.read(log_file)).to include
+        "Step 'write hello world' successfully finished."
+    end
+  end
+
+  describe 'log_level' do
+    context 'INFO' do
+      it 'should not contain debug output in the log file' do
+        command "dopi --log_dir ./mylog --log_level INFO oneshot hello_world.yaml"
+        log_file = File.join(temp_path, 'mylog', 'dopi.log')
+        expect(File.read(log_file)).not_to include
+          'Executing echo "hello world"'
+      end
+    end
+    context 'DEBUG' do
+      it 'should contain debug output in the log file' do
+        command "dopi --log_dir ./mylog --log_level DEBUG oneshot hello_world.yaml"
+        log_file = File.join(temp_path, 'mylog', 'dopi.log')
+        expect(File.read(log_file)).to include
+          'Executing echo "hello world"'
+      end
+    end
+  end
+
+  describe 'mco_config' do
+    pending
+  end
+
+  describe 'mco_dopi_logger' do
+    pending
+  end
+
+  describe 'plan_store_dir' do
+    before :each do
+      command "dopi --plan_store_dir ./plan_store add hello_world.yaml"
+    end
+    it 'should create the directory' do
+      expect(Dir.entries(temp_path)).to include 'plan_store'
+    end
+    it 'should add the plan to the plan_store' do
+      stored_plan = File.join(temp_path, 'plan_store', 'hello_world')
+      expect(File.exists?(stored_plan)).to be true
+    end
+  end
+
+  describe 'role_default' do
+    context 'role_default is not set' do
+      command 'dopi oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.to include 'No role found for linux01.example.com'}
+    end
+    context 'role_default is set' do
+      command 'dopi --role_default testnode oneshot test_role_variable.yaml'
+      its(:stdout) {is_expected.to include "Step 'write hello world' successfully finished."}
+    end
+  end
+
+  describe 'role_variable' do
+    context 'role_variable is not set' do
+      command 'dopi --hiera_yaml hiera.yaml oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.to include 'No role found for linux01.example.com'}
+    end
+    context 'role_default is set' do
+      command 'dopi --hiera_yaml hiera.yaml --role_variable test_role oneshot test_role_variable.yaml'
+      its(:stdout) {is_expected.to include "Step 'write hello world' successfully finished."}
+    end
+  end
+
+  describe 'use_hiera' do
+    context 'default behaviour' do
+      command 'dopi --hiera_yaml ./nothiera.yaml oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.to include 'hiera.yaml not found! Using empty config'}
+    end
+    context 'Disable hiera' do
+      command 'dopi --no-use_hiera oneshot test_role_variable.yaml', :allow_error => true
+      its(:stdout) {is_expected.not_to include 'hiera.yaml not found! Using empty config'}
+    end
+  end
+
+  describe '--verbosity' do
+    context 'INFO' do
+      command "dopi --verbosity INFO oneshot hello_world.yaml"
+      its(:stdout) {is_expected.not_to include 'Executing echo "hello world"'}
+    end
+    context 'DEBUG' do
+      command "dopi --verbosity DEBUG oneshot hello_world.yaml"
+      its(:stdout) {is_expected.to include 'Executing echo "hello world"'}
+    end
+  end
+
+end

data/spec/integration/dopi/command_spec.rb

@@ -0,0 +1,66 @@
+require 'spec_helper'
+
+DopCommon.config.mco_config    = 'spec/fixtures/mco_client.cfg'
+DopCommon.config.hiera_yaml    = 'spec/fixtures/puppet/hiera.yaml'
+
+describe 'Basic integration test built from plan files' do
+
+  # Create plugin tests from plugin test yaml files
+  # and check if they run successfully
+  Dir['spec/integration/dopi/plans/**/*.yaml'].each do |plan_file|
+    describe plan_file do
+      plan_parser = DopCommon::Plan.new(YAML.load_file(plan_file))
+      plan = Dopi::Plan.new(plan_parser)
+      node_names = plan.nodes.map{|n| n.name}
+      plan.instance_variable_set(:@context_logger, DopCommon::ThreadContextLogger.new('/tmp/dopi_test', node_names))
+      it "is a valid plan file" do
+        expect(plan.valid?).to be true
+      end
+      plan.step_sets.each do |step_set|
+        step_set.steps.each do |step|
+          it "successfully runs the step: '#{step.name}'" do
+            step.run({:run_for_nodes => :all, :noop => false})
+            expect(step.state).to be :done
+          end
+        end
+      end
+    end
+  end
+
+  # Create plugin tests from plugin test yaml files
+  # and check if they fail
+  Dir['spec/integration/dopi/fail_check_plans/**/*.yaml'].each do |plan_file|
+    describe plan_file do
+      plan_parser = DopCommon::Plan.new(YAML.load_file(plan_file))
+      plan = Dopi::Plan.new(plan_parser)
+      node_names = plan.nodes.map{|n| n.name}
+      plan.instance_variable_set(:@context_logger, DopCommon::ThreadContextLogger.new('/tmp/dopi_test', node_names))
+      it "is a valid plan file" do
+        expect(plan.valid?).to be true
+      end
+      plan.step_sets.each do |step_set|
+        step_set.steps.each do |step|
+          it "successfully runs the step: '#{step.name}'" do
+            step.run({:run_for_nodes => :all, :noop => false})
+            expect(step.state).to be :failed
+          end
+        end
+      end
+    end
+  end
+
+  # Create plugin tests from plugin test yaml files
+  # and check if they are not valid
+  Dir['spec/integration/dopi/invalid_plans/**/*.yaml'].each do |plan_file|
+    describe plan_file do
+      plan_parser = DopCommon::Plan.new(YAML.load_file(plan_file))
+      plan = Dopi::Plan.new(plan_parser)
+      node_names = plan.nodes.map{|n| n.name}
+      plan.instance_variable_set(:@context_logger, DopCommon::ThreadContextLogger.new('/tmp/dopi_test', node_names))
+      it "is not a valid plan file" do
+        expect(plan.valid?).to be false
+      end
+    end
+  end
+
+end

data/spec/integration/dopi/fail_check_plans/file_exists_fails.yaml

@@ -0,0 +1,38 @@
+name: 'test_powershell_exitcode_handling'
+max_in_flight: 1
+
+infrastructures:
+  'test':
+    type: 'baremetal'
+
+nodes:
+  'windows01.example.com':
+    infrastructure: 'test'
+  'linux01.example.com':
+    infrastructure: 'test'
+
+credentials:
+  'windows_login':
+    type: 'username_password'
+    username: 'administrator'
+    password: 'vagrant'
+  'linux_login':
+    type: 'username_password'
+    username: 'root'
+    password: 'puppet'
+
+steps:
+
+  - name: 'Check if the plugin fails for the windows command'
+    nodes: 'windows01.example.com'
+    command:
+      plugin: 'winrm/file_exists'
+      credentials: 'windows_login'
+      file: 'C:\nonexistingfile.txt'
+
+  - name: 'Check if the plugin fails for the linux command'
+    nodes: 'linux01.example.com'
+    command:
+      plugin: 'ssh/file_exists'
+      credentials: 'linux_login'
+      file: '/nonexistingfile.txt'

data/spec/integration/dopi/fail_check_plans/output_parser.yaml

@@ -0,0 +1,39 @@
+name: 'fail_test_for_ouptut_parser'
+
+infrastructures:
+  'test':
+    type: 'baremetal'
+
+nodes:
+  'linux01.example.com':
+    infrastructure: 'test'
+
+steps:
+
+  #
+  # Check if the parser is working as intended
+  #
+
+  - name: 'Check if the output parser can catch an error'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      exec: 'echo'
+      arguments: '"Error: this is a simulated error"'
+      parse_output:
+        error:
+          - '^Error:'
+
+  - name: 'Check if the output parser can catch a warning'
+    nodes: 'all'
+    command:
+      plugin: 'custom'
+      exec: 'echo'
+      arguments: '"Warning: This is a simulated warning"'
+      fail_on_warning: true
+      parse_output:
+        warning:
+          - '^Warning:'
+
+
+