dtk-common 0.9.0 → 0.9.1

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODUwNmVmZjNkNDZiMzdhOGEyZGM0Y2QyN2FjNzdmNjkzYTBmMzg5ZA==
-  data.tar.gz: !binary |-
-    OWM5ZGEwOWQ5NzNlNWIzNmNkMGE4M2I0NDhjNjcxYTJiOGEyZjc2MA==
+SHA1:
+  metadata.gz: 1ac08df1794efde0192217d1c31ac6085b50b8e7
+  data.tar.gz: 59890da1e2a0e361e54309aebd24d926e349db5b
 SHA512:
-  metadata.gz: !binary |-
-    MzY0YTFmN2FmNDJkNTkxODc1NmNkNzgxOTc0ZGI2OTc0NTM1ZTgyNDkwMTNk
-    MmYxMjZmMTc4ZDQzNGViY2E0NTYxNTc4Yzg2MWM4NmQzNDAxODYyYmRhNGZi
-    NDg1YjFlMzIxZDM3YmU0ZmZkODU5OTk2MjgyYmFjMDdjNGYxOTE=
-  data.tar.gz: !binary |-
-    MzhlNTUwNGQ3ZjcyMDExNzhiOWM5NWQ5ZDIwZTVmZTgwOWI5OWU2OGExODZk
-    ZWExZWM3MmE0NTk5ZWEyY2QzNGY0NWI5Yzc4OTliY2M1ODA1NDYwNDIxYzZi
-    ZGFkMWM2OTM0YTE5ZDVlNDczNDA5NmU3OTM3ZDUwOGQ2ZmQ4OWM=
+  metadata.gz: 32d2291226dc4058ea6c296153010467409ea5d5f0de4a862f750461b2dd26a24e2fc9e29142e6a176964fb22b913626db52768cb45bb1d79ae30dc86b27adf1
+  data.tar.gz: cf093c3e581738e15d9722b1db0436fb35f99713b417262602ad4d755c0f7131bc370bdf62c051e1221057289d5f49ac64e1303d54512383ecf1d679447bd658

data/Gemfile.lock

@@ -1,12 +1,35 @@
 PATH
   remote: .
   specs:
-    dtk-common (0.1.10)
+    dtk-common (0.9.0)
+      colorize (~> 0.5.8)
+      docker-api (~> 1.26)
+      dtk-common-core (= 0.9.0)
+      erubis (~> 2.7)
+      grit (~> 2.5)
       rugged (~> 0.17.0.b7)
 
 GEM
   remote: http://rubygems.org/
   specs:
+    colorize (0.5.8)
+    diff-lcs (1.2.5)
+    docker-api (1.26.2)
+      excon (>= 0.38.0)
+      json
+    dtk-common-core (0.9.0)
+      rest-client (= 1.6.7)
+    erubis (2.7.0)
+    excon (0.45.4)
+    grit (2.5.0)
+      diff-lcs (~> 1.1)
+      mime-types (~> 1.15)
+      posix-spawn (~> 0.3.6)
+    json (1.8.3)
+    mime-types (1.25.1)
+    posix-spawn (0.3.8)
+    rest-client (1.6.7)
+      mime-types (>= 1.16)
     rugged (0.17.0.b7)
 
 PLATFORMS
@@ -14,3 +37,6 @@ PLATFORMS
 
 DEPENDENCIES
   dtk-common!
+
+BUNDLED WITH
+   1.10.6

data/dtk-common.gemspec

@@ -18,6 +18,9 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'rugged','~> 0.17.0.b7'
   gem.add_dependency 'dtk-common-core','0.9.0'
   gem.add_dependency 'colorize','~> 0.5.8'
+  gem.add_dependency 'docker-api', '~> 1.26'
+  gem.add_dependency 'grit', '~> 2.5'
+  gem.add_dependency 'erubis', '~> 2.7'
   # gem.add_dependency 'sequel','~> 3.40.0'
   # gem.add_dependency 'rdoc','~> 3.12'
 end

data/lib/docker/commander.rb

@@ -0,0 +1,98 @@
+require 'docker'
+require File.expand_path('../utils/open3.rb', File.dirname(__FILE__))
+
+module DTK
+  module Docker
+    class Commander
+
+      include Common::Open3
+
+      def initialize(docker_image, docker_command, puppet_manifest, execution_type, dockerfile, workdir = '/usr/share/dtk/docker-worker')
+        @docker_image = docker_image
+        @docker_command = docker_command
+        @dockerfile = dockerfile
+        @puppet_manifest = puppet_manifest
+        @execution_type = execution_type
+        @workdir = workdir
+
+        unless ::Docker::Image.exist?(docker_image) && !@dockerfile
+          Log.info "Getting docker image '#{docker_image}', this may take a while"
+          ::Docker::Image.create('fromImage' => docker_image)
+        end
+
+         # build required docker image if requested
+        if @dockerfile
+          Log.info "Building docker image: #{docker_image}"
+          image = ::Docker::Image.build(@dockerfile)
+          image.tag('repo' => docker_image, 'force' => true)
+        end
+      end
+
+      def run
+        # require 'debugger'; debugger
+        docker_container_name = "dtk#{Time.now.to_i}"
+        output_dir = "#{@workdir}/#{docker_container_name}"
+        output_dir_tmp = "#{output_dir}/tmp"
+        output_dir_container = "/host_volume"
+        output_file = "#{output_dir}/report.yml"
+        puppet_modules_dir = '/usr/share/dtk/puppet-modules'
+
+
+        FileUtils.mkdir_p output_dir_tmp
+        # make sure dtkyaml reporter is available to puppet
+        FileUtils.ln_sf '/usr/share/dtk/dtk-arbiter/dtkyaml', '/etc/puppet/modules/dtkyaml' unless File.exist? '/etc/puppet/modules/dtkyaml'
+        # write puppet manifest
+        File.open("#{output_dir}/manifest.pp", 'w') { |file| file.write(@puppet_manifest) }
+        # make sure r8 module is available
+        FileUtils.cp_r "/etc/puppet/modules/r8", puppet_modules_dir unless File.exist? "#{puppet_modules_dir}/r8"
+
+        docker_cli_cmd = "docker run --name #{docker_container_name} -v #{output_dir}:#{output_dir_container} -v #{output_dir}/tmp:/tmp" +
+                        ((@execution_type.eql? 'puppet') ? " -v #{puppet_modules_dir}:/etc/puppet/modules" : "") + " #{@docker_image} #{@docker_command}"
+
+        Log.info "Starting Docker container..."
+
+        docker_run_stdout, docker_run_stderr, exit_status, results = capture3_with_timeout(docker_cli_cmd)
+
+        if @execution_type.eql? 'puppet'
+          begin
+            docker_puppet_report = Hash.new
+            docker_puppet_report = YAML.load_file(output_file) if File.exist? output_file
+          rescue Exception => e
+            Log.error("Docker (puppet) report error: #{e.message}", e.backtrace)
+          end
+        end
+
+        container = ::Docker::Container.get(docker_container_name)
+
+        @results = Hash.new
+
+        @results[:puppet_report] = docker_puppet_report ||= ''
+        @results[:stdout] = docker_run_stdout
+        @results[:stderr] = docker_run_stderr
+        @results[:status] = exit_status.exitstatus
+        @results[:dynamic_attributes] = read_dynamic_attributes(output_dir_tmp)
+
+        # cleanup
+        Log.info("Deleting container and doing cleanup")
+        container.delete(:force => true)
+
+        # http://unix.stackexchange.com/a/117848
+        system("find #{output_dir} -depth -type f -exec shred -n 1 -z -u {} \\;")
+        FileUtils.rm_rf(output_dir)
+      end
+
+      def results
+        @results
+      end
+
+    private
+
+      def read_dynamic_attributes(path, identifier = 'dtk_exported_variables')
+        full_path = File.join(path, identifier)
+        return nil unless File.exists?(full_path)
+        Marshal.load(File.read(full_path))
+      end
+
+    end
+  end
+end

data/lib/dtk-common/version.rb

@@ -16,5 +16,5 @@
 # limitations under the License.
 #
 module DtkCommon
-  VERSION="0.9.0"
+  VERSION="0.9.1"
 end

data/lib/gitolite/repo.rb

@@ -54,10 +54,8 @@ module Gitolite
 
     end
 
-    def initialize(repo_name, configuration_, logger_, gitolite_path, gitolite_branch="master")
-      # IMPORTANT! Tenants user are always included with each module
-
-      @rights_hash = { 'R' => [], 'W' => [], 'RW' => ['@tenants'], 'RW+' => []}
+    def initialize(repo_name, configuration_, logger_, gitolite_path, gitolite_branch = 'master')
+      @rights_hash = { 'R' => [], 'W' => [], 'RW' => [], 'RW+' => []}
       @repo_name = repo_name
       @user_groups = []
       @commit_messages = []

data/lib/utils/open3.rb

@@ -0,0 +1,103 @@
+require 'timeout'
+require 'open3'
+
+module DTK
+  module Common
+    module Open3
+
+      STREAM_TIMEOUT  = 5
+
+      # Running puppet directly on system and from ruby process has proven different. Following is set of environment variables
+      # that assures proper execution from bundler / rvm ruby
+      AGNOSTIC_PUPPET_VARS = {
+         "BUNDLE_GEMFILE" => nil,
+         "BUNDLE_BIN_PATH" => nil,
+         "RUBYOPT" => nil,
+         "rvm_" => nil,
+         "RACK_ENV" => nil,
+         "RAILS_ENV" => nil
+       }
+
+      ##
+      # Open3 method extended with timeout, more info https://gist.github.com/pasela/9392115
+      #
+
+      def capture3_with_timeout(*cmd)
+        spawn_opts = Hash === cmd.last ? cmd.pop.dup : {}
+        opts = {
+          :stdin_data => "",
+          :timeout    => @timeout || 0,
+          :signal     => :TERM,
+          :kill_after => nil,
+        }
+
+        in_r,  in_w  = IO.pipe
+        out_r, out_w = IO.pipe
+        err_r, err_w = IO.pipe
+        in_w.sync = true
+
+        spawn_opts[:in]  = in_r
+        spawn_opts[:out] = out_w
+        spawn_opts[:err] = err_w
+
+        result = {
+          :pid     => nil,
+          :status  => nil,
+          :stdout  => nil,
+          :stderr  => nil,
+          :timeout => false,
+        }
+
+        out_reader = nil
+        err_reader = nil
+        wait_thr = nil
+
+        begin
+          Timeout.timeout(opts[:timeout]) do
+            result[:pid] = spawn(AGNOSTIC_PUPPET_VARS, *cmd, spawn_opts)
+            wait_thr = Process.detach(result[:pid])
+            in_r.close
+            out_w.close
+            err_w.close
+
+            out_reader = Thread.new { out_r.read }
+            err_reader = Thread.new { err_r.read }
+
+            in_w.close
+
+            result[:status] = wait_thr.value
+          end
+        rescue Timeout::Error
+          result[:timeout] = true
+          pid = result[:pid]
+          Process.kill(opts[:signal], pid)
+          if opts[:kill_after]
+            unless wait_thr.join(opts[:kill_after])
+              Process.kill(:KILL, pid)
+            end
+          end
+        ensure
+          result[:status] = wait_thr.value if wait_thr
+          begin
+            # there is a bug where there is infinite leg on out_reader (e.g. hohup) commands
+            Timeout.timeout(STREAM_TIMEOUT) do
+              result[:stdout] = out_reader.value if out_reader
+              result[:stderr] = err_reader.value if err_reader
+            end
+          rescue Timeout::Error
+            result[:stdout] ||= ''
+            result[:stderr] ||= ''
+          end
+          out_r.close unless out_r.closed?
+          err_r.close unless err_r.closed?
+        end
+
+        result[:stdout] = result[:stdout].gsub(/\e\[([;\d]+)?m/, '') if result[:stdout]
+        result[:stderr] = result[:stderr].gsub(/\e\[([;\d]+)?m/, '') if result[:stderr]
+
+        [result[:stdout], result[:stderr], result[:status], result]
+      end
+
+    end
+  end
+end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: dtk-common
 version: !ruby/object:Gem::Version
-  version: 0.9.0
+  version: 0.9.1
 platform: ruby
 authors:
 - Rich PELAVIN
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-10 00:00:00.000000000 Z
+date: 2016-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rugged
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0.b7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0.b7
 - !ruby/object:Gem::Dependency
@@ -42,16 +42,58 @@ dependencies:
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.8
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.5.8
+- !ruby/object:Gem::Dependency
+  name: docker-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.26'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.26'
+- !ruby/object:Gem::Dependency
+  name: grit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: erubis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
 description: Dtk common is needed to use dtk-client gem, provides common libraries
   for running DTK CLI.
 email:
@@ -60,8 +102,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .license_header
+- ".gitignore"
+- ".license_header"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -69,6 +111,7 @@ files:
 - Rakefile
 - dtk-common.gemspec
 - lib/common_patch.rb
+- lib/docker/commander.rb
 - lib/dsl.rb
 - lib/dsl/directory_parser.rb
 - lib/dsl/directory_parser/git.rb
@@ -102,6 +145,7 @@ files:
 - lib/module_parser/module_parser.rb
 - lib/module_version.rb
 - lib/require_first.rb
+- lib/utils/open3.rb
 - rich_tests/README.md
 - rich_tests/dsl_test1.rb
 - rich_tests/dsl_test2.rb
@@ -190,17 +234,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.4.1
 signing_key: 
 specification_version: 4
 summary: Common libraries used for DTK CLI client.