knife-github 0.0.3 → 0.0.6

data/.gitignore

@@ -1,2 +1,5 @@
 *.gem
 pkg/*
+.ruby-version
+.ruby-gemset
+.rvmrc

data/README.md

@@ -12,7 +12,6 @@ You can configure the following attributes within your knife.rb
     knife[:github_organizations]   = [ 'customer-cookbooks', 'central-cookbooks' ] 
     knife[:github_link]            = 'ssh' 
     knife[:github_api_version]     = 'v3'  
-    knife[:github_cache]           = 900  
     knife[:github_ssl_verify_mode] = 'verify_none'
 
 ###### github_url
@@ -30,11 +29,15 @@ Options are <tt>ssh</tt> <tt>git</tt> <tt>http</tt> <tt>https</tt> <tt>svn</tt>
 ###### github_api_version \<optional\>
 The current and default version of the api is <tt>v3</tt> but this will allow you to target older versions if needed.
 
-###### github_cache \<optional\>
-This will be the lifetime of the cache files in seconds, default <tt>900</tt>. Cache files will be created into the: ~/.chef directory.  
-We use cache files to offload the api calls and increase the performance for additional executions.
-
 ###### github_ssl_verify_mode \<optional\>
 The plugin is using the underlying knife http implementation, hence it will have the same options to handle ssl.  
-Currently the options are: <tt>verify_peer</tt> <tt>verify_none</tt> 
+Currently the options are: <tt>verify_peer</tt> <tt>verify_none</tt>   
+
+Other
+=====
+
+Cache files will be created into the: ~/.chef directory.
+We use cache files to offload the api calls and increase the performance for additional executions.  
+Updated to any repo inside the organization will cause the cache files to update.  
+But in case of any problems, the cache files can be safely deleted.
 

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/knife-github.gemspec

@@ -18,5 +18,6 @@ Gem::Specification.new do |s|
   s.executables       = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths     = ["lib"]
 
+  s.add_dependency "mixlib-versioning", ">= 1.0.0"
   # s.add_dependency "chef", "~> 11.0.0"
 end

data/lib/chef/knife/github_base.rb

@@ -17,6 +17,7 @@
 #
 
 # require 'chef/knife'
+require "knife-github/version"
 
 class Chef
   class Knife
@@ -27,6 +28,7 @@ class Chef
 
           deps do
             require 'chef/mixin/shell_out'
+            require 'mixlib/versioning'
           end
 
           option :github_url,
@@ -51,10 +53,15 @@ class Chef
                  :description => "SSL verify mode: verify_peer, verify_none (default: verify_peer)",
                  :boolean => true
 
-          option :github_cache,
-                 :long => "--github_cache MIN",
-                 :description => "Max life-time for local cache files in minutes (default: 900)"
+          option :github_tmp,
+                 :long => "--github_tmp PATH",
+                 :description => "A path where temporary files for the diff function will be made default: /tmp/gitdiff)"
 
+          option :github_no_update,
+                 :long => "--github_no_update",
+                 :description => "Turn github update checking off",
+                 :boolean => true
+ 
           def validate_base_options
             unless locate_config_value('github_url')
               ui.error "Github URL not specified"
@@ -64,22 +71,47 @@ class Chef
               ui.error "Github organization(s) not specified"
               exit 1
             end
+            unless locate_config_value('github_no_update')
+              check_gem_version
+            end
 
             @github_url             = locate_config_value("github_url")
             @github_organizations   = locate_config_value("github_organizations")
-            @github_cache           = (locate_config_value("github_cache") || 900).to_i
             @github_link            = locate_config_value("github_link") || 'ssh'
             @github_api_version     = locate_config_value("github_api_version") || 'v3'
             @github_ssl_verify_mode = locate_config_value("github_ssl_verify_mode") || 'verify_peer'
+            @github_tmp             = locate_config_value("github_tmp") || '/var/tmp/gitdiff'
+            @github_tmp             = "#{@github_tmp}#{Process.pid}"
+          end
+
+          def check_gem_version
+            url  = 'http://rubygems.org/api/v1/gems/knife-github.json'
+            result = `curl -L -s #{url}`
+            begin
+              json = JSON.parse(result)
+              webversion = Mixlib::Versioning.parse(json['version'])
+              thisversion = Mixlib::Versioning.parse(::Knife::Github::VERSION)
+              if webversion > thisversion
+                ui.info "INFO: New version (#{webversion.to_s}) of knife-github is available!"
+                ui.info "INFO: Turn off this message with --github_no_update or add knife[:github_no_update] = true to your configuration"
+              end 
+              Chef::Log.debug("local_gem_version    : " + thisversion.to_s)
+              Chef::Log.debug("repo_gem_version     : " + webversion.to_s)
+              Chef::Log.debug("repo_downloads       : " + json['version_downloads'].to_s)
+              Chef::Log.debug("repo_total_downloads : " + json['downloads'].to_s)
+ 
+            rescue
+              ui.info "INFO: Cannot verify gem version information from rubygems.org"
+              ui.info "INFO: Turn off this message with --github_no_update or add knife[:github_no_update] = true to your configuration"
+            end
           end
 
           def display_debug_info
-            Chef::Log.debug("github_url: " + @github_url.to_s)
-            Chef::Log.debug("github_org: " + @github_organizations.to_s)
-            Chef::Log.debug("github_api: " + @github_api_version.to_s)
-            Chef::Log.debug("github_link: " + @github_link.to_s)
-            Chef::Log.debug("github_cache: " + @github_cache.to_s)
-            Chef::Log.debug("github_ssl_mode: " + @github_ssl_verify_mode.to_s)
+            Chef::Log.debug("github_url           : " + @github_url.to_s)
+            Chef::Log.debug("github_org           : " + @github_organizations.to_s)
+            Chef::Log.debug("github_api           : " + @github_api_version.to_s)
+            Chef::Log.debug("github_link          : " + @github_link.to_s)
+            Chef::Log.debug("github_ssl_mode      : " + @github_ssl_verify_mode.to_s)
           end
 
           def locate_config_value(key)
@@ -87,8 +119,9 @@ class Chef
             config[key] || Chef::Config[:knife][key]
           end
 
-          def get_github_link(link)
-            git_link = case link
+          def  get_repo_clone_link
+            link = locate_config_value('github_link')
+            repo_link = case link
               when 'ssh' then 'ssh_url'
               when 'http' then 'clone_url'
               when 'https' then 'clone_url'
@@ -97,34 +130,203 @@ class Chef
               when 'git' then 'git_url'
               else 'ssh_url'
             end
-            git_link
+            return repo_link
+          end
+
+          def get_all_repos(orgs)
+            # Parse every org and merge all into one hash
+            repos = {}
+            orgs.each do |org|
+              get_repos(org).each { |repo| name = repo['name'] ; repos["#{name}"] = repo } 
+            end
+            repos
+          end
+
+          def get_repos(org)
+            dns_name  = get_dns_name(@github_url)
+            file_cache = "#{ENV['HOME']}/.chef/.#{dns_name.downcase}_#{org.downcase}" 
+
+            if File.exists?(file_cache + ".json")
+              json =  JSON.parse(File.read(file_cache + ".json"))
+              json_updated = Time.parse(json['updated_at'])
+              Chef::Log.info("#{org} - cache created at : " + json_updated.to_s)
+              repo_updated = get_org_updated_time(org)
+              Chef::Log.info("#{org} - repos updated at : " + repo_updated.to_s)
+        
+	      unless json_updated >= repo_updated 
+                # update cache file
+                create_cache_file(file_cache + ".cache", org)
+                create_cache_json(file_cache + ".json", org)
+              end
+            else
+              create_cache_file(file_cache + ".cache", org)
+              create_cache_json(file_cache + ".json", org)
+            end
+            
+            # use cache files
+            JSON.parse(File.read(file_cache + ".cache"))
+          end
+
+          def create_cache_json(file, org)
+            Chef::Log.debug("Updating the cache file: #{file}")
+            url  = @github_url + "/api/" + @github_api_version + "/orgs/" + org
+            params = {'response' => 'json'} 
+            result = send_request(url, params)
+            File.open(file, 'w') { |f| f.write(JSON.pretty_generate(result)) }
+          end
+
+          def create_cache_file(file, org)
+            Chef::Log.debug("Updating the cache file: #{file}")
+            result = get_repos_github(org)
+            File.open(file, 'w') { |f| f.write(JSON.pretty_generate(result)) }
+          end
+         
+          def get_org_updated_time(org)
+            url  = @github_url + "/api/" + @github_api_version + "/orgs/" + org
+            params = {'response' => 'json'}
+            result = send_request(url, params)
+            Time.parse(result['updated_at'])
+          end
+
+          def get_repos_github(org)
+            # Get all repo's for the org from github
+            arr  = []
+            page = 1
+            url  = @github_url + "/api/" + @github_api_version + "/orgs/" + org + "/repos" 
+            while true
+              params = {'response' => 'json', 'page' => page }
+              result = send_request(url, params)
+              break if result.nil? || result.count < 1
+              result.each { |key|
+                if key['tags_url']
+                  tags = get_tags(key)
+                  key['tags'] = tags unless tags.nil? || tags.empty?
+                  key['latest_tag'] = get_latest_tag(tags)
+                  arr << key
+                else 
+                  arr << key 
+                end
+              }
+              page = page + 1
+            end
+            arr
+          end
+
+          def get_tags(repo)
+            params = {'response' => 'json'}
+            tags = send_request(repo['tags_url'], params)
+            tags
+          end
+
+          def get_latest_tag(tags)
+            return "" if tags.nil? || tags.empty?
+            tags_arr =[]
+            tags.each do |tag|
+              tags_arr.push(Mixlib::Versioning.parse(tag['name'])) if tag['name'] =~ /^(\d*)\.(\d*)\.(\d*)$/
+            end
+            return "" if tags_arr.nil? || tags_arr.empty?
+            return tags_arr.sort.last.to_s
+          end
+
+          def get_dns_name(url)
+            url = url.downcase.gsub("http://","") if url.downcase.start_with?("http://")
+            url = url.downcase.gsub("https://","") if url.downcase.start_with?("https://")
+            url
           end
 
           def send_request(url, params = {})
-            params['response'] = 'json'
-
-            params_arr = []
-            params.sort.each { |elem|
-              params_arr << elem[0].to_s + '=' + CGI.escape(elem[1].to_s).gsub('+', '%20').gsub(' ','%20')
-            }
-            data = params_arr.join('&')
-    
-            github_url = "#{url}?#{data}"
-            # Chef::Log.debug("URL: #{github_url}")
-
-            uri = URI.parse(github_url)
+            unless params.empty?
+              params_arr = []
+              params.sort.each { |elem|
+                params_arr << elem[0].to_s + '=' + CGI.escape(elem[1].to_s).gsub('+', '%20').gsub(' ','%20')
+              }
+              data = params_arr.join('&')
+              url = "#{url}?#{data}" 
+            end
+
+            if @github_ssl_verify_mode == "verify_none"
+              config[:ssl_verify_mode] = :verify_none
+            elsif @github_ssl_verify_mode == "verify_peer"
+              config[:ssl_verify_mode] = :verify_peer
+            end
+
+            Chef::Log.debug("URL: " + url.to_s)
+
+            uri = URI.parse(url)
             req_body = Net::HTTP::Get.new(uri.request_uri)
             request = Chef::REST::RESTRequest.new("GET", uri, req_body, headers={})
-
+ 
             response = request.call
-
-            if !response.is_a?(Net::HTTPOK) then
+          
+            unless response.is_a?(Net::HTTPOK) then
               puts "Error #{response.code}: #{response.message}"
               puts JSON.pretty_generate(JSON.parse(response.body))
               puts "URL: #{url}"
               exit 1
             end
-            json = JSON.parse(response.body)
+
+            begin
+              json = JSON.parse(response.body)
+            rescue
+              ui.warn "The result on the RESTRequest is not in json format"
+              ui.warn "Output: " + response.body
+              exit 1
+            end
+            return json
+          end
+
+          def get_clone(url, cookbook)
+              if ! File.directory? @github_tmp
+                 Dir.mkdir("#{@github_tmp}")
+              end
+              Dir.mkdir("#{@github_tmp}/git")
+              ui.info("Getting #{@cookbook_name} from #{url}")
+              output = `git clone #{url} #{@github_tmp}/git/#{cookbook} 2>&1`
+              if $?.exitstatus != 0
+                 Chef::Log.error("Could not clone the repository for: #{cookbook}")
+                 FileUtils.remove_entry(@github_tmp)
+                 exit 1
+              end
+              return true
+          end
+
+          def add_tag(version)
+              cpath = cookbook_path_valid?(@cookbook_name, false)
+              Dir.chdir(cpath)
+              Chef::Log.debug "Adding tag"
+              output = `git tag -a "#{version}" -m "Added tag #{version}" 2>&1`
+              if $?.exitstatus != 0
+                 Chef::Log.error("Could not add tag for: #{@cookbook_name}")
+                 FileUtils.remove_entry(@github_tmp)
+                 exit 1
+              end
+          end
+
+          def cookbook_path_valid?(cookbook_name, check_exists)
+            cookbook_path = config[:cookbook_path] || Chef::Config[:cookbook_path]
+            if cookbook_path.nil? || cookbook_path.empty?
+              Chef::Log.error("Please specify a cookbook path")
+              exit 1
+            end
+
+            unless File.exists?(cookbook_path.first) && File.directory?(cookbook_path.first)
+              Chef::Log.error("Cannot find the directory: #{cookbook_path.first}")
+              exit 1
+            end
+
+            cookbook_path = File.join(cookbook_path.first,cookbook_name)
+            if check_exists
+                if File.exists?(cookbook_path)
+                  ui.info("Processing [S] #{cookbook_name}")
+                  Chef::Log.info("Path to #{cookbook_path} already exists, skipping.")
+                  return nil
+                end
+            else
+                if ! File.exists?(cookbook_path)
+                  return nil
+                end
+            end
+            return cookbook_path
           end
 
         end

data/lib/chef/knife/github_baselist.rb

@@ -0,0 +1,102 @@
+#
+# Author:: Sander Botman (<sbotman@schubergphilis.com>)
+# Copyright:: Copyright (c) 2013 Sander Botman.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+require 'chef/knife'
+
+class Chef
+  class Knife
+    module GithubBaseList
+
+      def self.included(includer)
+        includer.class_eval do
+    
+          option :fields,
+                 :long => "--fields 'NAME, NAME'",
+                 :description => "The fields to output, comma-separated"
+      
+          option :fieldlist,
+                 :long => "--fieldlist",
+                 :description => "The available fields to output/filter",
+                 :boolean => true
+      
+          option :noheader,
+                 :long => "--noheader",
+                 :description => "Removes header from output",
+                 :boolean => true
+      
+          def display_info(data, columns, match = [])
+            object_list = []
+      
+            if config[:fields]
+              config[:fields].split(',').each { |n| object_list << ui.color(("#{n}").capitalize.strip, :bold) }
+            else
+              columns.each { |c| r = c.split(","); object_list << ui.color(("#{r.last}").strip, :bold) }
+            end
+      
+            col = object_list.count
+            object_list = [] if config[:noheader]
+      
+            data.each do |k,v|
+              if config[:fields]
+                 config[:fields].downcase.split(',').each { |n| object_list << ((v["#{n}".strip]).to_s || 'n/a') }
+              else
+                color = :white
+                if config[:mismatch] && !match.empty? && !config[:all]
+                  matches = []; match.each { |m| matches << v[m].to_s }
+                  next if matches.uniq.count == 1 
+                  color = :yellow
+                end
+                columns.each { |c|  r = c.split(","); object_list << ui.color((v["#{r.first}"]).to_s, color) || 'n/a' }
+              end
+            end
+      
+            puts ui.list(object_list, :uneven_columns_across, col)
+            display_object_fields(data) if locate_config_value(:fieldlist)
+          end
+      
+          def display_object_fields(object)
+            exit 1 if object.nil? || object.empty?
+            object_fields = [
+              ui.color('Key', :bold),
+              ui.color('Type', :bold),
+              ui.color('Value', :bold)
+             ]
+             object.first.each do |n|
+              if n.class == Hash
+                n.keys.each do |k,v|
+                  object_fields << ui.color(k, :yellow, :bold)
+                  object_fields << n[k].class.to_s
+                  if n[k].kind_of?(Array)
+                    object_fields << '<Array>'
+                  elsif n[k].kind_of?(Hash)
+                    object_fields << '<Hash>'
+                  else
+                    object_fields << ("#{n[k]}").strip.to_s
+                  end
+                end
+              end
+            end
+            puts "\n"
+            puts ui.list(object_fields, :uneven_columns_across, 3)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef/knife/github_cleanup.rb

@@ -0,0 +1,120 @@
+#
+# Author:: Sander Botman (<sbotman@schubergphilis.com>)
+# Copyright:: Copyright (c) 2013 Sander Botman.
+# License:: Apache License, Version 2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ---------------------------------------------------------------------------- #
+# Abstract
+# ---------------------------------------------------------------------------- #
+# This code is intended to help you cleaning up your locate repo's.
+# It will check if your repo if insync with the github and will not touch if
+# this is not the case. Then it will check if you have any branches local
+# and not on the github.
+#
+# If it cannot find any uncommitted changes, it will safely remove your repo.
+# It's good practice to cleanup and re-download repos because this way they 
+# can move from organization to organization.
+# ---------------------------------------------------------------------------- #
+#
+require 'chef/knife'
+
+module KnifeGithubCleanup
+  class GithubCleanup < Chef::Knife
+
+  deps do
+    require 'chef/knife/github_base'
+      include Chef::Knife::GithubBase
+      require 'chef/mixin/shell_out'
+    end
+    
+    banner "knife github cleanup [REPO] (options)"
+    category "github"
+
+    option :all,
+           :short => "-a",
+           :long => "--all",
+           :description => "Delete all repos from cookbook path.",
+           :boolean => true
+
+    option :force,
+           :short => "-f",
+           :long => "--force",
+           :description => "Force deletion even if commits are still present.",
+           :boolean => true
+
+    def run
+
+      #executing shell commands
+      extend Chef::Mixin::ShellOut
+
+      # validate base options from base module.
+      validate_base_options      
+
+      # Display information if debug mode is on.
+      display_debug_info
+
+      # Gather all repo information from github.
+      all_repos = get_all_repos(@github_organizations.reverse)
+
+      # Get all chef cookbooks and versions (hopefully chef does the error handeling).
+      cookbooks = rest.get_rest("/cookbooks?num_version=1")
+
+      # Get the cookbook names from the command line
+      @cookbook_name = name_args.first unless name_args.empty?
+      if @cookbook_name
+        # repo = all_repos.select { |k,v| v["name"] == @cookbook_name }
+        repo_cleanup(@cookbook_name)
+      elsif config[:all]
+        cookbooks.each do |c,v|
+          repo_cleanup(c)
+        end
+      else
+        Chef::Log.error("Please specify a repo name")
+      end
+    end
+
+    def repo_cleanup(repo)
+      cookbook_path = config[:cookbook_path] || Chef::Config[:cookbook_path]
+      cookbook = File.join(cookbook_path.first,repo)
+      if File.exists?(cookbook)
+        if repo_status_clean?(repo, cookbook)
+          # delete the repo
+          ui.info("Processing [D] #{repo}")
+          FileUtils.remove_entry(cookbook) 
+        end
+      else
+        puts "cannot find repo path for: #{repo}" unless config[:all]
+      end
+    end
+
+    def repo_status_clean?(repo, cookbook)
+      shell_out!("git fetch", :cwd => cookbook)
+      status = shell_out!("git status", :cwd => cookbook)
+      unless status.stdout == "# On branch master\nnothing to commit (working directory clean)\n"
+        ui.info("Processing [C] #{repo} (Action needed!)")
+        status.stdout.lines.each { |l| puts l.sub( /^/, "    ") }
+        return false   
+      end
+      log = shell_out!("git log --branches --not --remotes --simplify-by-decoration --decorate --oneline", :cwd => cookbook)
+      unless log.stdout.empty?
+        ui.info("Processing [B] #{repo} (Action needed!)")
+        ui.info("    Please check your branches, one of them has unsaved changes")
+        log.stdout.lines.each { |l| puts l.sub( /^/, "    ") }
+        return false   
+      end
+      return true
+    end
+  end
+end