gitchefsync 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d42fe53a3355fac66554627bcc30fedc5a28d9a
-  data.tar.gz: 6a277f678f793afdf12308510e8be5466711068a
+  metadata.gz: a57596b84302167acc4a08933a924eded9d26f57
+  data.tar.gz: f672d7e712813563a212cab7aa5118de2b840c31
 SHA512:
-  metadata.gz: c6c0d391f1033f08f0e919c38be05033e057b1195d6db064d2033abae366eaf1c1c62156f6a0873f9b8e15509c443c2354e0d39a46dba7e9fcc6969e1b8cd590
-  data.tar.gz: b1f724fa77ee87ab65efa1bd48234fa34bacc96026222e96a25deced9cebee57469ee0207eaa0321295885651bab3018874bb3c35e73f59f0d46750831676486
+  metadata.gz: ae21d609ac0004a3825f44a6b12278abe40b1feabb739152aabb532ae4bade6e9d3ee29561dd44bedfaa37145ca11f7cd40696e54fe831ef3b2b896f91550957
+  data.tar.gz: 94da27966929e77ac27cd6ade202776801d9c0110f86fb1ff523f2a778df42c7338e764d800202d1f19a480318618140ff03bef742ecd346cbf54795d746b7ec

data/gitchefsync.gemspec

@@ -1,4 +1,20 @@
 # coding: utf-8
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'gitchefsync/version'
@@ -23,7 +39,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "gitlab", '~> 3.0.0'
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "rake"
-  spec.homepage = "https://gitlab.rim.net/mandolin/gitchefsync/tree/master"
+  spec.homepage = "https://gitlab.rim.net/chef/gitchefsync/tree/master"
 
   
 end

data/lib/gitchefsync.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'optparse'
 require 'gitlab'
 require 'json'
@@ -36,10 +52,10 @@ module Gitchefsync
   #number of files
   def self.trimAudit
     logger.debug("event_id=trim_audit_files:keep=#{@audit_keep_trim}")
-    audit = Audit.new(@config['stage_dir'], 'env' )
+    audit = Audit.new(@audit_dir, 'env' )
     audit.trim(@audit_keep_trim)
     
-    audit = Audit.new(@config['stage_dir'], 'cb' )
+    audit = Audit.new(@audit_dir, 'cb' )
     audit.trim(@audit_keep_trim)
     
   end
@@ -48,8 +64,8 @@ module Gitchefsync
     
     notification = Notification.new(@config['smtp_server'])
     
-    notification.notifyFromAudit(@config['stage_dir'], 'cb' )
-    notification.singleNotifyFromAudit(@config['stage_dir'],'env',@config['default_notify_email']) 
+    notification.singleNotifyFromAudit(@audit_dir, 'cb',@config['default_notify_email'] )
+    notification.singleNotifyFromAudit(@audit_dir,'env',@config['default_notify_email']) 
     
     notification.close
   end
@@ -70,6 +86,7 @@ module Gitchefsync
     
     
     FS.knifeReady(@options[:git_local], @options[:knife_config])
+    @knife_ready = true
     
     if !@config['sync_local'] 
      
@@ -163,18 +180,19 @@ module Gitchefsync
     include FS,Git
    
     logger.info "event_id=stage_cookbooks:git_local=#{@git_local}"
-    FS.knifeReady(@options[:git_local], @options[:knife_config])
+    FS.knifeReady(@options[:git_local], @options[:knife_config]) unless @knife_ready
     ret_status = Hash.new
     
     
     #not sure if this should be globally governed?
-    audit = Audit.new(@config['stage_dir'], 'cb')
+    audit = Audit.new(@audit_dir, 'cb')
    
     
     knifeUtil = KnifeUtil.new(@knife, @git_local)
     #Have a delta point: interact with the chef server to identify delta
     listCB = knifeUtil.listCookbooks
-    
+    #list of cookbooks processed
+    list_processed = Array.new
     cookbook_dirs = Dir.entries(@git_local).reject! {|item| item.start_with?(".") }
     cookbook_dirs.each do |dir|
 
@@ -192,7 +210,9 @@ module Gitchefsync
           logger.debug "event_id=git_checkout:path=#{path}:tag=#{tag}"
           Git.cmd "cd #{path} && #{@git_bin} checkout #{tag}"
           
-          self.processCookbook(path,audit)
+          cb = self.processCookbook(path,audit)
+          list_processed << @stage_dir + "/" + cb.berksTar() unless cb.nil?
+          
         rescue NoMetaDataError => e
           #No audit written on failure to parse metadata
           logger.info "event_id=nometadata:dir=#{dir}"
@@ -216,7 +236,17 @@ module Gitchefsync
       end
     end
     
-
+    #compared list_processed with what is in stage
+    #and delete the tars
+    stage = @stage_dir + "/*tar.gz"
+    existing = Dir.glob(stage)
+    to_del = existing - list_processed
+    logger.info "event_id=list_tar_delta:del+list=#{to_del}"
+    to_del.each do |file| 
+      File.delete(file)
+    end
+    #reconcile method will actually delete the cookbooks from server
+     
     #write out the audit file
     audit.write
     #clean the audit files
@@ -232,34 +262,44 @@ module Gitchefsync
     cookbook = knifeUtil.parseMetaData(path)
     logger.debug "event_id=processing:cookbook=#{cookbook}"
     
+    
     if cookbook != nil
       stage_tar = @config['stage_dir'] +"/" + cookbook.berksTar()
       tar_exists = File.exists?(stage_tar)
     end
     
     begin
+      
+      #get some git historical info
+      extra = Hash.new
+      extra['sha'] = (Git.cmd "cd #{path} && git log -1 --pretty=%H").chomp
+      extra['author_email'] = (Git.cmd "cd #{path} && git log -1 --pretty=%ce").chomp
+      extra['date'] = (Git.cmd "cd #{path} && git log -1 --pretty=%cd").chomp
+      extra['subject'] = (Git.cmd "cd #{path} && git log -1 --pretty=%s").chomp
+        
       if  (cookbook !=nil && (!knifeUtil.isCBinList(cookbook, self.serverCookbooks()) || !tar_exists ))
         berks_tar = self.stageBerks(path ,  @config['stage_dir'])
         #upload cookbooks still puts a Berksfile, will refactor this method
         self.uploadBerks(path)
         logger.debug("event_id=staging:cookbook=#{cookbook}:berks_tar=#{berks_tar}")
         self.stageCBUpload(berks_tar, @stage_cb_dir, knifeUtil, self.serverCookbooks())
-        audit.addCookbook(cookbook) if berks_tar.nil?
+        audit.addCookbook(cookbook,"UPDATE",nil,extra) if berks_tar.nil?
         logger.info "event_id=cookbook_staged:cookbook=#{cookbook}"
   
       elsif cookbook !=nil && @config['force_package']
         logger.info "event_id=cookbook_force_package:cookbook=#{cookbook}"
         self.stageBerks(path, @config['stage_dir'])
-      else
-        audit.addCookbook(cookbook, "EXISTING")
+       elsif cookbook != nil
+        audit.addCookbook(cookbook, "EXISTING",nil,extra)
         logger.info "event_id=cookbook_untouched:cookbook=#{cookbook}"
       end
     rescue BerksError => e
       logger.error "event_id=berks_package_failure:msg=#{e.message}:trace=#{e.backtrace}"
-      audit.addCookbook(cookbook, "ERROR", e)
+      audit.addCookbook(cookbook, "ERROR", e,  extra)
     end
     
     Git.cmd "cd #{path} && git clean -xdf"
+    return cookbook
   end
 
   #do a berks upload of the path
@@ -307,7 +347,11 @@ module Gitchefsync
   #returns the path to the berks tar file
   def self.stageBerks(path, stage_dir)
     include FS
+    
     begin
+      if File.exists?(File.join(path,"Berfile.lock"))
+        raise BerksLockError, "Berks lock file found"
+      end
       if File.exists?(File.join(path, "Berksfile"))
         logger.debug "event_id=Stage_cookbook:path=#{path}"
         
@@ -345,7 +389,8 @@ module Gitchefsync
       else
         raise NoBerksError, "Unable to locate Berks file for #{path}"
       end
-    rescue NoBerksError => e 
+    
+    rescue NoBerksError,BerksLockError => e 
       raise e
     rescue Exception => e
       raise BerksError.new(e.message) 
@@ -366,7 +411,7 @@ module Gitchefsync
 
    
     #read in the latest audit - fail on non-null exceptions
-    audit = Audit.new(@config['stage_dir'],'cb' )
+    audit = Audit.new(@audit_dir,'cb' )
      
     json = audit.parseLatest 
     if json != nil && audit.hasError(json)
@@ -409,13 +454,31 @@ module Gitchefsync
         raise InvalidTar, "Invalid tar name #{file}"
       end
 
-      logger.debug "In chef server? #{knifeUtil.inList(match[1],match[2],listCB)}"
+      #logger.debug "In chef server? #{knifeUtil.inList(match[1],match[2],listCB)}"
       
+     
       if !knifeUtil.inList(match[1],match[2],listCB) || forceUpload
         logger.info "event_id=stage_upload:cookbook=#{match[1]}:ver=#{match[2]}:dir=#{cookbook_dir}"
         FS.cmd "tar -xf #{file} -C #{cookbook_dir}"
-        out = FS.cmd "cd #{cookbook_dir} && #{@knife} cookbook upload -a --force --cookbook-path=#{cookbook_dir}/cookbooks"
-        
+        new_cb_list = Array.new       
+        cb_dir = Dir.entries(cookbook_dir + "/cookbooks")
+        cb_dir.each do |dir|
+          
+          begin
+            cb_info = knifeUtil.parseMetaData(cookbook_dir  + "/cookbooks/" + dir)
+            if knifeUtil.inList(cb_info.name(),cb_info.version,listCB)
+              logger.debug "event_id=del_cb_in_server:name=#{cb_info}"
+              FS.cmdNoError "rm -fr #{cookbook_dir}/cookbooks/#{cb_info.name}"
+            else
+              #TODO: add this as a concat method in knife_util class
+               new_cb_list << cb_info
+            end
+          rescue NoMetaDataError => e
+            logger.debug "no metadata on #{dir}"
+          end
+        end
+        out = FS.cmd "cd #{cookbook_dir} && #{@knife} cookbook upload -a --cookbook-path=#{cookbook_dir}/cookbooks"
+        listCB.concat(new_cb_list)
         logger.debug "event_id=stage_upload_output=\n#{out}"
       else
         logger.info"event_id=stage_no_upload:cookbook=#{match[1]}:ver=#{match[2]}"
@@ -453,7 +516,7 @@ module Gitchefsync
   def self.reconcile
 
     #Validation
-    if Audit.new(@config['stage_dir'], 'cb').latest == nil
+    if Audit.new(@audit_dir, 'cb').latest == nil
       logger.warn "event_id=reconcile_no_audit_detected"
       return
     end

data/lib/gitchefsync/audit.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'json'
 require 'gitchefsync/config'
 require 'gitchefsync/io_util'
@@ -15,7 +31,7 @@ module Gitchefsync
 
     #adds an audit item
     def add ( item )
-      @list << item.to_hash
+      @list << item
     end
 
     def addItem(name, version)
@@ -23,16 +39,16 @@ module Gitchefsync
       add(item)
     end
 
-    def addCookbook(cookbook,action="UPDATE",exception=nil)
-      item = AuditItem.new(cookbook.name,cookbook.version,exception,action)
+    def addCookbook(cookbook,action="UPDATE",exception=nil, extra_info = nil)
+      item = AuditItem.new(cookbook.name,cookbook.version,exception,action,extra_info)
       item.setCookbook(cookbook)
       add(item)
     end
 
     #This gives enough information
-    def addEnv(name,action='UPDATE',exception=nil)
+    def addEnv(name,action='UPDATE',exception=nil, extra_info=nil)
       cb = Cookbook.new(name,'','mandolin','mandolin@blackberry.com')
-      addCookbook(cb,action,exception)
+      addCookbook(cb,action,exception,extra_info)
     end
 
     #writes the audit out
@@ -40,12 +56,26 @@ module Gitchefsync
     #and an audit-type-timestamp.json
     def write
       begin
+        unless File.exists? @fileLocation
+          FS.cmd "mkdir -p #{@fileLocation}"
+        end
         fileLoc = @fileLocation + "/audit-" +@type+ @ts.to_s + ".json"
         #fileCurrent = @fileLocation + "/audit-" +@type+ "-current" + ".json"
         if @list.length > 0
           Gitchefsync.logger.debug "event_id=write_audit:file_loc=#{fileLoc}"
           file = File.open(fileLoc, "w")
-          json = JSON.generate(@list)
+          list_hash = Array.new
+          @list.each do |item|
+            list_hash << item.to_hash
+          end
+          audit_hash = Hash.new
+          audit_hash['host_source'] = FS.cmd "hostname"
+          audit_hash['date'] = Time.now
+          #time taken from time of start of audit process -construction, until it's writing (now)
+          audit_hash['audit_written_secs'] = Time.now.to_i - @ts
+          audit_hash['num_items'] = list_hash.length()
+          audit_hash['items'] = list_hash
+          json = JSON.generate(audit_hash)
           file.write(json)
           #create sym link to this file
           latest = @fileLocation+ "/audit_" + @type + "_latest.json"
@@ -65,11 +95,24 @@ module Gitchefsync
 
     #returns the latest audit file
     def latest
+      latest = @fileLocation+ "/audit_" + @type + "_latest.json"
+      if File.exists? latest
+        return latest
+      end            
+      file = fileFrom(-1)
+      file
+    end
+    
+    #returns the file from the latest:-1, next:-2 and so on
+    #returning nil if nothing found
+    def fileFrom(index)
       entries = Dir.glob(@fileLocation + "/audit-#{@type}*")
       file = nil
-      if entries != nil
-        file = entries.sort[entries.length-1]
+     
+      if entries != nil && (entries.length >= -index) 
+        file = entries.sort[entries.length + index]
       end
+      
       file
     end
 
@@ -131,23 +174,74 @@ module Gitchefsync
       end
       json
     end
-
-    def latestAuditItems
-      json = parseLatest
+    
+    #get audit hash from the 
+    #
+    def auditItems(index)
+      file = fileFrom(index)
+      audit_list = Array.new
+      
+      if !file.nil?
+        json = JSON.parse(File.read(file))
+      end
       if json.nil?
-        raise AuditError, "No json available"
+        return nil
       end
       audit_list = Array.new
-      json.each do |audit_item|
+      #keep backward compatibility
+      if json.kind_of?(Array)
+        items = json
+      else
+        items = json['items']
+      end
+      items.each do |audit_item|
         audit_list << AuditItem.new(nil,nil).from_hash(audit_item)
       end
       audit_list
+            
+    end
+    
+    def latestAuditItems
+      auditItems(-1)
+    end
+    
+    #finds the audit item by audit item name - to be used with a 
+    #method latestAuditItems (above)
+    #@param name - the name of the item
+    #@param audit - an array of audit items
+    def itemByName(name, audit)
+      ret = nil
+      audit.each do |item|
+        
+        if item.name.eql? name
+          ret = item
+          break
+        end 
+      end
+      ret
+    end
+    def itemByNameVersion(name,version, audit)
+      ret = nil
+      audit.each do |item|
+        
+        if item.name.eql?(name) && item.version.eql?(version)
+          ret = item
+          break
+        end 
+      end
+      ret
     end
-
     #if the json has exceptions
     def hasError (json_obj)
       ret = false
-      json_obj.each do |item|
+      #keep backward compatibility
+      if json_obj.kind_of?(Array)
+        items = json_obj
+      else
+        items = json_obj['items']
+      end
+   
+      items.each do |item|
         if item['exception'] != nil then ret = true end
       end
       ret
@@ -158,17 +252,21 @@ module Gitchefsync
   #TODO: just reference Cookbook clas in knife_util
   class AuditItem
 
-    def initialize(name, version, exception = nil, action = 'UPDATE', ts = Time.now)
+    def initialize(name, version, exception = nil, action = 'UPDATE', extra_info = nil, ts = Time.now)
       @name = name
       @version = version
       @ts = ts.to_i
       @exception = exception
       @action = action
+      @extra_info = extra_info
     end
 
     def name
       @name
     end
+    def version
+      @version
+    end
     def ex
       @exception
     end
@@ -187,6 +285,12 @@ module Gitchefsync
       @action = action
     end
 
+    def extra_info
+      @extra_info 
+    end
+    def set_extra_info hash
+      @extra_info = hash
+    end
     #this method doesn't work when called when exception is created from json (from_hash)
     def to_hash
       h = Hash.new
@@ -202,6 +306,7 @@ module Gitchefsync
       h[:action] = @action unless @action == nil
       h[:maintainer] = @cookbook.maintainer unless @cookbook == nil
       h[:maintainer_email] = @cookbook.maintainer_email unless @cookbook == nil
+      h[:extra_info] = @extra_info unless @extra_info == nil
       h
     end
 
@@ -213,6 +318,7 @@ module Gitchefsync
       @type = h['type']
       @action = h['action']
       @cookbook = Cookbook.new(@name,@version,h['maintainer'],h['maintainer_email'])
+      @extra_info = h['extra_info']
       return self
     end
   end

data/lib/gitchefsync/common.rb

@@ -1,3 +1,20 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
+
 module Gitchefsync
 
   extend Gitchefsync::Configuration
@@ -43,6 +60,11 @@ module Gitchefsync
  
   #@param project - Gitlab project object
   def self.pullProject(project, verify_yml=false)
+    
+    #MAND-791 skip private repositories
+    if !project['public']
+      logger.warn "event_id=private_project_detected:project=#{project['path_with_namespace']}"
+    end
     p_name = project['path_with_namespace'].split('/').join('_')
     project_path = File.join(@git_local, p_name)
 

data/lib/gitchefsync/config.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitchefsync/opts'
 require 'logger'
 #Not available till ruby 2.0
@@ -27,6 +43,8 @@ module Gitchefsync
       @git_local = options[:git_local]
       @token =  options[:private_token]
       @stage_dir = config['stage_dir']
+      @audit_dir = config['stage_dir'] + "/audit"
+      config['audit_dir'] = @audit_dir
       @rel_branch = config['release_branch']
       @rel_branch ||= 'master'
       @stage_cb_dir = options[:stage_cookbook_dir]

data/lib/gitchefsync/env_sync.rb

@@ -1,9 +1,26 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitchefsync/git_util'
 require 'gitchefsync/errors'
 require 'gitchefsync/io_util'
 require 'gitchefsync/audit'
 require 'gitchefsync/config'
 require 'gitchefsync/common'
+require 'digest'
 
 module Gitchefsync
 
@@ -84,7 +101,9 @@ module Gitchefsync
       @stage_filepath = config['stage_dir']
       @force_upload = config['force_upload']
       @repo_list = repo_list
-      @audit = Audit.new(config['stage_dir'], 'env' )
+      #this is a bit of a hack to determine if we're writing audit
+      @sous = config['sync_local']
+      @audit = Audit.new(config['audit_dir'], 'env' )
       @audit_keep_trim = config['audit_keep_trim']
       @audit_keep_trim ||= 20
       @env_file_list = Array.new()
@@ -92,13 +111,13 @@ module Gitchefsync
       @role_file_list = Array.new()
     end
 
-    def reject_json file
+    def reject_json content
       file_json = nil
       begin
-        json = File.read file
-        file_json = JSON.parse json
+        
+        file_json = JSON.parse content
       rescue Exception => e
-        Gitchefsync.logger.error "event_id=env_parse_error:file=#{file}"
+        Gitchefsync.logger.error "event_id=env_json_parse_error:file=#{file}"
         @audit.addEnv(file,'UPDATE', e )
       end
       file_json
@@ -117,23 +136,23 @@ module Gitchefsync
       Gitchefsync.logger.debug "event_id=json_is_valid:iden=#{iden}:basename=#{f['basename']}"
     end
 
-    def upload_env(f, git_delta)
-      Gitchefsync.logger.debug "event_id=upload_env:filepath=#{f['fullpath']}"
+    def upload_env(f, delta)
+      Gitchefsync.logger.debug "event_id=upload_env:filepath=#{f['fullpath']}:delta=#{delta}"
       begin
         validate_json(f, 'name')
         @env_file_list << f['json']['name']
-        show_out = FS.cmdNoError "#{@knife} environment show #{f['json']['name']}"
-        if show_out.match("ERROR:") || git_delta || @force_upload
+        
+        if delta || @force_upload
           FS.cmd "#{@knife} environment from file #{f['fullpath']} --yes"
           Gitchefsync.logger.info "event_id=environment_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'UPDATE' )
+          @audit.addEnv(f['fullpath'],'UPDATE',nil,f['extra_info'] )
         else
           Gitchefsync.logger.debug "event_id=environment_not_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'EXISTING' )
+          @audit.addEnv(f['fullpath'],'EXISTING',nil,f['extra_info'] )
         end
       rescue ValidationError => e
         Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
-        @audit.addEnv(f['fullpath'],'UPDATE', e )
+        @audit.addEnv(f['fullpath'],'UPDATE', e ,f['extra_info'])
       end
     end
 
@@ -147,45 +166,45 @@ module Gitchefsync
       raise ValidationError, "event_id=invalid_path_to_data_bag_json:path=#{fullpath}"
     end
 
-    def upload_db(f, git_delta)
-      Gitchefsync.logger.debug "event_id=upload_data_bag:filepath=#{f['fullpath']}"
+    def upload_db(f, delta)
+      Gitchefsync.logger.debug "event_id=upload_data_bag:filepath=#{f['fullpath']}:delta=#{delta}"
       db_iden = data_bag_iden(f['fullpath'])
       begin
         validate_json(f, 'id')
         @db_file_list << [db_iden, f['json']['id']]
-        show_out = FS.cmdNoError "#{@knife} data bag show #{db_iden} #{f['json']['id']}"
-        if show_out.match("ERROR:") || git_delta || @force_upload
+        #show_out = FS.cmdNoError "#{@knife} data bag show #{db_iden} #{f['json']['id']}"
+        if delta || @force_upload
           FS.cmd "#{@knife} data bag create #{db_iden}"
           FS.cmd "#{@knife} data bag from file #{db_iden} #{f['fullpath']}"
           Gitchefsync.logger.info "event_id=databag_uploaded:file_json_name=#{f['json']['id']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'UPDATE' )
+          @audit.addEnv(f['fullpath'],'UPDATE', nil, f['extra_info'] )
         else
           Gitchefsync.logger.debug "event_id=data_bag_not_uploaded:file_json_name=#{f['json']['id']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'EXISTING')
+          @audit.addEnv(f['fullpath'],'EXISTING', nil, f['extra_info'])
         end
       rescue ValidationError => e
         Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
-        @audit.addEnv(f['fullpath'],'UPDATE', e )
+        @audit.addEnv(f['fullpath'],'UPDATE', e , f['extra_info'])
       end
     end
 
-    def upload_role(f, git_delta)
-      Gitchefsync.logger.debug "event_id=upload_role:fullpath=#{f['fullpath']}"
+    def upload_role(f, delta)
+      Gitchefsync.logger.debug "event_id=upload_role:fullpath=#{f['fullpath']}:delta=#{delta}"
       begin
         validate_json(f, 'name')
         @role_file_list << f['json']['name']
-        show_out = FS.cmdNoError "#{@knife} role show #{f['json']['name']}"
-        if show_out.match("ERROR:") || git_delta || @force_upload
+        
+        if delta || @force_upload
           FS.cmd "#{@knife} role from file #{f['fullpath']} --yes"
           Gitchefsync.logger.info "event_id=role_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'UPDATE' )
+          @audit.addEnv(f['fullpath'],'UPDATE',nil, f['extra_info'] )
         else
           Gitchefsync.logger.debug "event_id=role_not_uploaded:file_json_name=#{f['json']['name']}:file=#{f['fullpath']}"
-          @audit.addEnv(f['fullpath'],'EXISTING' )
+          @audit.addEnv(f['fullpath'],'EXISTING', nil, f['extra_info'] )
         end
       rescue ValidationError => e
         Gitchefsync.logger.error("event_id=validation_error:msg=#{e.message}")
-        @audit.addEnv(f['fullpath'],'UPDATE', e )
+        @audit.addEnv(f['fullpath'],'UPDATE', e , f['extra_info'])
       end
     end
 
@@ -244,13 +263,15 @@ module Gitchefsync
         @audit.add(a)
       end
 
-      #TODO: must create audit for removal
-
-      @audit.write
-      #trim the audit file
-      @audit.trim(@audit_keep_trim)
+     
+      if !@sous
+        @audit.write
+        #trim the audit file
+        @audit.trim(@audit_keep_trim)
+      end 
     end
 
+    
     def update_json_files
       Gitchefsync.logger.info "event_id=update_json_files"
       @env_file_list.clear
@@ -258,25 +279,52 @@ module Gitchefsync
       @db_file_list.clear
       @role_file_list.clear
 
+      #latest audit info for delta purposes
+      begin
+        latest_audit = @audit.latestAuditItems
+        latest_audit ||= Array.new
+        Gitchefsync.logger.info "event_id=audit_file_found:length=#{latest_audit.length}"
+      rescue AuditError => e
+        Gitchefsync.logger.warn "event_id=no_audit_file_found"
+        latest_audit = Array.new
+      end
+     
+      
       @repo_list.each do |repo|
         env_dir = repo.chef_path + "/**/*json"
 
         Dir.glob(env_dir).each  do |file|
 
           file_attr = Hash.new()
-          file_attr['json'] = reject_json(file)
+          content = File.read(file)
+          
+          file_attr['json'] = reject_json(content)
           next if file_attr['json'].nil?
           file_attr['type'] = json_type(file)
           file_attr['filename'] = File.basename(file)
           file_attr['basename'] = File.basename(file).chomp(".json")
           file_attr['fullpath'] = file
-
+          file_attr['extra_info'] = Hash.new
+          #To do comparison we'll hash current
+          file_attr['extra_info']['digest'] = Digest::SHA256.hexdigest content
+          #and compare hash to last audit
+          item = @audit.itemByName(file,latest_audit)
+          delta = true
+          if !item.nil?
+            Gitchefsync.logger.debug "event_id=audit_item_found:name=#{file}:digest=#{file_attr['extra_info']['digest']}"
+            if item.extra_info != nil && file_attr['extra_info']['digest'].eql?(item.extra_info['digest'])
+              Gitchefsync.logger.debug "file_digest=#{item.extra_info['digest']}"
+              delta = false
+            end
+          else
+            Gitchefsync.logger.warn "event_id=no_audit_item_found:name=#{file}"
+          end
           if file_attr['type'].eql? "env"
-            upload_env(file_attr, repo.git_delta)
+            upload_env(file_attr, delta)
           elsif file_attr['type'].eql? "db"
-            upload_db(file_attr, repo.git_delta)
+            upload_db(file_attr, delta)
           elsif file_attr['type'].eql? "role"
-            upload_role(file_attr, repo.git_delta)
+            upload_role(file_attr, delta)
           end
         end
       end

data/lib/gitchefsync/errors.rb

@@ -1,3 +1,18 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
 
 #Define custom exception classes here
 module Gitchefsync
@@ -18,6 +33,8 @@ module Gitchefsync
   class BerksError < CmdError; end
 
   class NoBerksError < BerksError; end
+    
+  class BerksLockError < BerksError; end
 
   class KnifeError < CmdError; end
 

data/lib/gitchefsync/git_util.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitchefsync/config'
 require 'gitlab'
 #Git helper module

data/lib/gitchefsync/io_util.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'open3'
 require 'gitchefsync/errors'
 # raise an excetpion here

data/lib/gitchefsync/knife_util.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 #helper file for parsing knife commands
 #this may get refactored
 

data/lib/gitchefsync/log.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 module Gitchefsync
   module Log
     #Wrapped sys log Logger

data/lib/gitchefsync/notify.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitchefsync/audit'
 require 'gitchefsync/io_util'
 require 'net/smtp'
@@ -36,27 +52,67 @@ module Gitchefsync
       Gitchefsync.logger.info("event_id=notification_complete:audit_type=#{audit_type}")
     end
 
+    def hasDelta(item1,item2)
+     
+      if item1.nil? || item2.nil?
+        return true 
+      end
+      i1 = item1.to_hash
+      i2 = item2.to_hash
+     
+      if !i1[:extra_info].nil? && !i2[:extra_info].nil?
+        
+        if !i1[:extra_info]['digest'].nil? && !i1[:extra_info]['digest'].eql?(i2[:extra_info]['digest']) then return true end   
+        if !i1[:extra_info]['sha'].nil? && !i1[:extra_info]['sha'].eql?(i2[:extra_info]['sha']) then return true end
+      
+      end
+      return false
+    end
     #Aggregates a single email to the "to" email parameter
     def singleNotifyFromAudit(audit_dir,audit_type,to)
       audit = Audit.new(audit_dir,audit_type)
       audit_list = audit.latestAuditItems
-      msg = "From: gichefsync <mandolin@blackberry.com>\nTo: #{to}\nSubject: gitchefsync failure: summary\n\n"
-      msg << "Alert from Hostname: #{@hostname}\n\n"
+      prev_audit = audit.auditItems(-2)
+      
+      if audit_list.nil?
+        Gitchefsync.logger.warn "event_id=unable_to_notify:msg=audit_list_isnull:type=#{audit_type}"
+        return
+      end
+      empty = true
+      msg = "From: gitchefsync <mandolin@blackberry.com>\nTo: #{to}\nSubject: gitchefsync failure: summary\n\n"
+      msg << "Alert from Hostname: #{@hostname}\n"
+      type = "Environment" 
+      if audit_type.eql?("cb")
+        type = "Cookbooks"
+      end
+      
+      msg << "Notification Type: #{type}\n\n"
       audit_list.each do |audit_item|
         h = audit_item.to_hash
-
-        if h[:exception] != nil
-          ver = ""
-          if !h[:version].empty? then ver = ":" + h[:version] end
-
-          msg << "item: #{h[:name]}#{ver} was NOT processed with status #{h[:action]} "
-          msg << "\nERROR #{h[:exception]}"
-        else
-          msg << "item: #{h[:name]} was NOT processed with status #{h[:action]} "
+        Gitchefsync.logger.debug "processing item: #{h} ex=#{h[:exception]} #{!h[:exception].nil? && !h[:exception].empty?}"
+        
+        if !h[:exception].nil? && !h[:exception].empty?
+          
+          if !prev_audit.nil? && hasDelta(audit_item,audit.itemByNameVersion(h[:name],h[:version],prev_audit))
+            Gitchefsync.logger.debug  "item_has_exception=#{h}"
+            msg << "item: #{h[:name]}:#{h[:version]} was NOT processed with status #{h[:action]}\n"
+            msg << "audit_json= #{h}"
+            msg << "ERROR #{h[:exception]}\n\n"
+            empty = false
+            
+          end         
+          
         end
-        msg << "\n\n"
+        
+      end
+      
+      if !empty
+        Gitchefsync.logger.debug  "sending msg=#{msg}"
+        sendTo(to,msg) 
+      else
+        Gitchefsync.logger.info "event_id=no_message_sent"
       end
-      sendTo(to,msg)
+        
     end
 
     def send(body)

data/lib/gitchefsync/opts.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitlab'
 require 'gitchefsync/errors'
 module Gitchefsync

data/lib/gitchefsync/schedule.rb

@@ -1,3 +1,18 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
 require 'timeout'
 
 module Gitchefsync
@@ -57,7 +72,8 @@ module Gitchefsync
           Gitchefsync.reconcile
           cleanTmp()
           Gitchefsync.logger.info "event_id=master_sync_completed"
-          
+          #notification
+          Gitchefsync.notifyFromAudit()
           # MAND-615 - This file will signal its ok for this directory to be sous sync target
           File.write(File.join(@sous_rsync_src, "master_sync_completed"), "")
         end

data/lib/gitchefsync/version.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 module Gitchefsync
-  VERSION = "0.6.1"
+  VERSION = "0.6.2"
 end

data/spec/gitchefsync_spec.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'rspec'
 require 'gitchefsync'
 require 'gitchefsync/opts'

data/spec/integration.rb

@@ -1,3 +1,19 @@
+# Gitchefsync - git to chef sync toolset
+#
+# Copyright 2014, BlackBerry, Inc.
+#
+#Licensed under the Apache License, Version 2.0 (the "License");
+#you may not use this file except in compliance with the License.
+#You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing, software
+#distributed under the License is distributed on an "AS IS" BASIS,
+#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#See the License for the specific language governing permissions and
+#limitations under the License.
+
 require 'gitchefsync'
 require 'gitlab'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gitchefsync
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Marcus Simonsen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-14 00:00:00.000000000 Z
+date: 2014-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gitlab
@@ -87,7 +87,7 @@ files:
 - spec/gitchefsync_spec.rb
 - spec/integration.rb
 - spec/sync-config.json
-homepage: https://gitlab.rim.net/mandolin/gitchefsync/tree/master
+homepage: https://gitlab.rim.net/chef/gitchefsync/tree/master
 licenses:
 - MIT
 metadata: {}