gitchefsync 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7d42fe53a3355fac66554627bcc30fedc5a28d9a
+  data.tar.gz: 6a277f678f793afdf12308510e8be5466711068a
+SHA512:
+  metadata.gz: c6c0d391f1033f08f0e919c38be05033e057b1195d6db064d2033abae366eaf1c1c62156f6a0873f9b8e15509c443c2354e0d39a46dba7e9fcc6969e1b8cd590
+  data.tar.gz: b1f724fa77ee87ab65efa1bd48234fa34bacc96026222e96a25deced9cebee57469ee0207eaa0321295885651bab3018874bb3c35e73f59f0d46750831676486

data/.gitignore

@@ -0,0 +1,25 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+one.sh
+.buildpath
+.project

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,2 @@
+Copyright (c) 2014 Blackberry
+

data/README.md

@@ -0,0 +1,150 @@
+# GitChefSync
+
+This is a Git -> Chef synchronization toolset.
+
+### Requirements:
+- Git (1.8.x)
+- Chef
+- Berkshelf
+
+### Gem dependencies:
+- gitlab + transitive dependencies
+
+
+To install (from source):
+- git clone [Mandolin GitChefSync](http://gitlab.rim.net/chef/gitchefsync.git)
+- cd gitchefsync && rake install
+
+### Running:
+
+- Typical "Master" options are:
+- Cookbook sync: gitchefsync syncCookbooks\[Local\] -c /path_to_config -t xxxyyy
+- Environment sync: gitchefsync syncEnv -c /path_to_config -t xxxyyy
+- "Sous" operation:
+- stage upload: gitchefsync stagedUpload -c /path_to_config -t xxxyyy
+
+
+### Configurtion
+- VM configuration of gitchefsync [chefsync](https://gitlab.rim.net/mand-common/chefsync)
+- sync-config.json is the central config file as is required for all operations
+- relies on knife and chef in general it's best to create a .chef directory in the working directory and fill it with appropriate values, if not a .chef directory and it's corresponding knife.rb file will be created
+- The script relies on 3 important pieces: Git, Chef and Berks.  Installation options: see chefsync (https://gitlab.rim.net/mand-common/chefsync).
+- There are 2 modes of operation: pulling from master node in git and pushing those changes into chef server, OR using the working directory (which is a git repository).  Running locally can be achieved with the "sync_local flag".
+- syncCookbooks still however depends on berks upload, which for all intents and purposes requires access to both git and the internet for getting and installing cookbook dependencies.
+
+### Sample sync-config.json
+
+    {
+        "knife":"/usr/local/bin/knife",
+        "git":"/usr/bin/git",
+        "berks":"/usr/local/bin/berks",
+        "working_directory":"/working_directory",
+        "knife_file":"path_to_knife/knife.rb",
+        "gitlab_group_names": [MAND_IEMS,MAND_BBM],
+        "git_env_repo": ""https://gitlab.rim.net/mand-common/global_chef_env.git",
+        "cookbook_repo_list" : ["https://gitlab.rim.net/mand-ems/iems.git"],
+        "sync_local" : true,
+        "stage_dir": "path_to_stage_",
+        "tmp_dir" : "path_to_store_tmp_data"
+    }
+
+### Dependencies:
+- Berks: likely requires you to disable SSL if running through a proxy
+- Git - goes without saying, need git
+- Gitlab - https://gitlab.rim.net is hardcoded currently
+- knife.rb file
+
+Example of knife.rb : 
+node_name                'admin'
+client_key               '/etc/chef-server/admin.pem'
+chef_server_url          'https://yourserver'
+
+### Cookbook synchronization
+- the current "gitlab_group_names" are the gitlab groups that are the target set of repositories for synchronization
+- this can work concurrently with "cookbook_repo_list" an explicit set of git urls for those cookbooks 
+- cookbooks are vetted with berks and knife
+- requires appropriate knife configuration
+- requires that your gitlab token is set correctly
+	
+### Environment synchronization
+
+- the "git_env_group" is the repository that houses the standard configuration for environment, data bags and roles
+- chef-repo
+	- environments/
+		env.json
+	- data_bags/
+		- db_name1/
+			databag.json
+		- db_name2/
+	-roles/
+
+OR structure:
+
+- chef-repo
+	- env_identifier1/
+		- environments/
+			env.json
+		- data_bags/
+			- db_name1/
+				databag.json
+			- db_name2/
+		-roles/
+	-env_identifier2/
+		- environments/
+			env.json
+		- data_bags/
+			- db_name1/
+				databag.json
+			- db_name2/
+		-roles/
+
+See (http://gitlab.rim.net/mand-common/global_chef_env) for example of such
+
+### Audit
+- An audit file will be generated for each run of the synchronization, where each cookbook will generate a json structure of which cookbooks were deployed or staged
+- 2 sets of audit files will be generated, one for the cookbook related synchronization and another for 
+- Audit will be in a parsable json format stored in the staging directory  
+    [
+        {
+            "name": "epagent",
+            "ts": 1400702312,
+            "version": "1.0.3"
+        },
+        {
+            "name": "epagent",
+            "ts": 1400702315,
+            "version": "1.0.4"
+        },
+        
+        {
+            "name": "bb_jboss-cookbook",
+            "ts": 1400702327,
+            "version": "1.0.0"
+        },
+        {
+            "name": "relay_node-cookbook",
+            "ts": 1400702329,
+            "exception": "Some Exception msg",
+            "version": "1.0.0"
+        },
+        {
+            "name": "bb_jssecert-cookbook",
+            "ts": 1400702333,
+            "version": "1.0.0"
+        },
+        {
+            "name": "nsupdate",
+            "ts": 1400702336,
+            "version": "1.0.0"
+        },
+        {
+            "name": "jdk-certicom",
+            "ts": 1400702339
+        },
+        {
+            "name": "bb_apt-cookbook",
+            "ts": 1400702343,
+            "version": "1.0.0"
+        }
+        
+    ]

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/gitchefsync

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+require 'gitchefsync'
+
+if ARGV.length == 0
+  Gitchefsync.help
+else
+  begin
+    method = Gitchefsync.method( ARGV[0] )
+    #find method opts: stop when encountering arguments --x -y
+    
+    options = Gitchefsync.parseAndConfigure( ARGV )
+    method.call
+  rescue Gitchefsync::ConfigError => e
+    puts e.backtrace
+    Gitchefsync.help        
+  end
+end

data/build.sh

@@ -0,0 +1,20 @@
+#sudo rake install
+mkdir -p /opt/gitchefsync/working_dir
+mkdir -p /opt/gitchefsync/staging
+
+#
+#export PATH=/opt/chefdk/embedded/bin:~/.chefdk/gem/ruby/2.1.0/bin
+
+CONF=/opt/gitchefsync/config/sync-config.json
+rm pkg/*gem
+rake build
+wd=`pwd`
+gem install pkg/*gem 
+cd bin
+#
+gitchefsync syncCookbooks --private_token=LQv1BEwgo3Z4RkuKpYyb --config=$CONF
+#gitchefsync syncCookbooksLocal --private_token=LQv1BEwgo3Z4RkuKpYyb --config=$CONF
+gitchefsync syncEnv --private_token=LQv1BEwgo3Z4RkuKpYyb --config=$CONF
+
+gitchefsync stagedUpload  --private_token=LQv1BEwgo3Z4RkuKpYyb  --config=$CONF
+cd $wd

data/gitchefsync.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'gitchefsync/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "gitchefsync"
+  spec.version       = Gitchefsync::VERSION
+  spec.authors       = ["Marcus Simonsen"]
+  spec.email         = ["msimonsen@blackberry.com"]
+  spec.summary       = "Git to Chef sync"
+  spec.description   = "Tool(s) to help synchronize Git -> Chef server"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+  ##Need to figure out the right dependency versions first, otherwise errors out
+  #spec.add_runtime_dependency 'chef'
+  #spec.add_runtime_dependency 'berkshelf'
+  spec.add_runtime_dependency "gitlab", '~> 3.0.0'
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+  spec.homepage = "https://gitlab.rim.net/mandolin/gitchefsync/tree/master"
+
+  
+end

data/lib/gitchefsync/audit.rb

@@ -0,0 +1,219 @@
+require 'json'
+require 'gitchefsync/config'
+require 'gitchefsync/io_util'
+require 'gitchefsync/knife_util'
+module Gitchefsync
+
+  class Audit
+
+    def initialize (fileLocation, type = 'cb')
+      @fileLocation = fileLocation
+      @ts = Time.now.to_i
+      @list = Array.new
+      @type = type
+    end
+
+    #adds an audit item
+    def add ( item )
+      @list << item.to_hash
+    end
+
+    def addItem(name, version)
+      item = AuditItem.new(name,version)
+      add(item)
+    end
+
+    def addCookbook(cookbook,action="UPDATE",exception=nil)
+      item = AuditItem.new(cookbook.name,cookbook.version,exception,action)
+      item.setCookbook(cookbook)
+      add(item)
+    end
+
+    #This gives enough information
+    def addEnv(name,action='UPDATE',exception=nil)
+      cb = Cookbook.new(name,'','mandolin','mandolin@blackberry.com')
+      addCookbook(cb,action,exception)
+    end
+
+    #writes the audit out
+    #write out a current file audit-type-current.json
+    #and an audit-type-timestamp.json
+    def write
+      begin
+        fileLoc = @fileLocation + "/audit-" +@type+ @ts.to_s + ".json"
+        #fileCurrent = @fileLocation + "/audit-" +@type+ "-current" + ".json"
+        if @list.length > 0
+          Gitchefsync.logger.debug "event_id=write_audit:file_loc=#{fileLoc}"
+          file = File.open(fileLoc, "w")
+          json = JSON.generate(@list)
+          file.write(json)
+          #create sym link to this file
+          latest = @fileLocation+ "/audit_" + @type + "_latest.json"
+          if File.exists? latest
+            File.delete latest
+          end
+          File.symlink(file,latest)
+        else
+          Gitchefsync.logger.debug "event_id=no_write_audit"
+        end
+      rescue IOError => e
+        raise e
+      ensure
+        file.close unless file.nil?
+      end
+    end
+
+    #returns the latest audit file
+    def latest
+      entries = Dir.glob(@fileLocation + "/audit-#{@type}*")
+      file = nil
+      if entries != nil
+        file = entries.sort[entries.length-1]
+      end
+      file
+    end
+
+    #trims the oldest number of audit files to a max to to_num
+    #Additionally archives the audit file in an audit-archive.tar.gz
+    # in the fileLocation directory
+    #
+    #@param suffix - the audit suffix
+    #@param to_num
+    def trim(to_num)
+      entries = Dir.glob(@fileLocation + "/audit-#{@type}*")
+      Gitchefsync.logger.debug "#{@fileLocation}:#{@type} num audit files: #{entries.length}, keep=#{to_num}"
+      files_trimmed = 0
+      files_to_archive = Array.new
+      if entries != nil
+        #sorted in descending order (timestamp)
+        sorted = entries.sort
+        sl = sorted.length - to_num
+        if sl > 0
+          for i in 0..(sl-1)
+            #File.delete sorted[i]
+            files_to_archive << sorted[i]
+          end
+          files_trimmed = sl
+        end
+      end
+      #Archiving and cleanup
+      begin
+        if files_to_archive.length > 0
+          Gitchefsync.logger.debug "executing: tar uf #{@fileLocation}/audit-archive.tar.gz on #{flatten(files_to_archive)}"
+          FS.cmd("cd #{@fileLocation} && tar uf audit-archive.tar.gz #{flatten(files_to_archive)}")
+          Gitchefsync.logger.info "event_id=audit_archive:files=#{files_to_archive}"
+          #delete them
+          for f in files_to_archive
+            File.delete(f)
+          end
+        end
+      rescue Exception => e
+        Gitchefsync.logger.error "event_id=no_write_audit:msg=#{e.message}:trace=#{e.backtrace}"
+      end
+      Gitchefsync.logger.debug("files trimmed:#{files_trimmed}")
+      files_trimmed
+    end
+
+    def flatten(files_array)
+      str = ""
+      for f in files_array
+        str << File.basename(f) << " "
+      end
+      str
+    end
+
+    #returns json structure of the latest audit
+    def parseLatest
+      file = latest
+      if file != nil
+        json = File.read(file)
+        json = JSON.parse(json)
+      end
+      json
+    end
+
+    def latestAuditItems
+      json = parseLatest
+      if json.nil?
+        raise AuditError, "No json available"
+      end
+      audit_list = Array.new
+      json.each do |audit_item|
+        audit_list << AuditItem.new(nil,nil).from_hash(audit_item)
+      end
+      audit_list
+    end
+
+    #if the json has exceptions
+    def hasError (json_obj)
+      ret = false
+      json_obj.each do |item|
+        if item['exception'] != nil then ret = true end
+      end
+      ret
+    end
+  end
+
+  #contains functionality associated to audit information gathering
+  #TODO: just reference Cookbook clas in knife_util
+  class AuditItem
+
+    def initialize(name, version, exception = nil, action = 'UPDATE', ts = Time.now)
+      @name = name
+      @version = version
+      @ts = ts.to_i
+      @exception = exception
+      @action = action
+    end
+
+    def name
+      @name
+    end
+    def ex
+      @exception
+    end
+
+    #types are CB and ENV
+    def setType type
+      @type = type
+    end
+
+    def setCookbook(cb)
+      @cookbook = cb
+    end
+
+    #TODO action should be an enumeration
+    def setAction action
+      @action = action
+    end
+
+    #this method doesn't work when called when exception is created from json (from_hash)
+    def to_hash
+      h = Hash.new
+      h[:name] = @name
+      h[:ts] = @ts
+      if @exception.is_a? Exception
+        h[:exception] = @exception.message unless @exception == nil
+      else
+        h[:exception] = @exception unless @exception == nil
+      end
+      h[:version] = @version unless @version == nil
+      h[:type] = @type unless @type == nil
+      h[:action] = @action unless @action == nil
+      h[:maintainer] = @cookbook.maintainer unless @cookbook == nil
+      h[:maintainer_email] = @cookbook.maintainer_email unless @cookbook == nil
+      h
+    end
+
+    def from_hash(h)
+      @name = h['name']
+      @ts = h['ts']
+      @exception = h['exception']
+      @version = h['version']
+      @type = h['type']
+      @action = h['action']
+      @cookbook = Cookbook.new(@name,@version,h['maintainer'],h['maintainer_email'])
+      return self
+    end
+  end
+end

data/lib/gitchefsync/common.rb

@@ -0,0 +1,178 @@
+module Gitchefsync
+
+  extend Gitchefsync::Configuration
+  def self.included base
+    base.extend ClassMethods
+  end
+
+  #git installed?
+  def self.checkGit
+    include Git
+    if Git.hasGit == false
+      logger.error "event_id=git_error:msg=Git was not found on the path"
+      raise GitError, "Git was not detected"
+    end
+  end
+
+  def self.gitDelta(path, remote_ref)
+    include Git
+    env_repo = @config['git_env_repo']
+
+    local = Git.cmd "cd #{path} && #{@git_bin} rev-parse HEAD"
+    #logger.debug "local #{local}: path=#{path}"
+    remote = Git.cmd "cd #{path} && #{@git_bin} ls-remote origin #{remote_ref}"
+    return false if remote.empty?
+    remote = remote.split(/\s/)[0]
+
+    delta = (local.chomp != remote.chomp)
+    logger.debug "event_id=gitDelta:local=#{local.chomp}:remote=#{remote.chomp}:delta=#{delta}"
+    delta
+  end
+
+  # Verify .gitchefsync at HEAD of default_branch
+  def self.checkProjectConfig(cmd, log)
+    begin
+      proc_cmd, cmd_line = cmd
+      return proc_cmd.call(cmd_line)
+    rescue GitError, CmdError
+      proc_log, msg = log
+      proc_log.call(msg)
+    end
+    return
+  end
+ 
+  #@param project - Gitlab project object
+  def self.pullProject(project, verify_yml=false)
+    p_name = project['path_with_namespace'].split('/').join('_')
+    project_path = File.join(@git_local, p_name)
+
+    default_branch =  project['default_branch']
+
+    url_type = @config['gitlab_url_type']
+    # "http" is default if @config['gitlab_url_type'] not configured
+    url_type ||= "http"
+
+    if url_type.eql?("http")
+      project_url = project['http_url_to_repo']
+      # Verify .gitchefsync at HEAD of default_branch using `wget -O - url`
+      cmd_line = "wget -qO - #{project['web_url']}/raw/#{default_branch}/.gitchefsync.yml"
+      cmd = [ Proc.new{ |cmd_line| FS.cmd cmd_line }, cmd_line ]
+
+    elsif url_type.eql?("ssh")
+      project_url = project['ssh_url_to_repo']
+      # Verify .gitchefsync at HEAD of default_branch using `git archive` (not currently supported/enabled using https protocol)
+      cmd_line = "git archive --remote=#{project_url} #{default_branch}: .gitchefsync.yml"
+      cmd = [ Proc.new { |cmd_line| Git.cmd("#{cmd_line}") }, cmd_line ]
+    end
+
+    msg = "event_id=project_missing_.gitchefsync.yml:project_url=#{project_url}:default_branch=#{default_branch}"
+    proc_log = Proc.new { |msg| logger.info "#{msg}" }
+    log = [ proc_log, msg ]
+
+    if verify_yml
+      check = checkProjectConfig(cmd, log)
+      if check.nil? || check.empty?
+        proc_log.call(msg)
+        return
+      end
+    end
+
+    begin
+      self.updateGit(project_path, project_url)
+    rescue GitError => e
+      logger.error "event_id=git_error:msg=#{e.message}:trace=#{e.backtrace}"
+      logger.error "event_id=remove_project_path: #{project_path}"
+      FS.cmd "rm -rf #{project_path}"
+    end
+  end
+ 
+  #central place to get cookbooks from server
+  #will be determined once - and will be (eventually) thread safe
+  def self.serverCookbooks
+    if @serverCookbooks.nil?
+      knifeUtil = KnifeUtil.new(@knife, @git_local)
+      @serverCookbooks = knifeUtil.listCookbooks()
+    end
+    @serverCookbooks
+  end
+
+  # Pulls all known projects (determined by configured gitlab-token)
+  def self.pullAllProjects
+    all_projects = []
+    done = false
+    page, per_page = 1, 100
+    while !done do 
+      page_opts = { :per_page => per_page, :page => page}
+      repos = Gitlab.projects(page_opts)
+      if (repos.length == 0)
+        done = true
+      else 
+        page += 1
+      end
+      repos.each do |project|
+        pullProject(project.to_hash, true)
+      end
+    end
+  end
+
+  # Get subset of known groups (determined by configured gitlab-token)
+  def self.getAllGroupIDs(group_names=[], group_ids=[])
+    groups = Array.new
+    done = false
+    page, per_page = 1, 100
+    while !done do
+      page_opts = { :per_page => per_page, :page => page}
+      known_groups = Gitlab.groups(page_opts)
+      if (known_groups.length == 0)
+        done = true
+      else
+        page += 1
+      end
+      known_groups.each do |group|
+        name = group.to_hash['name']
+        id = group.to_hash['id']
+        if (group_names.include? name) || (group_ids.include? id)
+          groups << id
+        end
+      end
+    end
+    groups.uniq!
+    groups
+  end
+
+  #TODO: change from git to git with path
+  def self.updateGit (project_path, git_path)
+    include Git
+    begin
+      branch = @rel_branch
+      logger.debug "using release branch: #{branch}"
+      _git = @git_bin
+      if !Git.gitInit(project_path)
+        FS.cmd "mkdir #{project_path}"
+        logger.debug "event_id=git_int:path=#{git_path}:project_path=#{project_path}"
+        Git.cmd "cd #{project_path} && #{_git} init"
+        Git.cmd "cd #{project_path} && #{_git} remote add origin #{git_path}"
+        Git.cmd "cd #{project_path} && #{_git} pull origin #{branch}"
+      else
+        logger.info "event_id=git_repo_exists:project_path=#{project_path}"
+        #wipe tags and re-fetch them
+        tags = Git.cmd "cd #{project_path} && git tag"
+        arr_tags = tags.split(/\n/)
+        arr_tags.each do |tag|
+          Git.cmd "cd #{project_path} && git tag -d #{tag}"
+        end
+      end
+
+      #get all commits and tags
+      Git.cmd "cd #{project_path} && #{_git} clean -xdf"
+      Git.cmd "cd #{project_path} && #{_git} checkout #{branch}"
+      Git.cmd "cd #{project_path} && #{_git} pull origin #{branch}"
+
+      git_tags = Git.cmd "cd #{project_path} && #{_git} fetch --tags"
+
+    rescue CmdError => e
+      raise GitError, "An error occurred synchronizing cookbooks #{e.message}"
+    end
+  end
+
+end