berktacular 0.1.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NzNiYWFhMGQ4YTY3ZTM5Zjg3Y2E1MDRjYjA2MWY2Yzg5OTk2MWIwNQ==
+  data.tar.gz: !binary |-
+    ZjI0MmRhODJmYmM1YWVmNjNiYjE4YTY5Y2EzMGVhNDQwNDkyODBjNg==
+SHA512:
+  metadata.gz: !binary |-
+    YjI2ZWIyZGFmZjgxYWY1ZDEzYjZlMDlmYzk4NjRkNDNiMTk1MTMzODc0ZDU2
+    NTgzODU1NmM1Y2JjNTk1ZmI3YTc4ODhkMThjNWQ0NWNjZmMxNDFkMDZhM2Qy
+    OTFkZGRmMzRmYTYzZTk4MWE3NDE1OWY0ODViYzdjYmQ3N2QyNDg=
+  data.tar.gz: !binary |-
+    ODcxYmUwODZlZjNkM2M0YjkxZmIwMDkxYTJhNmI0ZDMyYWUxM2VhYWI0ZDg3
+    YzcyNTcwMjBmOWEwYjAwY2VmY2Y5YzA4MmIyN2ViNjBiZTRkYTRlN2I1NWIw
+    MWQwNGE3YmMxZGVhMGUwNTRkZGExN2QzNzdkZmIyOTI2ZjA2MDQ=

data/bin/berktacular

@@ -0,0 +1,155 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'fileutils'
+require 'berktacular'
+
+env_file    = nil
+check       = false
+upgrade     = false
+gtoken_path = File.join(ENV['HOME'], '.github-token')
+github_token= nil
+printit     = false
+berksfile   = nil
+new_envfile = nil
+preserve    = false
+verify      = nil
+upload      = false
+berks_conf  = nil
+knife_conf  = nil
+workdir     = nil
+verbose     = false
+
+options =  OptionParser.new do |opts|
+  opts.banner = "Read environment json file and spit out a berksfile"
+  opts.on("-e ENV.json", "--environment ENV.json", String, "Path to the environment file to use.  Required.") do |f|
+    env_file = f
+  end
+  opts.on("-c", "--check", "Check for updates and print what they would be") do |c|
+    check = true
+  end
+  opts.on("-u", "--upgrade", "Auto-upgrade cookbooks that support it") do |u|
+    upgrade = true
+  end
+  opts.on("-p", "--print", "Print the berksfile to stdout") do |p|
+    printit = true
+  end
+  opts.on("-b PATH", "--berksfile PATH", String, "Write the berksfile to PATH.") do |b|
+    berksfile = b
+  end
+  opts.on("-n PATH", "--new-envfile PATH", String,  "Write a new (updated) env file to PATH.") do |n|
+    new_envfile = n
+  end
+  opts.on("--[no-]verify", "Install cookbooks to WORKDIR and confirm that all dependancies are met") do |v|
+    verify = v
+  end
+  opts.on("--upload", "Upload the cookbooks and updated env file to the chef-server") do
+    upload = true
+  end
+  opts.on("--berksconfig BERKS_CONF", "The berkshelf config file to use.  Defaults to ~/.chef/<env_name>-berkshelf.json") do |k|
+    berks_conf = k
+  end
+  opts.on("--knifeconfig KNIFE_CONF", "The knife config file to use.  Defaults to ~/.chef/<env_name>-knife.rb") do |k|
+    knife_conf = k
+  end
+  opts.on("--github-token-file FILE", String, "Path to the github token to use") do |g|
+    gtoken_path = g
+  end
+  opts.on("-g TOKEN", "--github-token TOKEN", String, "The github token to use, not the path.  This will show up in ps output") do |t|
+    github_token = t
+  end
+  opts.on("-w PATH","--workdir PATH", String, "The working directory to use, it will be created if need be.",
+                                              "The working directory is used to test the cookbook install.",
+                                              "Using this option prevents berktacular from deleting the workdir when done.",
+                                              "Using a tmpfs filesystem is recommended (used by default)",
+                                              "Default is a tmpdir somewhere (probably /tmp or /dev/shm)") do |w|
+    workdir = w
+    preserve = true
+  end
+  opts.on("-v", "--verbose", "Turn on verbose output" ) do
+    verbose = true
+  end
+  opts.on_tail("-h", "--help", "Print this help message" ) do
+    puts opts
+    exit 0
+  end
+  begin
+    opts.parse!
+  rescue OptionParser::ParseError, OptionParser::MissingArgument
+    warn opts
+    exit 1
+  end
+end
+# Enable verify by default if uploading, unless the user has surpressed it.
+verify ||= upload
+
+require 'json'
+require 'solve'
+
+unless env_file && File.exist?(env_file)
+  warn "No env file found at: '#{env_file}'"
+  warn options
+  exit 2
+end
+
+my_env = JSON.parse( File.read( env_file ) )
+
+unless github_token
+  if File.exists? gtoken_path
+    github_token = IO.read(gtoken_path).strip
+  elsif upgrade || check
+    warn "No github token, cannot continue"
+    warn options
+    exit 3
+  end
+end
+
+if upload
+  berks_conf ||= "#{File.join(ENV['HOME'], File.basename(env_file, ".json") + "-berkshelf.json" )}"
+  unless File.exists? berks_conf
+    warn "The berkshelf config file could not be found at '#{berks_conf}'"
+    warn options
+    exit 4
+  end
+  knife_conf ||= "#{File.join(ENV['HOME'], File.basename(env_file, ".json") + "-knife.rb" )}"
+  unless File.exists? knife_conf
+    warn "The knife config file could not be found at '#{knife_conf}'"
+    warn options
+    exit 5
+  end
+end
+
+# Create a new berksfile
+puts "Checking updates, this can take some time..." if check || upgrade
+b = Berktacular::Berksfile.new(my_env, upgrade: upgrade, github_token: github_token, verbose: verbose)
+b.check_updates           if check
+puts "#{b}"               if printit
+if berksfile
+  FileUtils.mkdir_p( File.dirname(berksfile) )
+  File.write(berksfile, b)
+end
+if new_envfile
+  FileUtils.mkdir_p( File.dirname(new_envfile))
+  File.write(new_envfile, JSON.pretty_generate(b.env_file))
+end
+
+if verify
+  unless b.verify(workdir)
+    puts "Berksfile verification failed"
+    b.missing_deps.each do |name, msg|
+      puts "#{name}\n\t#{msg}"
+    end
+    exit 10
+  end
+  puts "Berksfile Verified"
+end
+
+if upload
+  unless b.upload(knife_conf, workdir)
+    puts "Upload failed!"
+    exit 11
+  end
+  puts "Cookbooks and new environment file have been uploaded to the server"
+end
+
+b.clean unless preserve

data/lib/berktacular.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'berktacular/version'
+
+# This module contains classes that allow for generating Berksfiles from chef environment files.
+module Berktacular
+  
+  # @param h [Object] does a deep copy of whatever is passed in.
+  # @return [Object] a deep copy of the passed in object.
+  def self.deep_copy(h)
+    Marshal.load(Marshal.dump(h))
+  end
+
+  # @param [String] a command to run.
+  # @return [True] or raise on failure.
+  def self.run_command(cmd)
+    puts "Running command: #{cmd}"
+    unless system(cmd)
+      raise "Command failed with exit code #{$?.exitstatus}: #{cmd}"
+    end
+  end
+
+  # @return [String] the best tmpdir to use for this machine.  Prefers /dev/shm if available.
+  def self.best_temp_dir
+    require 'tempfile'
+    tmp = if File.directory?("/dev/shm") && File.writable?("/dev/shm")
+      '/dev/shm'
+    else
+      '/tmp'
+    end
+    pat = [
+      Time.now().strftime('%Y_%m_%d-%H.%M.%S_'),
+      '_berktacular'
+    ]
+    Dir.mktmpdir(pat, tmp)
+  end
+
+  # Matches the numric version information from a tag.
+  VERSION_RE = Regexp.new(/\d+(?:\.\d+)*/)
+
+  autoload  :Cookbook,  'berktacular/cookbook'
+  autoload  :Berksfile, 'berktacular/berksfile'
+
+end

data/lib/berktacular/berksfile.rb

@@ -0,0 +1,199 @@
+module Berktacular
+
+  # This class represents a Berksfile
+
+  class Berksfile
+
+    # @!attribute [r] name
+    #   @return [String] the name of the environment.
+    # @!attribute [r] description
+    #   @return [String] a description of the enviroment.
+    # @!attribute [r] installed
+    #   @return [Hash] a hash of installed cookbook directories.
+    # @!attribute [r] missing_deps
+    #   @return [Hash] a hash of cookbooks missing dependencies after calling verify.
+    attr_reader :name, :description, :installed, :missing_deps
+
+    # Creates a new Berksfile from a chef environment file.
+    #
+    # @param environment [Hash] a parsed JSON chef environment config.
+    # @option opts [String] :github_token (nil) the github token to use.
+    # @option opts [True,False] :upgrade (False) whether or not to check for upgraded cookbooks.
+    # @option opts [True,False] :verbose (False) be more verbose.
+    def initialize( environment, opts = {})
+      @env_hash           = environment # Save the whole thing so we can emit an updated version if needed.
+      @name               = environment['name']               || nil
+      @description        = environment['description']        || nil
+      @cookbook_versions  = environment['cookbook_versions']  || {}
+      @cookbook_locations = environment['cookbook_locations'] || {}
+      @opts = {
+        :upgrade      => opts.has_key?(:upgrade)      ? opts[:upgrade]      : false,
+        :github_token => opts.has_key?(:github_token) ? opts[:github_token] : nil,
+        :verbose      => opts.has_key?(:verbose)      ? opts[:verbose]      : false
+      }
+      @installed = {}
+      # only connect once, pass the client to each cookbook.  and only if needed
+      connect_to_git if @opts[:upgrade]
+    end
+
+    # @return [Hash] representation of the env_file.
+    def env_file
+      if @opts[:upgrade]
+        cookbooks.each do |book|
+          @env_hash['cookbook_versions'][book.name] = book.version_specifier
+        end
+      end
+      @env_hash
+    end
+
+    # @return [String] representation of the env_file in pretty json.
+    def env_file_json
+      if @opts[:upgrade]
+        cookbooks.each do |book|
+          @env_hash['cookbook_versions'][book.name] = book.version_specifier
+        end
+      end
+      JSON.pretty_generate(@env_hash)
+    end
+
+    # @param workdir [String] the directory in which to install.  If nil, Berktacular.best_temp_dir is used.
+    # @return [String] the directory path where the cookbooks were installed.
+    def install(workdir = nil)
+      if workdir
+        FileUtils.mkdir_p(workdir)
+      else
+        workdir = Berktacular.best_temp_dir
+      end
+      unless @installed[workdir]
+        # remove the Berksfile.lock if it exists (it shouldn't).
+        berksfile = File.join(workdir, "Berksfile")
+        lck       = berksfile + ".lock"
+        FileUtils.rm(lck) if File.exists? lck
+        File.write(berksfile, self)
+        Berktacular.run_command("berks install --berksfile #{berksfile} --path #{workdir}")
+        @installed[workdir] = {berksfile: berksfile, lck: lck}
+      end
+      workdir
+    end
+
+    # @params workdir [String] the directory in which to install.  If nill, Berktacular.best_temp_dir is used.
+    # @return [True,False] the status of the verify.
+    def verify(workdir = nil)
+      require 'ridley'
+      @missing_deps = {}
+      workdir       = install(workdir)
+      versions      = {}
+      dependencies  = {}
+      Dir["#{workdir}/*"].each do |cookbook_dir|
+        next unless File.directory?(cookbook_dir)
+        metadata_path   = File.join(cookbook_dir, 'metadata.rb')
+        metadata        = Ridley::Chef::Cookbook::Metadata.from_file(metadata_path)
+        cookbook_name   = metadata.name
+        name_from_path  = File.basename(cookbook_dir)
+        unless cookbook_name == name_from_path
+          if cookbook_name.empty?
+            puts "Cookbook #{name_from_path} has no name specified in metadata.rb"
+            cookbook_name = name_from_path
+          else
+            warn "Cookbook name from metadata.rb does not match the directory name!",
+                 "metadata.rb: '#{cookbook_name}'",
+                 "cookbook directory name: '#{name_from_path}'"
+          end
+        end
+        versions[cookbook_name] = metadata.version
+        dependencies[cookbook_name] = metadata.dependencies
+      end
+      errors = false
+      dependencies.each do |name, deps|
+        deps.each do |dep_name, constraint|
+          actual_version = versions[dep_name]
+          if !actual_version
+            @missing_deps[name] = "#{name}-#{versions[name]} depends on #{dep_name} which was not installed!"
+            warn @missing_deps[name]
+            errors = true
+          elsif !Solve::Constraint.new(constraint).satisfies?(actual_version)
+            @missing_deps[name] = "#{name}-#{versions[name]} depends on #{dep_name} #{constraint} but #{dep_name} is #{actual_version}!"
+            warn @missing_deps[name]
+            errors = true
+          end
+        end
+      end
+      !errors  
+    end
+
+    # @param berks_conf [String] path to the berkshelf config file to use.
+    # @param knife_conf [String] path to the knife config file to use.
+    # @param workdir [String] Path to use as the working directory.
+    #   @default Berktacular.best_temp_dir
+    # @return [True] or raise on error.
+    def upload(berks_conf, knife_conf, workdir=nil)
+      raise "No berks config, required for upload" unless berks_conf && File.exists?(berks_conf)
+      raise "No knife config, required for upload" unless knife_conf && File.exists?(knife_conf)
+      workdir       = install(workdir)
+      new_env_file  = File.write(File.join(workdir, @name + ".rb"), env_file_json )
+      Berktacular.run_command("berks upload --berksfile #{@installed[workdir][:berksfile]} --c #{berks_conf}")
+      Berktacular.run_command("knife environment from file #{new_env_file} -c #{knife_conf}")
+    end
+
+    # param workdir [String,nil] the workdir to remove.  If nil, remove all installed working directories.
+    def clean(workdir = nil)
+      if workdir
+        Fileutils.rm_r(workdir)
+        @installed.delete(workdir)
+      else
+        # clean them all
+        @installed.keys.each { |d| FileUtils.rm_r(d) }
+        @installed = {}
+      end
+    end
+
+    # @param [IO] where to write the data.
+    def print_berksfile( io = STDOUT )
+      io.puts to_s  
+    end
+
+    # @return [String] the berksfile as a String object
+    def to_s
+      str = ''
+      str << "# Name: '#{@name}'\n" if @name
+      str << "# Description: #{@description}\n\n" if @description
+      str << "# This file is auto-generated, changes will be overwritten\n"
+      str << "# Modify the .json environment file and regenerate this Berksfile to make changes.\n\n"
+
+      str << "site :opscode\n\n"
+      cookbooks.each { |l| str << l.to_s << "\n" }
+      str
+    end
+
+    # @return [Array] a list of Cookbook objects for this environment.
+    def cookbooks
+      @cookbooks ||= @cookbook_versions.sort.map do |book, version|
+        Cookbook.new(book, version, @cookbook_locations[book], @opts ) 
+      end
+    end
+
+    # print out the cookbooks that have newer version available on github.
+    def check_updates
+      connect_to_git
+      cookbooks.each do |b|
+        candidates = b.check_updates
+        next unless candidates.any?
+        puts  "Cookbook: #{b.name} (auto upgrade: #{b.auto_upgrade ? 'enabled' : 'disabled'})",
+              "\tCurrent:#{b.version_number}",
+              "\tUpdates: #{candidates.join(", ")}"
+      end
+    end
+
+    private
+
+    # connect to github using the token in @opts[:github_token].
+    # @return [Octokit::Client] a connected github client.
+    def connect_to_git
+      raise "No token given, can't connect to git" unless @opts[:github_token]
+      puts "Connecting to git with supplied github_token" if @opts[:verbose]
+      require 'octokit'
+      @opts[:git_client] ||= Octokit::Client.new(access_token: @opts[:github_token])
+    end
+
+  end
+end

data/lib/berktacular/cookbook.rb

@@ -0,0 +1,115 @@
+module Berktacular
+
+  # This class represents a cookbook entry form a Berksfile
+
+  class Cookbook
+
+    # @!attribute [r] name
+    #   @return [String] the name of the cookbook.
+    # @!attribute [r] version_number
+    #   @return [String] the exact version of the cookbook.
+    # @!attribute [r] auto_upgrade
+    #   @return [True, False] whether or now this cookbook can autoupgrade.
+    # @!attribute [r] config
+    #   @return [Hash, nil] the cookbook_location hash associated with this cookbook or nil
+    attr_reader :name, :version_number, :auto_upgrade, :config
+
+    # Creates a new cookbook entry for a Berksfile.
+    #
+    # @param name [String] the name of the cookbook to use.
+    # @param version_spec [String] the exact version number as in a chef environment file. eg. '= 1.2.3'
+    # @param config [Hash,nil] the cookbook_location hash to for this cookbook.  Optional.
+    # @option opts [Octokit::Client] :git_client (nil) the github client to use.
+    # @option opts [True,False] :upgrade (False) whether or not to check for updates.  auto_upgrade must also be enbaled for the updated entry to be used.
+    # @option opts [True,False] :verbose (False) be more vervose.
+    def initialize( name, version_spec, config = nil, opts = {} )
+      @name             = name          || raise( "Missing cookbook name" )
+      @version_spec     = version_spec  || raise( "Missing cookbook version" )
+      @version_number   = VERSION_RE.match( version_spec )[0]
+      @version_solved   = Solve::Version.new(@version_number)
+      @auto_upgrade     = config && config['auto_upgrade']  || false
+      @versions         = config && config['versions']      || {}
+      @config           = config ? config.reject{ |k,v| k == 'auto_upgrade' || k == 'versions' } : nil
+      @upgrade          = opts.has_key?(:upgrade)      ? opts[:upgrade]         : false
+      @git_client       = opts.has_key?(:git_client)   ? opts[:git_client].dup  : nil
+      @verbose          = opts.has_key?(:verbose)      ? opts[:verbose]         : false
+      check_updates if @auto_upgrade && @upgrade
+    end
+
+    # @return [String] the exact version of the cookbook
+    def version_specifier
+      "= #{(@auto_upgrade && @upgrade && check_updates.any?) ? check_updates.first : @version_number }"
+    end
+
+    # @return [String] the latest available version number of the cookbook
+    def latest_version
+      check_updates.any? ? check_updates.first : @version_number
+    end
+
+    # @return [String] a Berksfile line for this cookbook
+    def to_s
+      line
+    end
+
+    # param upgrade [True,False] ('@upgrade') whether or not to force the lastest version when @auto_update is enabled
+    # @return [String] a Berksfile line for this cookbook
+    def line(upgrade = @upgrade)
+      "cookbook \"#{@name}\", #{generate_conf_line(upgrade, @config )}"
+    end
+
+    # @return [Array] a list of available cookbook version newer then we started with, with most recent first
+    def check_updates
+      @candidates ||= if @config && @config['github']
+        get_tags_from_github
+      else
+        []
+      end.select do |tag|
+        next if @config.has_key?('rel') && ! /^#{@name}-[v\d]/.match(tag)
+        m = VERSION_RE.match(tag)
+        next unless m
+        v = m[0]
+        begin
+          t = Solve::Version.new(v)
+        rescue Solve::Errors::InvalidVersionFormat
+          next
+        end
+        t > @version_solved
+      end.sort.reverse
+    end
+
+    private
+    
+    # @param upgrade [True,False] use updated cookbook version if @auto_update is also true.
+    # @param config [Hash] the cookbook_locations hash associated with this cookbook.
+    # @return [String] the config line for this cookbook, everything after the cookbook name.
+    def generate_conf_line(upgrade, config)
+      ver = (upgrade && @candidates && @candiates.first) || @version_number
+      line = []
+      if config
+        if config.has_key?('github')
+          line << "github: \"#{config['github']}\""
+          line << "rel: \"#{config['rel']}\"" if config.has_key?('rel')
+          line << 'protocol: :ssh'
+        end
+        if @versions.has_key?(ver)
+          line << "ref: \"#{@versions[ver]['ref']}\""
+        else
+          if !@config.has_key?('tag')
+            line << "tag: \"#{ver}\""
+          else
+            line << "tag: \"#{@config['tag']}\""
+          end
+        end
+      else
+        line << " \"#{ver}\""
+      end
+      line.join(", ").gsub('%{version}', ver)
+    end
+    
+    # return [Array] a list of tags from the github repository of this cookbook.
+    def get_tags_from_github
+      @git_client.repo(@config['github']).rels[:tags].get.data.map { |obj| obj.name }
+    end
+
+  end
+end

data/lib/berktacular/version.rb

@@ -0,0 +1,4 @@
+module Berktacular
+  # the gem version.
+  VERSION = '0.1.1'
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: berktacular
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Jeff Harvey-Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: solve
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.8.2
+- !ruby/object:Gem::Dependency
+  name: ridley
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.3
+- !ruby/object:Gem::Dependency
+  name: octokit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+description: Generates a Berksfile from JSON style Chef environment files.  Also support
+  extension to environment file with 'cookbook_locations'. Verifies the Berksfile
+  is consistent (all dependencies met) and will upload updated cookbooks and env files
+  to a chef server.
+email:
+- jeff@clearstorydata.com
+executables:
+- berktacular
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/berktacular
+- lib/berktacular.rb
+- lib/berktacular/berksfile.rb
+- lib/berktacular/cookbook.rb
+- lib/berktacular/version.rb
+homepage: https://rubygems.org/gems/berktacular
+licenses:
+- Apache License, Version 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ~>
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Parse chef env files, generates a Berksfile and verifies it.
+test_files: []
+has_rdoc: