conan_deploy 0.0.2

data/lib/conan/manifest_builder.rb

@@ -0,0 +1,494 @@
+require 'rest_client'
+require 'rexml/document'
+require 'properties-ruby'
+require 'json'
+require 'securerandom'
+
+require 'conan/output'
+require 'conan/repository'
+require 'conan/stackato'
+require 'conan/newrelic'
+
+module ManifestBuilder
+  def self.build(options, pipeline_id, env_id, artifact_repo=nil, &block)    
+    manifest_dir = options[:directory]
+    output_type = options[:format]
+
+
+    artifact_repo ||= DefaultArtifactRepository.new
+
+    # this is a feature toggle to allow the current property file based impl to be migrated to the stackato output impl
+    if (output_type == :stackato)
+      outthingy = StackatoOutThingy.new(manifest_dir)
+    else
+      outthingy = PropertiesFileOutThingy.new(manifest_dir)
+    end
+    
+    manifest = Manifest.new(options, artifact_repo, outthingy)
+    manifest.instance_eval(&block)
+    manifest.select(pipeline_id, env_id)
+    manifest
+  end
+end
+
+class Manifest
+
+  def initialize(options, artifact_repo, output)    
+    @pipelines = {}
+    @environments = {}
+    @options = options
+    @artifact_repo = artifact_repo
+    @output = output   
+  end
+
+  # select a pipeline and environment to do operations on
+  def select(pipeline_id, env_id)
+    env = @environments[env_id]
+    raise ArgumentError.new "Invalid environment id: '#{env_id}'. Valid options are #{@environments.keys.join(", ")}" if env.nil?
+    @current_environment = env
+    pl = @pipelines[pipeline_id]
+    raise ArgumentError.new "Invalid pipeline id: '#{pipeline_id}'. Valid options are #{@pipelines.keys.join(", ")}" if pl.nil?
+    @current_pipeline = pl
+    pl.load!(env, @options[:'deploy-shipcloud'])
+  end
+
+  def pipeline(pipeline_id, &block)
+    pl = Pipeline.new(pipeline_id, @artifact_repo, @output)
+    pl.instance_eval(&block)
+    @pipelines[pipeline_id] = pl
+    pl
+  end
+
+  def environment(env_id, &block)
+    e = Environment.new(env_id)
+    e.instance_eval(&block)
+    @environments[env_id] = e
+    e
+  end
+
+  def appVersion(pipeline_id, app_id)
+    @pipelines[pipeline_id].appVersion(app_id)
+  end
+
+  def provision
+    validateThatEnvironmentIsSelected
+    # TODO banners
+    puts "Update #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.update(@current_environment, @environments[@current_environment.upstream_env])
+  end
+
+  def configure
+    validateThatEnvironmentIsSelected
+    puts "Configure #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.configure(@current_environment)
+  end
+
+  def bg_configure
+    validateThatEnvironmentIsSelected
+    puts "Blue/green configure #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.configure(@current_environment, true)
+  end
+
+  def paas
+    paas_user = @options[:'paas-user'] || ENV['PAAS_USER']
+    paas_pwd = @options[:'paas-password'] || ENV['PAAS_PASSWORD']  
+    dry_run = @options[:'dry-run'] || false
+    trace = @options[:'verbose'] || false
+    
+    Stackato.new(paas_user, paas_pwd, trace, dry_run)
+  end
+
+  def deploy
+    validateThatEnvironmentIsSelected
+    puts "Deploy #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    nr_api_key = @options[:'new-relic-api-key'] || ENV['NEWRELIC_API_KEY']
+    force = @options[:'force-deploy'] || false
+
+    @current_pipeline.deploy(@current_environment, paas, force) { |app_name| 
+      if (nr_api_key)
+        puts 'Reporting Deployment to New Relic'
+        job = @options[:'job-url'] || 'nexus-app-manifest'
+        NewRelic.alertDeployment(nr_api_key, app_name, "Deployed by #{job}")
+      else
+        puts "WARNING: Skipping New Relic alert. No API defined"
+      end
+    }
+  end
+
+  def bg_deploy
+    validateThatEnvironmentIsSelected
+    puts "Blue/green deploy #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    force = @options[:'force-deploy'] || false
+
+    @current_pipeline.bg_deploy(@current_environment, paas, force)
+  end
+
+  def bg_switch
+    validateThatEnvironmentIsSelected
+    puts "Blue/green switch #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    @current_pipeline.bg_switch(@current_environment, paas)
+  end
+
+  def bg_clean
+    validateThatEnvironmentIsSelected
+    puts "Blue/green clean #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    @current_pipeline.bg_clean(@current_environment, paas)
+  end
+
+  def validateThatEnvironmentIsSelected
+    unless @current_environment && @current_pipeline
+      raise ScriptError, 'Environment and Pipeline must be selected'
+    end
+  end
+end
+
+class Pipeline
+  attr_accessor :id, :apps
+
+  def initialize(id, artifact_repo, output)
+    @id = id    
+    @artifact_repo = artifact_repo    
+    @output = output
+    @apps = {}    
+  end
+
+  def app(app_id, project_name, platform_type)
+    @apps[app_id] = case platform_type
+      when :jvm
+        JvmApp.new(app_id, project_name, @artifact_repo)
+      when :rails_zip
+        RailsZipApp.new(app_id, project_name, @artifact_repo)
+      else
+        raise "unknown platform type: #{platform_type}"
+      end     
+  end
+
+  def appVersion(app_id)
+    @apps[app_id].version
+  end  
+
+  def load!(environment, deploy_shipcloud)
+    eachAppDeployment(environment) { |app, deploy| 
+      # load the persisted meta-data
+      @output.loadAppMetadataFromDeploymentIfExists(app, deploy)
+    }
+    @deploy_shipcloud = deploy_shipcloud
+  end
+
+  def update(environment, from_upstream_env)
+    @apps.values.each { |app| 
+      puts ''
+      # promote versions from an upstream deploy
+      if (from_upstream_env)        
+        # TODO: this needs improving
+        puts "Promote #{app.id} from #{from_upstream_env.id} to #{environment.id}"
+        upstream_deploy = from_upstream_env.deployments.select { |d| d.app_ids.select {|x| x == app.id }.size > 0 }.first             
+        @output.loadAppMetadataFromDeployment(app, upstream_deploy)
+      else
+        puts "Find latest #{app.artifact_id}"
+        # lookup most recent from repo
+        app.findLatestVersion()
+      end
+      @output.clean(app, environment)
+      # download artifacts
+      @output.provision(app) 
+    }
+
+    # write out deployment artifact meta-data
+    eachAppDeployment(environment) { |app, deploy| @output.writeArtifactMetadata(app, deploy) } 
+  end  
+
+  def configure(environment, bg=false)    
+    eachAppDeployment(environment) { |app, deploy| @output.writeConfiguration(app, deploy, bg) }
+  end
+
+  def deploy(environment, paas, force=false)
+    eachAppDeployment(environment) { |app, deploy|
+      work_dir = @output.workingDir(app)
+      if (force)
+        paas.deploy(work_dir, app, deploy, true)
+      else
+        paas.deploy(work_dir, app, deploy) unless app.isDeployedAt(deploy.manifest_url(app.id))
+      end
+    }  
+  end
+
+  def bg_deploy(environment, paas, force=false)
+    eachAppDeployment(environment) { |app, deploy|
+      if (force or !app.isDeployedAt(deploy.manifest_url(app.id)))        
+        paas.bg_deploy(@output.workingDir(app), app, deploy)
+      end
+    }  
+  end
+
+  def bg_switch(environment, paas)
+    eachAppDeployment(environment) { |app, deploy| paas.bg_switch(app, deploy) }
+  end
+
+  def bg_clean(environment, paas)
+    eachAppDeployment(environment) { |app, deploy| paas.bg_clean(app, deploy) }
+  end
+
+  def eachAppDeployment(environment)
+    selected_deployments = environment.deployments.select{ |d| @deploy_shipcloud.nil? || d.shipcloud == @deploy_shipcloud }
+    puts "WARNING no deploys selected: #{@deploy_shipcloud}" if selected_deployments.empty?
+    selected_deployments.each { |d|      
+      d.app_ids.each { |a| 
+        app = @apps[a]
+        yield app, d  if app
+      }
+    } 
+  end
+end
+
+class Application
+
+  attr_accessor :id, :platform_type, :group_id, :artifact_id, :version, :extension, :sha1, :artifact_meta_data
+
+  def initialize(id, project_name, platform_type)
+    @id = id
+    @platform_type = platform_type  
+    mvn_id = project_name.split(':')
+    @group_id = mvn_id[0]
+    @artifact_id = mvn_id[1] 
+  end
+
+  def findLatestVersion    
+    extension = case platform_type
+      when :jvm
+        "jar"
+      when :rails_zip
+        "tar.gz"
+      else
+        raise "Unsupported platform type: #{platform_type}"
+      end  
+    readArtifactMetadata(@artifact_repo.resolveArtifact(group_id, artifact_id, extension))
+    puts "- Found latest #{@id} artifact\n"+
+      "  Artifact: #{self}\n" +
+      "  Checksum: #{@sha1}"    
+  end
+
+  def readArtifactMetadata(xml)
+    @artifact_meta_data = xml    
+    begin
+      doc = REXML::Document.new(xml)  
+    rescue Exception => e
+      puts "Failed to parse meta-data: #{e}"
+      puts "--------------------------------"
+      puts xml
+      puts "--------------------------------"
+      raise "Unreadable artifact meta-data for #{self}"
+    end
+    
+    meta_data = doc.elements['/artifact-resolution/data']
+    @version = meta_data.elements['version'].text
+    @sha1 = meta_data.elements['sha1'].text
+    @extension = meta_data.elements['extension'].text
+  end  
+
+  def download(location)
+    puts "- Provision #{group_id}:#{artifact_id}:#{extension}:#{version}"
+    @artifact_repo.downloadArtifact(location, group_id, artifact_id, extension, version, sha1)    
+  end
+
+  def isDeployedAt(url)
+    # only deploy if an older version is found, or no app found          
+    res = false
+    begin
+      response = RestClient::Request.new(
+        :method => :get,
+        :url => url,
+        :headers => { 
+          'accept' => :json,
+          'content_type' => :json
+        },
+        :timeout => 5,
+        :open_timeout => 5
+      ).execute
+      if (response.code == 200)
+        build_metadata = JSON.parse(response.to_str)
+        puts ''
+        puts "+- Currently at #{url}:"
+        printM = ->(s) {
+          puts " | #{s}: #{build_metadata[s]}"
+        }            
+        printM.call 'Implementation-Title'
+        printM.call 'Build-Version'
+        printM.call 'Build-Url'
+        printM.call 'Git-Commit'            
+        current_ver = build_metadata["Build-Version"]
+      end
+      #TODO make sure the artifact id matches too, currently finding  "Implementation-Title": "apollo" in build metadata
+      if (compareVersion(@version, current_ver))
+        puts "#{@artifact_id} #{@version} is newer than version found at #{url}: #{current_ver}"
+      else
+        puts "#{@artifact_id} #{@version} found at #{url}. (skipping deployment)"
+        res = true
+      end          
+    rescue => e
+      puts "#{@artifact_id} not found at #{url}: #{e}"
+    end
+    res
+  end  
+
+  def compareVersion(target_version, deployed_version)    
+    target = Gem::Version.new(target_version)
+    begin
+      found = Gem::Version.new(deployed_version)
+      target > found
+    rescue => e
+      puts "Error parsing version number: #{e}"
+      false
+    end    
+  end  
+
+  def to_s
+    "#{artifact_id}-#{version}.#{extension}"
+  end   
+end
+
+class JvmApp < Application
+  attr_accessor :sha1, :artifact_repo
+
+  def initialize(id, project_name, artifact_repo)
+    super(id, project_name, :jvm)
+    @sha1 = nil
+    @artifact_repo = artifact_repo
+  end
+
+  def download(location)
+    super
+    puts "- Provision NewRelic Agent jar"    
+    @artifact_repo.downloadNewRelicAgent(File.join(location, @artifact_id), '3.2.3', 'c07cd82e033af9628cd7f90df874daee7000e471')    
+  end
+end
+
+class RailsZipApp < Application
+  attr_accessor :sha1, :artifact_repo
+
+  def initialize(id, project_name, artifact_repo)
+    super(id, project_name, :rails_zip)
+    @sha1 = nil
+    @artifact_repo = artifact_repo
+  end
+end
+
+class HasOptions
+  attr_accessor :options
+
+  def initialize(options)
+    @options = options
+  end
+
+  def option(key, value)
+    if @options[key].nil?
+      @options[key] = [value]
+    else
+      @options[key] << value
+    end
+  end
+end
+
+class Environment #< HasOptions
+  attr_accessor :id, :upstream_env, :deployments
+
+  def initialize(env_id)
+    #super({})
+    @id = env_id
+    @deployments = []
+    @org_holder = nil
+  end
+
+  def promotes_from(env_id)
+    @upstream_env = env_id    
+  end
+
+  def org(org_name, &block)
+    @org_holder = org_name
+    self.instance_eval(&block)
+  end
+
+  def deploy(ship, &block)
+    d = Deployment.new(self, @org_holder, ship)
+    d.instance_eval(&block)
+    @deployments << d    
+  end
+end
+
+class Deployment < HasOptions
+
+  @@paas_domain = 'mtnsatcloud.com'
+
+  attr_accessor :environment, :org, :ship, :shipcloud, :app_ids, :facility_id
+
+  def initialize(environment, org, ship)
+    # inherit options from the environment
+    #super(environment.options)  # TODO:  copy?!
+    super({})
+    @environment = environment
+    @org = org    
+    @ship = ship
+    @shipcloud = "#{org}-#{ship}"
+    @app_ids = []
+    @facility_id = nil
+    @enabled = true
+    @randomid = SecureRandom.hex(3)
+  end  
+
+  def apps(*app_ids)
+    @app_ids = app_ids
+  end
+
+  def facility(facility_id)
+    @facility_id = facility_id
+  end
+
+  def enabled(b)
+    @enabled = b
+  end
+
+  def enabled?
+    @enabled
+  end
+
+  def name(app_id)
+    "#{app_id}-#{@environment.id}-#{@ship}"
+  end
+
+  def dns_name(app_id)
+    "#{@environment.id}.#{app_id}.#{@ship}.#{@org}.#{@@paas_domain}"
+  end
+
+  def agnostic_dns_name(app_id)
+    "#{@environment.id}.#{app_id}.#{@@paas_domain}"
+  end  
+
+  def unique_name(app_id)
+    "#{name(app_id)}-#{@randomid}"
+  end
+
+  def active_urls(app_id)
+    [ dns_name(app_id), agnostic_dns_name(app_id) ]
+  end
+
+  def inactive_urls(app_id)
+    ["inactive.#{dns_name(app_id)}"]
+  end
+
+  def manifest_url(app_id)
+    "http://#{dns_name(app_id)}/status/manifest"
+  end
+
+  def paas_target
+    "https://api.paas.#{@ship}.#{@org}.#{@@paas_domain}"
+  end
+
+  def to_s
+    "#{org}-#{ship}"
+  end   
+end
+

data/lib/conan/newrelic.rb

@@ -0,0 +1,16 @@
+require 'rest_client'
+
+module NewRelic
+  def self.alertDeployment(api_key, app_name, description)    
+    response = RestClient.post(
+      'https://api.newrelic.com/deployments.xml',
+      :params => {
+        :'deployment[app_name]' => app_name, 
+        :'deployment[description]' => description
+      },
+      :headers => { 'x-api-key' => api_key }
+    )
+    puts response
+  end
+end
+

data/lib/conan/options.rb

@@ -0,0 +1,31 @@
+require 'conan/version'
+
+module Conan
+  class Options
+    def initialize(options = {})
+      @options = defaults.merge(options)
+    end
+
+    def []=(k, v)
+      @options[k] = v
+    end
+
+    def [](k)
+      @options[k]
+    end
+
+    def to_s
+      @options.to_s
+    end
+
+  private
+    def defaults
+      {
+        :version => Conan::VERSION,
+        :directory  => Dir.pwd,
+        :format => :stackato,
+        :action => :provision
+      }
+    end
+  end
+end

data/lib/conan/output.rb

@@ -0,0 +1,169 @@
+require 'conan/templates'
+
+class OutThingy
+  attr_accessor :output_dir, :time
+
+  def initialize(output_dir)
+    @output_dir = output_dir
+    @time = Time.new
+  end
+
+  def outputToFile(file, &block)
+    o = File.join(@output_dir, file)    
+    puts "Write: #{o}"  
+    FileUtils.mkdir_p File.dirname(o)
+    File.open(o, 'w') { |f| yield f }
+  end
+
+  def loadAppMetadataFromDeploymentIfExists(app, deploy)
+    loadAppMetadataFromDeployment(app, deploy, true)
+  end  
+end
+
+class StackatoOutThingy < OutThingy
+
+  attr_accessor :environments_dir, :apps_dir
+
+  @@stackato_file_name = "stackato.yml"
+  @@metadata_file_name = "metadata.xml"
+  @environments_dir = nil
+  @apps_dir = nil
+
+  def initialize(output_dir)
+    super(output_dir)
+    workdir = File.join(output_dir,'tmp')
+    @environments_dir = File.join(output_dir,'environments')
+    @apps_dir = File.join(workdir,'apps')
+    FileUtils.mkdir_p environments_dir
+    FileUtils.mkdir_p apps_dir
+  end
+
+  def clean(app, environment) 
+    #  TODO: cleanup
+  end
+
+  def provision(app)
+    app.download(@apps_dir) 
+  end
+
+  def filePath(app, deploy) 
+    "#{deploy.environment.id}/#{deploy.shipcloud}/#{app.id}"
+  end
+
+  def workingDir(app)
+    File.join(apps_dir, app.artifact_id)
+  end
+
+  # TODO this needs a refactor, it's knoted up with App
+  def loadAppMetadataFromDeployment(app, deploy, fail_silent=false)  
+    p = filePath(app, deploy)
+    path = File.join(@environments_dir, p) 
+    metadata_file = File.join(path, @@metadata_file_name)
+
+    if (File.exists? path)
+      File.open(metadata_file, 'r') { |f| app.readArtifactMetadata(f.read) }
+    else
+      raise "Build meta-data was not found: #{path}" unless fail_silent
+    end
+  end 
+
+  def writeArtifactMetadata(app, deploy)
+    if (deploy.enabled?)
+      path = filePath(app, deploy)
+      outputToFile("environments/#{path}/#{@@metadata_file_name}") { |f|
+        f.write(app.artifact_meta_data)
+      }
+    end
+  end
+     
+  def writeConfiguration(app, deploy, bg=false)
+    if (deploy.enabled?)      
+      target_dir = workingDir(app)
+      templates_dir = File.join(target_dir, "deploy-templates")
+      n_changes = Templates.evaluate templates_dir, target_dir, {
+        :app => app,
+        :deploy => deploy,
+
+        # short-cuts          
+        :app_id => app.id,
+        :deploy_base_name => deploy.name(app.id),
+        :deploy_name => bg ? deploy.unique_name(app.id) : deploy.name(app.id),
+        :deploy_urls => bg ? deploy.inactive_urls(app.id) : deploy.active_urls(app.id),
+        :env => deploy.environment.id,
+        :org => deploy.org,
+        :ship => deploy.ship,
+        :infra => deploy.shipcloud, # TODO: remove infra key, this is just to avoid breaking existing templates
+        :shipcloud => deploy.shipcloud,
+        :facility => deploy.facility_id,
+        :options => deploy.options    
+      }
+    end
+    #TODO: can this be used to provide idempotency for config-only deploymnent?
+    puts "#{n_changes} files changed"
+  end
+end
+
+class PropertiesFileOutThingy < OutThingy
+
+  def clean(app, environment) 
+    # cleanup current
+    FileUtils.rm Dir.glob("#{@output_dir}/#{app.id}-deploy-#{environment.id}-*.properties")    
+  end
+
+  def fileName(app, deploy) 
+    "#{app.id}-deploy-#{deploy.environment.id}-#{deploy.shipcloud}.properties"
+  end
+
+  def loadAppMetadataFromDeployment(app, deploy, fail_silent=false)
+    path = File.join(@output_dir, fileName(app, deploy))
+    if (File.exists? path)
+      props = Utils::Properties.load_from_file(path, true) 
+      app.version = props.get(:DEPLOY_VERSION) # TODO: preserve SHA1
+    else
+      raise "File not found: #{path}" unless fail_silent
+    end
+  end
+
+  def writeArtifactMetadata(app, deploy)
+    file_name = fileName(app, deploy)
+    out_file = File.join(@output_dir, file_name)
+    if (!deploy.enabled?)  
+      if (File.exists? out_file)        
+        File.delete(out_file)
+      end
+      out_file = File.join(@output_dir, "DISABLED-#{file_name}")        
+    end
+    puts "Write: #{out_file}"
+    File.open(out_file, 'w') { |f|
+      # TODO: would like @time in the manifest as audit field, but not sure it's such a good idea yet
+      # f << "\# #{file_name}\n\# #{@time}\n"      
+      f << "\# #{file_name}\n"
+      f << "DEPLOY_ENV=#{deploy.environment.id}\n"
+      f << "DEPLOY_INFRA=#{deploy.shipcloud}\n"
+      if (deploy.facility_id)
+        f << "FACILITY=facility-#{deploy.facility_id}\n"
+      end
+      unless (deploy.options.empty?)
+        options = []
+        deploy.options.each do |k, v|
+          options << k + '|' + v.join(',')
+        end
+        p options.join(';')
+        f << "OPTIONS=#{options.join(';')}\n"
+      end
+      f << "APP=#{app.id}\n"
+      f << "DEPLOY_SHA1=#{app.sha1}\n"
+      f << "DEPLOY_VERSION=#{app.version}\n"
+      f << "TYPE=#{app.platform_type}\n"
+      f << "\n"
+    }
+  end
+
+  def provision(app)
+    #noop
+  end
+
+  def writeConfiguration(app, deploy)
+    # noop
+  end
+end