conan_deploy_stackato3 0.1.1

data/lib/conan/deployment.rb

@@ -0,0 +1,91 @@
+require 'conan/has_options'
+class Deployment < HasOptions
+
+  attr_accessor :environment, :org, :ship, :shipcloud, :app_ids, :facility_id, :additional_mappings, :custom_smoke_test_path
+
+  def initialize(environment, org, ship)
+    # inherit options from the environment
+    #super(environment.options)  # TODO:  copy?!
+    super({})
+    @environment = environment
+    @org = org
+    @ship = ship
+    @shipcloud = "#{org}-#{ship}"
+    @app_ids = []
+    @additional_mappings = []
+    @facility_id = nil
+    @enabled = true
+    @randomid = SecureRandom.hex(3)
+    @custom_smoke_test_path = nil
+  end
+
+  def apps(*app_ids)
+    @app_ids = app_ids
+  end
+
+  def facility(facility_id)
+    @facility_id = facility_id
+  end
+
+  def hostnames(*hostnames)
+    hostnames.each do |hostname|
+      @additional_mappings.push(hostname) unless hostname.empty?
+    end
+  end
+
+  def enabled(b)
+    @enabled = b
+  end
+
+  def enabled?
+    @enabled
+  end
+
+  def name(app)
+    "#{app.id}-#{@environment.id}-#{@ship}"
+  end
+
+  def unique_name(app)
+    "#{name(app)}-#{@randomid}"
+  end
+
+  def dns_name(app)
+    "#{@environment.id}.#{app.url_segment}.#{@ship}.#{@org}.mtnsatcloud.com"
+  end
+
+  def agnostic_dns_name(app)
+    "#{app.url_segment}.mtnsat.io"
+  end
+
+  def active_urls(app)
+    [ dns_name(app), agnostic_dns_name(app) ] + @additional_mappings
+  end
+
+  def inactive_urls(app)
+    active_urls(app).map{ |url| "x.#{url}" }
+  end
+
+  def manifest_url(app)
+    "http://#{dns_name(app)}/status/manifest"
+  end
+
+  def active_smoke_test_url(app)
+    "http://#{dns_name(app)}/#{@custom_smoke_test_path || 'status/healthcheck'}"
+  end
+
+  def inactive_smoke_test_url(app)
+    "http://x.#{dns_name(app)}/#{@custom_smoke_test_path || 'status/healthcheck'}"
+  end
+
+  def paas_target
+    "https://api.paasv2.#{@ship}.#{@org}.mtnsatcloud.com"
+  end
+
+  def to_s
+    "#{org}-#{ship}"
+  end
+
+  def smoke_test_path(path=nil)
+    @custom_smoke_test_path = path
+  end
+end

data/lib/conan/environment.rb

@@ -0,0 +1,24 @@
+class Environment
+  attr_accessor :id, :upstream_env, :deployments
+
+  def initialize(env_id)
+    @id = env_id
+    @deployments = []
+    @org_holder = nil
+  end
+
+  def promotes_from(env_id)
+    @upstream_env = env_id
+  end
+
+  def org(org_name, &block)
+    @org_holder = org_name
+    self.instance_eval(&block)
+  end
+
+  def deploy(ship, &block)
+    d = Deployment.new(self, @org_holder, ship)
+    d.instance_eval(&block)
+    @deployments << d
+  end
+end

data/lib/conan/has_options.rb

@@ -0,0 +1,15 @@
+class HasOptions
+  attr_accessor :options
+
+  def initialize(options)
+    @options = options
+  end
+
+  def option(key, value)
+    if @options[key].nil?
+      @options[key] = [value]
+    else
+      @options[key] << value
+    end
+  end
+end

data/lib/conan/jvm_app.rb

@@ -0,0 +1,16 @@
+require 'conan/application'
+class JvmApp < Application
+  attr_accessor :sha1, :artifact_repo
+
+  def initialize(id, project_name, artifact_repo, options, url_segment=nil)
+    super(id, project_name, :jvm, options, url_segment)
+    @sha1 = nil
+    @artifact_repo = artifact_repo
+  end
+
+  def download(location)
+    super
+    puts "- Provision NewRelic Agent jar"
+    @artifact_repo.downloadNewRelicAgent(File.join(location, @artifact_id), '3.2.3', 'c07cd82e033af9628cd7f90df874daee7000e471')
+  end
+end

data/lib/conan/manifest.rb

@@ -0,0 +1,100 @@
+class Manifest
+
+  def initialize(options, artifact_repo, output)
+    @pipelines = {}
+    @environments = {}
+    @options = options
+    @artifact_repo = artifact_repo
+    @output = output
+  end
+
+  # select a pipeline and environment to do operations on
+  def select(pipeline_id, env_id)
+    env = @environments[env_id]
+    raise ArgumentError.new "Invalid environment id: '#{env_id}'. Valid options are #{@environments.keys.join(", ")}" if env.nil?
+    @current_environment = env
+    pl = @pipelines[pipeline_id]
+    raise ArgumentError.new "Invalid pipeline id: '#{pipeline_id}'. Valid options are #{@pipelines.keys.join(", ")}" if pl.nil?
+    @current_pipeline = pl
+    pl.load!(env, @options[:'deploy-shipcloud'])
+  end
+
+  def pipeline(pipeline_id, &block)
+    pl = Pipeline.new(pipeline_id, @artifact_repo, @output, @options)
+    pl.instance_eval(&block)
+    @pipelines[pipeline_id] = pl
+    pl
+  end
+
+  def environment(env_id, &block)
+    e = Environment.new(env_id)
+    e.instance_eval(&block)
+    @environments[env_id] = e
+    e
+  end
+
+  def appVersion(pipeline_id, app_id)
+    @pipelines[pipeline_id].appVersion(app_id)
+  end
+
+  def provision
+    validateThatEnvironmentIsSelected
+    # TODO banners
+    puts "Update #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.update(@current_environment, @environments[@current_environment.upstream_env])
+  end
+
+  def configure
+    validateThatEnvironmentIsSelected
+    puts "Configure #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.configure(@current_environment)
+  end
+
+  def bg_configure
+    validateThatEnvironmentIsSelected
+    puts "Blue/green configure #{@current_environment.id} manifest files for #{@current_pipeline.id} pipeline"
+    @current_pipeline.configure(@current_environment, true)
+  end
+
+  def paas
+    paas_user = @options[:'paas-user'] || ENV['PAAS_USER']
+    paas_pwd = @options[:'paas-password'] || ENV['PAAS_PASSWORD']
+    paas_space = @options[:'paas-space'] || ENV['PAAS_SPACE']
+    dry_run = @options[:'dry-run'] || false
+
+    Stackato.new(paas_user, paas_pwd, paas_space, dry_run)
+  end
+
+  def deploy
+    validateThatEnvironmentIsSelected
+    puts "Deploy #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    nr_api_key = @options[:'new-relic-api-key'] || ENV['NEWRELIC_API_KEY']
+    force = @options[:'force-deploy'] || false
+
+    @current_pipeline.deploy(@current_environment, paas, force) { |app_name|
+      if (nr_api_key)
+        puts 'Reporting Deployment to New Relic'
+        job = @options[:'job-url'] || 'nexus-app-manifest'
+        NewRelic.alertDeployment(nr_api_key, app_name, "Deployed by #{job}")
+      else
+        puts "WARNING: Skipping New Relic alert. No API defined"
+      end
+    }
+  end
+
+  def bg_deploy
+    validateThatEnvironmentIsSelected
+    puts "Blue/green deploy #{@current_environment.id} for #{@current_pipeline.id} pipeline"
+
+    force = @options[:'force-deploy'] || false
+
+    @current_pipeline.bg_deploy(@current_environment, paas, force)
+  end
+
+  def validateThatEnvironmentIsSelected
+    unless @current_environment && @current_pipeline
+      raise ScriptError, 'Environment and Pipeline must be selected'
+    end
+  end
+end

data/lib/conan/manifest_builder.rb

@@ -0,0 +1,14 @@
+module ManifestBuilder
+  def self.build(options, artifact_repo=nil, outthingy=nil, &block)
+    manifest_dir = options[:directory]
+    output_type = options[:format]
+
+    artifact_repo ||= DefaultArtifactRepository.new
+    outthingy ||= StackatoOutThingy.new(manifest_dir)
+
+    manifest = Manifest.new(options, artifact_repo, outthingy)
+    manifest.instance_eval(&block)
+    manifest.select(options[:pipeline], options[:environment])
+    manifest
+  end
+end

data/lib/conan/newrelic.rb

@@ -0,0 +1,16 @@
+require 'rest_client'
+
+module NewRelic
+  def self.alertDeployment(api_key, app_name, description)    
+    response = RestClient.post(
+      'https://api.newrelic.com/deployments.xml',
+      :params => {
+        :'deployment[app_name]' => app_name, 
+        :'deployment[description]' => description
+      },
+      :headers => { 'x-api-key' => api_key }
+    )
+    puts response
+  end
+end
+

data/lib/conan/options.rb

@@ -0,0 +1,31 @@
+require 'conan/version'
+
+module Conan
+  class Options
+    def initialize(options = {})
+      @options = defaults.merge(options)
+    end
+
+    def []=(k, v)
+      @options[k] = v
+    end
+
+    def [](k)
+      @options[k]
+    end
+
+    def to_s
+      @options.to_s
+    end
+
+  private
+    def defaults
+      {
+        :version => Conan::VERSION,
+        :directory  => Dir.pwd,
+        :format => :stackato,
+        :action => :provision
+      }
+    end
+  end
+end

data/lib/conan/output.rb

@@ -0,0 +1,180 @@
+require 'daphne_util'
+require 'conan/templates'
+
+class OutThingy
+  attr_accessor :output_dir, :time
+
+  def initialize(output_dir)
+    @output_dir = output_dir
+    @time = Time.new
+  end
+
+  def outputToFile(file, &block)
+    o = File.join(@output_dir, file)
+    puts "Write: #{o}"
+    FileUtils.mkdir_p File.dirname(o)
+    File.open(o, 'w') { |f| yield f }
+  end
+
+  def loadAppMetadataFromDeploymentIfExists(app, deploy)
+    loadAppMetadataFromDeployment(app, deploy, true)
+  end
+end
+
+class StackatoOutThingy < OutThingy
+
+  attr_accessor :environments_dir, :apps_dir
+
+  @@stackato_file_name = "stackato.yml"
+  @@metadata_file_name = "metadata.xml"
+  @environments_dir = nil
+  @apps_dir = nil
+
+  def initialize(output_dir)
+    super(output_dir)
+    workdir = File.join(output_dir,'tmp')
+    @environments_dir = File.join(output_dir,'environments')
+    @apps_dir = File.join(workdir,'apps')
+    FileUtils.mkdir_p environments_dir
+    FileUtils.mkdir_p apps_dir
+  end
+
+  def clean(app, environment)
+    #  TODO: cleanup
+  end
+
+  def provision(app)
+    app.download(@apps_dir)
+  end
+
+  def filePath(app, deploy)
+    "#{deploy.environment.id}/#{deploy.shipcloud}/#{app.id}"
+  end
+
+  def workingDir(app)
+    File.join(apps_dir, app.artifact_id)
+  end
+
+  # TODO this needs a refactor, it's knoted up with App
+  def loadAppMetadataFromDeployment(app, deploy, fail_silent=false)
+    p = filePath(app, deploy)
+    path = File.join(@environments_dir, p)
+    metadata_file = File.join(path, @@metadata_file_name)
+
+    if (File.exists? path)
+      File.open(metadata_file, 'r') { |f| app.readArtifactMetadata(f.read) }
+    else
+      raise "Build meta-data was not found: #{path}" unless fail_silent
+    end
+  end
+
+  def writeArtifactMetadata(app, deploy)
+    if (deploy.enabled?)
+      path = filePath(app, deploy)
+      outputToFile("environments/#{path}/#{@@metadata_file_name}") { |f|
+        f.write(app.artifact_meta_data)
+      }
+    end
+  end
+
+  def writeConfiguration(app, deploy, bg=false)
+    if (deploy.enabled?)
+      target_dir = workingDir(app)
+      templates_dir = File.join(target_dir, "deploy-templates")
+
+      daphne_tokens = [ "#{app.id}", "#{deploy.environment.id}" ]
+      daphne_tokens = daphne_tokens << "facility-#{deploy.facility_id}" unless deploy.facility_id.nil?
+
+      n_changes = Templates.evaluate templates_dir, target_dir, {
+        :app => app,
+        :deploy => deploy,
+
+        #daphne, generate oauth app tokens
+        #TODO: seems like this should be refactored
+        :oauth_id => DaphneUtil.generate_id(*daphne_tokens),
+        :oauth_secret => DaphneUtil.generate_secret(*daphne_tokens),
+        :api_key => DaphneUtil.generate_api_key(*daphne_tokens),
+
+        # short-cuts
+        :app_id => app.id,
+        :deploy_base_name => deploy.name(app),
+        :deploy_name => bg ? deploy.unique_name(app) : deploy.name(app),
+        :deploy_urls => bg ? deploy.inactive_urls(app) : deploy.active_urls(app),
+        :env => deploy.environment.id,
+        :org => deploy.org,
+        :ship => deploy.ship,
+        :infra => deploy.shipcloud, # TODO: remove infra key, this is just to avoid breaking existing templates
+        :shipcloud => deploy.shipcloud,
+        :facility => deploy.facility_id,
+        :options => deploy.options
+      }
+    end
+    #TODO: can this be used to provide idempotency for config-only deploymnent?
+    puts "#{n_changes} files changed"
+  end
+end
+
+class PropertiesFileOutThingy < OutThingy
+
+  def clean(app, environment)
+    # cleanup current
+    FileUtils.rm Dir.glob("#{@output_dir}/#{app.id}-deploy-#{environment.id}-*.properties")
+  end
+
+  def fileName(app, deploy)
+    "#{app.id}-deploy-#{deploy.environment.id}-#{deploy.shipcloud}.properties"
+  end
+
+  def loadAppMetadataFromDeployment(app, deploy, fail_silent=false)
+    path = File.join(@output_dir, fileName(app, deploy))
+    if (File.exists? path)
+      props = Utils::Properties.load_from_file(path, true)
+      app.version = props.get(:DEPLOY_VERSION) # TODO: preserve SHA1
+    else
+      raise "File not found: #{path}" unless fail_silent
+    end
+  end
+
+  def writeArtifactMetadata(app, deploy)
+    file_name = fileName(app, deploy)
+    out_file = File.join(@output_dir, file_name)
+    if (!deploy.enabled?)
+      if (File.exists? out_file)
+        File.delete(out_file)
+      end
+      out_file = File.join(@output_dir, "DISABLED-#{file_name}")
+    end
+    puts "Write: #{out_file}"
+    File.open(out_file, 'w') { |f|
+      # TODO: would like @time in the manifest as audit field, but not sure it's such a good idea yet
+      # f << "\# #{file_name}\n\# #{@time}\n"
+      f << "\# #{file_name}\n"
+      f << "DEPLOY_ENV=#{deploy.environment.id}\n"
+      f << "DEPLOY_INFRA=#{deploy.shipcloud}\n"
+      if (deploy.facility_id)
+        f << "FACILITY=facility-#{deploy.facility_id}\n"
+      end
+      unless (deploy.options.empty?)
+        options = []
+        deploy.options.each do |k, v|
+          options << k + '|' + v.join(',')
+        end
+        p options.join(';')
+        f << "OPTIONS=#{options.join(';')}\n"
+      end
+      f << "APP=#{app.id}\n"
+      f << "DEPLOY_SHA1=#{app.sha1}\n"
+      f << "DEPLOY_VERSION=#{app.version}\n"
+      f << "TYPE=#{app.platform_type}\n"
+      f << "\n"
+    }
+  end
+
+  def provision(app)
+    #noop
+  end
+
+  def writeConfiguration(app, deploy)
+    # noop
+  end
+end

data/lib/conan/pipeline.rb

@@ -0,0 +1,102 @@
+class Pipeline
+  include ApiHelper
+
+  attr_accessor :id, :apps
+
+  def initialize(id, artifact_repo, output, options)
+    @id = id
+    @artifact_repo = artifact_repo
+    @output = output
+    @apps = {}
+    @options = options
+  end
+
+  def app(app_id, project_name, platform_type, url_segment=nil)
+    if (@options[:'deploy-app-name'].nil?) || (@options[:'deploy-app-name'] == app_id.to_s)
+      puts "No app specified, or we matched the specified app: #{@options[:'deploy-app-name']}"
+      @apps[app_id] = case platform_type
+        when :jvm
+          JvmApp.new(app_id, project_name, @artifact_repo, @options, url_segment)
+        when :rails_zip
+          RailsZipApp.new(app_id, project_name, @artifact_repo, @options, url_segment)
+        else
+          raise "unknown platform type: #{platform_type}"
+        end
+    else
+      puts "App #{app_id} doesn't match specified app #{@options[:'deploy-app-name']}, skipping"
+    end
+  end
+
+  def appVersion(app_id)
+    @apps[app_id].version
+  end
+
+  def load!(environment, deploy_shipcloud)
+    eachAppDeployment(environment) { |app, deploy|
+      # load the persisted meta-data
+      @output.loadAppMetadataFromDeploymentIfExists(app, deploy)
+    }
+    @deploy_shipcloud = deploy_shipcloud
+  end
+
+  def update(environment, from_upstream_env)
+    @apps.values.each { |app|
+      puts ''
+      if !(@options[:'deploy-app-version'].nil?)
+        puts "Using version from options: #{@options[:'deploy-app-version']}"
+        app.findRequestedVersion(@options[:'deploy-app-version'])
+      elsif (from_upstream_env)
+        # promote versions from an upstream deploy
+        # TODO: this needs improving
+        puts "Promote #{app.id} from #{from_upstream_env.id} to #{environment.id}"
+        upstream_deploy = from_upstream_env.deployments.select { |d| d.app_ids.select {|x| x == app.id }.size > 0 }.first
+        @output.loadAppMetadataFromDeployment(app, upstream_deploy)
+      else
+        puts "Find latest #{app.artifact_id}"
+        # lookup most recent from repo
+        app.findRequestedVersion('LATEST')
+      end
+      @output.clean(app, environment)
+      # download artifacts
+      @output.provision(app)
+    }
+
+    # write out deployment artifact meta-data
+    eachAppDeployment(environment) { |app, deploy| @output.writeArtifactMetadata(app, deploy) }
+  end
+
+  def configure(environment, bg=false)
+    eachAppDeployment(environment) { |app, deploy| @output.writeConfiguration(app, deploy, bg) }
+  end
+
+  def deploy(environment, paas, force=false)
+    eachAppDeployment(environment) { |app, deploy|
+      work_dir = @output.workingDir(app)
+      if (force)
+        paas.deploy(work_dir, app, deploy, true)
+      else
+        paas.deploy(work_dir, app, deploy) unless app.isDeployedAt(deploy.manifest_url(app))
+      end
+    }
+  end
+
+  def bg_deploy(environment, paas, force=false)
+    apps = []
+    eachAppDeployment(environment) { |app, deploy|
+      if (force or !app.isDeployedAt(deploy.manifest_url(app)))
+        paas.bg_deploy(@output.workingDir(app), app, deploy)
+      end
+    }
+  end
+
+  def eachAppDeployment(environment)
+    selected_deployments = environment.deployments.select{ |d| @deploy_shipcloud.nil? || d.shipcloud == @deploy_shipcloud }
+    puts "WARNING no deploys selected: #{@deploy_shipcloud}" if selected_deployments.empty?
+    selected_deployments.each do |d|
+      d.app_ids.each do |a|
+        app = @apps[a]
+        yield app, d unless app.nil?
+      end
+    end
+  end
+end

data/lib/conan/rails_zip_app.rb

@@ -0,0 +1,10 @@
+require 'conan/application'
+class RailsZipApp < Application
+  attr_accessor :sha1, :artifact_repo
+
+  def initialize(id, project_name, artifact_repo, options, url_segment=nil)
+    super(id, project_name, :rails_zip, options, url_segment)
+    @sha1 = nil
+    @artifact_repo = artifact_repo
+  end
+end