lono 3.5.0 → 4.0.0

data/lib/lono/template/upload.rb

@@ -4,20 +4,20 @@ require 'base64'
 require 'digest'
 
 class Lono::Template::Upload
-  include Lono::Template::AwsServices
+  include Lono::Template::AwsService
 
   def initialize(options={})
     @options = options
     @checksums = {}
+    @prefix = "#{folder_key}/#{Lono.env}/templates" # s3://s3-bucket/folder/developemnt/templates
   end
 
   def run
     ensure_s3_setup!
     load_checksums!
 
-    paths = Dir.glob("#{Lono.root}/output/**/*")
-    paths.reject { |p| p =~ %r{output/params} }.
-          select { |p| File.file?(p) }.each do |path|
+    paths = Dir.glob("#{Lono.config.output_path}/templates/**/*")
+    paths.select { |p| File.file?(p) }.each do |path|
       upload(path)
     end
     say "Templates uploaded to s3."
@@ -35,12 +35,11 @@ class Lono::Template::Upload
   def load_checksums!
     return if @options[:noop]
 
-    prefix = "#{s3_path}/#{Lono.env}" # s3://s3-bucket-and-path-from-settings/prod
-    resp = s3.list_objects(bucket: s3_bucket, prefix: prefix)
+    resp = s3.list_objects(bucket: s3_bucket, prefix: @prefix)
     resp.contents.each do |object|
       # key does not include the bucket name
-      #    full path = s3://my-bucket/cloudformation-templates/prod/my-template.yml
-      #    key = cloudformation-templates/prod/my-template.yml
+      #    full path = s3://my-bucket/s3_folder/templates/production/my-template.yml
+      #    key = s3_folder/templates/production/my-template.yml
       # etag is the checksum as long as the file is not a multi-part file upload
       # it has extra double quotes wrapped around it.
       #    etag = "\"9cb437490cee2cc96101baf326e5ca81\""
@@ -55,7 +54,7 @@ class Lono::Template::Upload
 
   def upload(path)
     pretty_path = path.sub(/^\.\//, '')
-    key = "#{s3_path}/#{Lono.env}/#{pretty_path.sub(/^output\//,'')}"
+    key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
     s3_full_path = "s3://#{s3_bucket}/#{key}"
 
     local_checksum = Digest::MD5.hexdigest(IO.read(path))
@@ -73,57 +72,66 @@ class Lono::Template::Upload
     ) unless @options[:noop]
 
     # Example output:
-    # Uploaded: output/docker.yml to s3://boltops-stag/cloudformation-templates/stag/docker.yml
-    # Uploaded: output/ecs/private.yml to s3://boltops-stag/cloudformation-templates/stag/ecs/private.yml
+    # Uploaded: output/templates/docker.yml to s3://boltops-dev/s3_folder/templates/development/docker.yml
+    # Uploaded: output/templates/ecs/private.yml to s3://boltops-dev/s3_folder/templates/development/ecs/private.yml
     message = "Uploaded: #{pretty_path} to #{s3_full_path}".colorize(:green)
     message = "NOOP: #{message}" if @options[:noop]
     say message
   end
 
-  # @checksums map has a key format: cloudformation-templates/stag/docker.yml
+  # @checksums map has a key format: s3_folder/templates/development/docker.yml
   #
-  # path = ./output/docker.yml
-  # s3_path = s3://boltops-stag/cloudformation-templates/stag/docker.yml
+  # path = ./output/templates/docker.yml
+  # s3_folder = s3://boltops-dev/s3_folder/templates/development/docker.yml
   def remote_checksum(path)
     # first convert the local path to the path format that is stored in @checksums keys
-    # ./output/docker.yml => cloudformation-templates/stag/docker.yml
+    # ./output/templates/docker.yml => s3_folder/templates/development/docker.yml
     pretty_path = path.sub(/^\.\//, '')
-    key = "#{s3_path}/#{Lono.env}/#{pretty_path.sub(/^output\//,'')}"
+    key = "#{@prefix}/#{pretty_path.sub(%r{output/templates/},'')}"
     @checksums[key]
   end
 
-  # https://s3.amazonaws.com/mybucket/cloudformation-templates/prod/parent.yml
+  # https://s3.amazonaws.com/mybucket/s3_folder/templates/production/parent.yml
   def s3_https_url(template_path)
     ensure_s3_setup!
-    "https://s3.amazonaws.com/#{s3_bucket}/#{s3_path}/#{Lono.env}/#{template_path}"
+    "https://s3.amazonaws.com/#{s3_bucket}/#{@prefix}/#{template_path}"
   end
 
+  # Parse the s3_folder setting and remove the folder portion to leave the
+  # "s3_bucket" portion
   # Example:
-  #    s3_bucket('s3://mybucket/templates/storage/path') => mybucket
+  #    s3_bucket('s3://mybucket/templates/storage/path')
+  #    => mybucket
   def s3_bucket
-    return '' if @options[:noop] # to get spec passing
-    @s3_full_path.sub('s3://','').split('/').first
+    return nil if @options[:noop] # to get spec passing
+    return nil unless s3_folder
+    s3_folder.sub('s3://','').split('/').first
   end
 
+  # The folder_key is the s3_folder setting with the s3 bucket.
+  #
   # Example:
-  #    s3_bucket('s3://mybucket/templates/storage/path') => templates/storage/path
-  def s3_path
-    return '' if @options[:noop] # to get spec passing
-    @s3_full_path.sub('s3://','').split('/')[1..-1].join('/')
+  #    s3_bucket('s3://mybucket/templates/storage/path')
+  #    => templates/storage/path
+  def folder_key
+    return nil if @options[:noop] # to get spec passing
+    return nil unless s3_folder
+    s3_folder.sub('s3://','').split('/')[1..-1].join('/')
+  end
+
+  def s3_folder
+    settings = Lono::Setting.new
+    settings.s3_folder
   end
 
   # nice warning if the s3 path not found
   def ensure_s3_setup!
     return if @options[:noop]
+    return if s3_folder
 
-    settings = Lono::Settings.new
-    if settings.s3_path
-      @s3_full_path = settings.s3_path
-    else
-      say "Unable to upload templates to s3 because you have not configured the s3.path option in lono settings.yml.".colorize(:red)
-      say "Please configure  settings.yml with s3.path.  Refer to http://lono.cloud/docs/settings/ for more help.".colorize(:red)
-      exit 1
-    end
+    say "Unable to upload templates to s3 because you have not configured the s3_folder option in lono settings.yml.".colorize(:red)
+    say "Please configure settings.yml with s3_folder.  For more help: http://lono.cloud/docs/settings/".colorize(:red)
+    exit 1
   end
 
   def say(message)

data/lib/lono/upgrade4.rb

@@ -0,0 +1,175 @@
+require 'fileutils'
+require 'yaml'
+
+module Lono
+  class Upgrade4
+    def initialize(options)
+      @options = options
+    end
+
+    def run
+      checks
+      puts "Upgrading structure of your current project to the new lono version 4 project structure"
+
+      upgrade_settings("config/settings.yml")
+      upgrade_settings("#{ENV['HOME']}/.lono/settings.yml")
+      add_mandantory_settings
+
+      FileUtils.mkdir_p("app")
+      mv("helpers", "app/helpers")
+      mv("params", "config/params")
+      mv("config/templates", "app/definitions")
+      mv("templates", "app/templates")
+      mv("app/templates/partial", "app/partials")
+      mv("app/partials/user_data", "app/user_data")
+      update_variables
+      update_stacks
+      update_params
+
+      puts "Upgrade to lono version 4 complete!"
+    end
+
+    def update_stacks
+      update_structure("app/definitions")
+    end
+
+    def update_variables
+      update_structure("config/variables")
+    end
+
+    # Takes variable files in the subfolder like config/variables/production/*
+    # and combines the into a single file like config/variables/production.rb.
+    #
+    #   config/variables/base/* -> config/variables/base.rb
+    #   config/variables/development/* -> config/variables/development.rb
+    #   config/variables/production/* -> config/variables/production.rb
+    #   app/definitions/base/* -> app/definitions/base.rb
+    #   app/definitions/development/* -> app/definitions/development.rb
+    #   app/definitions/production/* -> app/definitions/production.rb
+    def update_structure(component_path)
+      puts "Updating structure of #{component_path}"
+      Dir.glob("#{component_path}/*").each do |path|
+        next unless File.directory?(path)
+        folder = File.basename(path)
+        update_structure_for(component_path, folder)
+      end
+
+      # remove the old folders
+      Dir.glob("#{component_path}/*").each do |path|
+        next unless File.directory?(path)
+        FileUtils.rm_rf(path)
+      end
+    end
+
+    # combines the files in the lono_env subfolder into one file
+    def update_structure_for(component_path, lono_env)
+      code = ""
+      Dir.glob("#{component_path}/#{lono_env}/*.rb").each do |path|
+        code << IO.read(path)
+        code << "\n"
+      end
+      IO.write("#{component_path}/#{env_map(lono_env)}.rb", code)
+    end
+
+    def update_params
+      Dir.glob("config/params/*").each do |path|
+        next unless File.directory?(path)
+        lono_env = File.basename(path)
+        mapped_env = env_map(lono_env)
+        if mapped_env != lono_env
+          mv("config/params/#{lono_env}", "config/params/#{mapped_env}")
+        end
+      end
+    end
+
+    # make to the longer full environment names
+    def env_map(lono_env)
+      map = {
+        "prod" => "production",
+        "stag" => "staging",
+        "dev" => "development",
+      }
+      map[lono_env] || lono_env
+    end
+
+    def checks
+      if File.exist?(Lono.config.definitions_path)
+        puts "It looks like you already have a #{Lono.config.definitions_path} folder in your project. This is the new project structure so exiting without updating anything."
+        exit
+      end
+
+      old_templates_path = "#{Lono.root}/config/templates"
+      if !File.exist?(old_templates_path)
+        puts "Could not find a #{old_templates_path} folder in your project. Maybe you want to run lono new to initialize a new lono project instead?"
+        exit
+      end
+    end
+
+    def upgrade_settings(path)
+      return unless File.exist?(path)
+
+      data = YAML.load_file(path)
+      return if data.key?("base") # already in new format
+
+      new_structure = {}
+
+      (data["aws_profile_lono_env_map"] || {}).each do |aws_profile, lono_env|
+        new_structure[env_map(lono_env)] ||= {}
+        new_structure[env_map(lono_env)]["aws_profiles"] ||= []
+        new_structure[env_map(lono_env)]["aws_profiles"] << aws_profile
+      end
+      data.delete("aws_profile_lono_env_map")
+
+      data = update_s3_setting(data)
+
+      new_structure["base"] = data
+      text = YAML.dump(new_structure)
+      IO.write(path, text)
+      puts "Upgraded settings: #{path}"
+      if path.include?(ENV['HOME'])
+        puts "NOTE: Your ~/.lono/settings.yml file was also upgraded to the new format. If you are using lono in other projects those will have to be upgraded also."
+      end
+    end
+
+    # accounts for most cases
+    def update_s3_setting(data)
+      return data unless data["s3"] && data["s3"]["path"]
+
+      options = data["s3"]["path"]
+      if options.is_a?(String)
+        data.delete("s3")
+        data["s3_folder"] = options
+        return data # return early if String
+      end
+
+      # Reach here: dealing with a Hash
+      if options.size == 1 and options["default"]
+        data["s3_folder"] = options["default"]
+      end
+
+      if options.size > 1
+        data["s3_folder"] = {}
+        options.each do |key, value|
+          data["s3_folder"][key] = value
+        end
+      end
+
+      data.delete("s3")
+      data
+    end
+
+    # If config/settings.yml does not exist, use the default one.
+    def add_mandantory_settings
+      return if File.exists?("config/settings.yml")
+
+      default_settings = File.expand_path("default/settings.yml", File.dirname(__FILE__))
+      FileUtils.cp(default_settings, "config/settings.yml")
+    end
+
+    def mv(src, dest)
+      return unless File.exist?(src)
+      puts "mv #{src} #{dest}"
+      FileUtils.mv(src, dest)
+    end
+  end
+end

data/lib/lono/user_data.rb

@@ -0,0 +1,31 @@
+module Lono
+  # This class is not use by lono internally. It is really only meant to be
+  # exposed to the lono userdata command so users can debug their generated
+  # app/user_data scripts. It is useful for debugging.
+  #
+  # Normally, the Lono::DSL#run method generates the CloudFormation templates
+  # and embeds user-data script into the template.
+  class UserData
+    def initialize(options)
+      @options = options
+      @name = options[:name]
+      @path = "#{Lono.root}/app/user_data/#{@name}.sh"
+    end
+
+    def generate
+      puts "Generating user_data for '#{@name}' at #{@path}"
+      if File.exist?(@path)
+        puts RenderMePretty.result(@path, context: context)
+      else
+        puts "ERROR: #{@path} does not exist".colorize(:red)
+        exit 1
+      end
+    end
+
+    # Context for ERB rendering.
+    # This is where we control what references get passed to the ERB rendering.
+    def context
+      @context ||= Lono::Template::Context.new(@options)
+    end
+  end
+end

data/lib/lono/version.rb

@@ -1,3 +1,3 @@
 module Lono
-  VERSION = "3.5.0"
+  VERSION = "4.0.0"
 end

data/lib/starter_projects/autoscaling/.gitignore

@@ -0,0 +1 @@
+output

data/lib/starter_projects/autoscaling/README.md

@@ -0,0 +1,118 @@
+# Autoscaling Lono Example
+
+## Usage
+
+Create a new project.
+
+```
+TEMPLATE=autoscaling lono new autoscaling
+cd autoscaling
+```
+
+Edit `config/params/base/autoscaling.txt` with your desired parameters.  Here are the current contents of `config/params/base/autoscaling.txt`:
+
+```sh
+VpcId=
+Subnets=
+OperatorEMail=
+KeyName=
+#InstanceType=        # optional
+#SSHLocation=         # optional
+```
+
+
+Then launch the stack.
+
+```sh
+lono cfn create autoscaling
+```
+
+## How This Example Was Originally Built
+
+First, generate a skeleton starter lono project.
+
+```sh
+lono new autoscaling
+```
+
+Then import a standard CloudFormation template into lono. The `--name` option is used to name the template `autoscaling`.
+
+```
+$ lono import https://s3-us-west-2.amazonaws.com/cloudformation-templates-us-west-2/AutoScalingMultiAZWithNotifications.template --name autoscaling
+=> Imported CloudFormation template and lono-fied it.
+Template definition added to app/definitions/base.rb
+Params file created to config/params/base/autoscaling.txt
+Template downloaded to app/templates/autoscaling.yml
+=> CloudFormation Template Summary:
+Parameters:
+Required:
+  VpcId (AWS::EC2::VPC::Id)
+  Subnets (List<AWS::EC2::Subnet::Id>)
+  OperatorEMail (String)
+  KeyName (AWS::EC2::KeyPair::KeyName)
+Optional:
+  InstanceType (String) Default: t2.small
+  SSHLocation (String) Default: 0.0.0.0/0
+Resources:
+  2 AWS::AutoScaling::ScalingPolicy
+  2 AWS::CloudWatch::Alarm
+  1 AWS::AutoScaling::LaunchConfiguration
+  1 AWS::ElasticLoadBalancingV2::LoadBalancer
+  1 AWS::ElasticLoadBalancingV2::Listener
+  1 AWS::ElasticLoadBalancingV2::TargetGroup
+  1 AWS::SNS::Topic
+  1 AWS::EC2::SecurityGroup
+  1 AWS::AutoScaling::AutoScalingGroup
+  9 Total
+Here are contents of the params config/params/base/autoscaling.txt file:
+VpcId=
+Subnets=
+OperatorEMail=
+KeyName=
+#InstanceType=        # optional
+#SSHLocation=         # optional
+$
+```
+
+The template was imported from [CloudFormation Auto Scaling Samples](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/sample-templates-services-us-west-2.html#w2ab2c23c48c13b7).
+
+A starter `config/params/base/autoscaling.txt` was generated with required parameters emptied and optional parameters commented out.  Fill in the required parameters.
+
+### Filling in the Parameters
+
+You can use the following commands to grab the default VPC and a subnet, and set the values needed for the parameters:
+
+```
+VPC=$(aws ec2 describe-vpcs | jq -r '.Vpcs[] | select(.IsDefault == true) | .VpcId')
+SUBNETS=$(aws ec2 describe-subnets | jq -r '.Subnets[].SubnetId' | tr -s '\n' ',' | sed 's/,*$//g')
+EMAIL=email@domain.com
+KEY_NAME=default
+```
+
+Now update the required parameters:
+
+```
+sed "s/VpcId=/VpcId=$VPC/; s/Subnets=/Subnets=$SUBNETS/; s/OperatorEMail=/OperatorEMail=$EMAIL/; s/KeyName=/KeyName=$KEY_NAME/;" config/params/base/autoscaling.txt > config/params/base/autoscaling.txt.1
+mv config/params/base/autoscaling.txt{.1,}
+```
+
+Now launch the stack:
+
+```
+lono cfn create autoscaling
+```
+
+## Commands Summarized
+
+```sh
+lono new autoscaling
+cd autoscaling
+lono import https://s3-us-west-2.amazonaws.com/cloudformation-templates-us-west-2/AutoScalingMultiAZWithNotifications.template --name autoscaling
+VPC=$(aws ec2 describe-vpcs | jq -r '.Vpcs[] | select(.IsDefault == true) | .VpcId')
+SUBNETS=$(aws ec2 describe-subnets | jq -r '.Subnets[].SubnetId' | tr -s '\n' ',' | sed 's/,*$//g')
+EMAIL=email@domain.com
+KEY_NAME=default
+sed "s/VpcId=/VpcId=$VPC/; s/Subnets=/Subnets=$SUBNETS/; s/OperatorEMail=/OperatorEMail=$EMAIL/; s/KeyName=/KeyName=$KEY_NAME/;" config/params/base/autoscaling.txt > config/params/base/autoscaling.txt.1
+mv config/params/base/autoscaling.txt{.1,}
+lono cfn create autoscaling --template autoscaling
+```