bosh-gen 0.8.2 → 0.9.0

data/ChangeLog.md

@@ -1,5 +1,18 @@
 # Change Log
 
+## v0.9.0
+
+Major news
+
+* `job` - more powerful initial scripts; templates nested in folders; scripts are much cleaner to read; scripts in bin/ & helpers/ do not have ERb; only data/properties.sh.erb & config/ are for ERb.
+
+Other changes:
+
+* `package` - detects .zip files (in addition to .tar.gz) and includes useful default unpacking script in `packaging`; describe available env vars in `packaging` script
+* `manifest` - auto-detects current BOSH UUID
+* `extract-pkg` - now a single argument - the path of the source package folder
+* `extract-job` - now a single argument - the path of the source job folder
+
 ## v0.8.0
 
 Changed:

data/lib/bosh/gen/cli.rb

@@ -13,8 +13,10 @@ module Bosh
       include Thor::Actions
     
       desc "new PATH", "Creates a new BOSH release"
-      method_option :s3, :alias => ["--aws"], :type => :boolean, :desc => "Use AWS S3 bucket for blobstore"
-      method_option :atmos, :type => :boolean, :desc => "Use EMC ATMOS for blobstore"
+      method_option :s3, :alias => ["--aws"], :type => :boolean, 
+        :desc => "Use AWS S3 bucket for blobstore"
+      method_option :atmos, :type => :boolean, 
+        :desc => "Use EMC ATMOS for blobstore"
       def new(path)
         flags = { :aws => options["s3"], :atmos => options["atmos"] }
         
@@ -23,17 +25,21 @@ module Bosh
       end
       
       desc "package NAME", "Create a new package"
-      method_option :dependencies, :aliases => ['-d'], :type => :array, :desc => "List of package dependencies"
-      method_option :files,        :aliases => ['-f', '--src'], :type => :array, :desc => "List of files copy into release"
+      method_option :dependencies, :aliases => ['-d'], :type => :array, 
+        :desc => "List of package dependencies"
+      method_option :files,        :aliases => ['-f', '--src'], :type => :array, 
+        :desc => "List of files copy into release"
       def package(name)
         dependencies = options[:dependencies] || []
         files        = options[:files] || []
         require 'bosh/gen/generators/package_generator'
-        Bosh::Gen::Generators::PackageGenerator.start([name, dependencies, files])
+        Bosh::Gen::Generators::PackageGenerator.start(
+          [name, dependencies, files])
       end
       
       desc "source NAME", "Downloads a source item into the named project"
-      method_option :blob, :aliases => ['-b'], :type => :boolean, :desc => "Store file in blobstore"
+      method_option :blob, :aliases => ['-b'], :type => :boolean, 
+        :desc => "Store file in blobstore"
       def source(name, uri)
         flags = { :blob => options[:blob] || false }
         dir = Dir.mktmpdir
@@ -49,49 +55,59 @@ module Bosh
         end
 
         require 'bosh/gen/generators/package_source_generator'
-        Bosh::Gen::Generators::PackageSourceGenerator.start([name, files, flags])
+        Bosh::Gen::Generators::PackageSourceGenerator.start(
+          [name, files, flags])
       end
       
-      desc "job NAME COMMAND", "Create a new job to run 'COMMAND' to launch the process"
-      method_option :dependencies, :aliases => ['-d'], :type => :array, :desc => "List of package dependencies"
-      method_option :ruby, :type => :boolean, :desc => "Use templates for running Ruby/Rack process"
-      def job(name, command=nil)
-        command ||= 'EXECUTABLE_SERVER'
-        flags = { :ruby => options["ruby"] || false }
-        dependencies   = options[:dependencies] || []
+      desc "job NAME", "Create a new job"
+      method_option :dependencies, :aliases => ['-d'], :type => :array, 
+        :desc => "List of package dependencies"
+      def job(name)
+        dependencies = options[:dependencies] || []
         require 'bosh/gen/generators/job_generator'
-        Bosh::Gen::Generators::JobGenerator.start([name, command, dependencies, flags])
+        Bosh::Gen::Generators::JobGenerator.start([name, dependencies, 'simple'])
       end
       
-      desc "template JOB FILE_PATH", "Add a Job template (example FILE_PATH: config/httpd.conf)"
+      desc "template JOB FILE_PATH", 
+        "Add a Job template (example FILE_PATH: config/httpd.conf)"
       def template(job_name, file_path)
         require 'bosh/gen/generators/job_template_generator'
         Bosh::Gen::Generators::JobTemplateGenerator.start([job_name, file_path])
       end
       
-      desc "extract-job SOURCE_RELEASE_PATH SOURCE_JOB_NAME [JOB_NAME]", "Extracts a job from another release and all its dependent packages and source"
-      def extract_job(source_release_path, source_job_name, target_job_name=nil)
-        target_job_name ||= source_job_name
+      desc "extract-job SOURCE_PACKAGE_PATH", 
+        "Extracts a job from another release and all its " +
+        "dependent packages and source"
+      def extract_job(source_package_path)
+        source_package_path = File.expand_path(source_package_path)
         require 'bosh/gen/generators/extract_job_generator'
-        Bosh::Gen::Generators::ExtractJobGenerator.start([source_release_path, source_job_name, target_job_name])
+        Bosh::Gen::Generators::ExtractJobGenerator.start([source_package_path])
       end
 
-      desc "extract-pkg SOURCE_RELEASE_PATH SOURCE_PACKAGE_NAME", "Extracts a package from another release and all its dependent packages and sources"
-      def extract_pkg(source_release_path, source_package_name)
+      desc "extract-pkg SOURCE_PACKAGE_PATH", 
+        "Extracts a package from another release and all its " +
+        "dependent packages and sources"
+      def extract_pkg(source_package_path)
+        source_package_path = File.expand_path(source_package_path)
         require 'bosh/gen/generators/extract_package_generator'
-        Bosh::Gen::Generators::ExtractPackageGenerator.start([source_release_path, source_package_name])
+        Bosh::Gen::Generators::ExtractPackageGenerator.start([source_package_path])
       end
 
-      desc "manifest NAME PATH UUID", "Creates a deployment manifest based on the release located at PATH"
-      method_option :force, :type => :boolean, :desc => "Force override existing target manifest file"
-      method_option :addresses, :aliases => ['-a'], :type => :array, :desc => "List of IP addresses available for jobs"
-      method_option :disk, :aliases => ['-d'], :type => :string, :desc => "Attach persistent disks to VMs of specific size, e.g. 8196"
-      def manifest(name, release_path, uuid)
+      desc "manifest NAME PATH", 
+        "Creates a deployment manifest based on the release located at PATH"
+      method_option :force, :type => :boolean, 
+        :desc => "Force override existing target manifest file"
+      method_option :addresses, :aliases => ['-a'], :type => :array, 
+        :desc => "List of IP addresses available for jobs"
+      method_option :disk, :aliases => ['-d'], :type => :string, 
+        :desc => "Attach persistent disks to VMs of specific size, e.g. 8196"
+      def manifest(name, release_path)
         release_path = File.expand_path(release_path)
         ip_addresses = options["addresses"] || []
         flags = { :force => options["force"] || false, :disk => options[:disk] }
         require 'bosh/gen/generators/deployment_manifest_generator'
-        Bosh::Gen::Generators::DeploymentManifestGenerator.start([name, release_path, uuid, ip_addresses, flags])
+        Bosh::Gen::Generators::DeploymentManifestGenerator.start(
+          [name, release_path, ip_addresses, flags])
       end
 
       no_tasks do

data/lib/bosh/gen/generators/deployment_manifest_generator.rb

@@ -8,7 +8,6 @@ module Bosh::Gen
 
       argument :name
       argument :release_path
-      argument :director_uuid
       argument :ip_addresses
       argument :flags, :type => :hash
 
@@ -31,10 +30,15 @@ module Bosh::Gen
 
       # Create a deployment manifest (initially for AWS only)
       def create_deployment_manifest
-        cloud_properties = { "instance_type" => "m1.small", "availability_zone" => "us-east-1e" }
+        cloud_properties = { 
+          "instance_type" => "m1.small", 
+          "availability_zone" => "us-east-1e"
+        }
         cloud_properties["persistent_disk"] = flags[:disk] if flags[:disk]
         cloud_properties["static_ips"] = ip_addresses
-        manifest = Bosh::Gen::Models::DeploymentManifest.new(name, director_uuid, release_properties, cloud_properties)
+        director_uuid = Bosh::Gen::Models::BoshConfig.new.target_uuid
+        manifest = Bosh::Gen::Models::DeploymentManifest.new(
+          name, director_uuid, release_properties, cloud_properties)
         manifest.jobs = job_manifests
         create_file manifest_file_name, manifest.to_yaml, :force => flags[:force]
       end

data/lib/bosh/gen/generators/extract_job_generator.rb

@@ -6,22 +6,41 @@ module Bosh::Gen
     class ExtractJobGenerator < Thor::Group
       include Thor::Actions
 
-      argument :source_release_path
-      argument :source_job_name
-      argument :job_name
-      
+      argument :source_job_path
+
       def check_root_is_release
         unless File.exist?("jobs") && File.exist?("packages")
           raise Thor::Error.new("run inside a BOSH release project")
         end
       end
       
+      def check_job_path_is_valid
+        unless File.exist?(source_job_path)
+          raise Thor::Error.new("source job path does not exist")
+        end
+      end
+      
+      def check_job_path_is_a_job
+        parent_dir = File.basename(File.dirname(source_job_path))
+        unless parent_dir == "jobs"
+          raise Thor::Error.new("source jobs path is not a BOSH job")
+        end
+      end
+      
+      def check_job_path_within_release
+        FileUtils.chdir(source_release_path) do
+          unless File.exist?("jobs") && File.exist?("packages")
+            raise Thor::Error.new("source job path is not within a BOSH release project")
+          end
+        end
+      end
+      
       def using_source_release_for_templates
         source_paths << File.join(source_release_path)
       end
 
       def copy_job_dir
-        directory "jobs/#{source_job_name}", "jobs/#{job_name}"
+        directory "jobs/#{source_job_name}", "jobs/#{target_job_name}"
       end
       
       def detect_dependent_packages
@@ -56,8 +75,20 @@ module Bosh::Gen
       end
       
       private
+      def source_release_path
+        File.expand_path(File.join(source_job_path, "..", ".."))
+      end
+      
+      def source_job_name
+        File.basename(source_job_path)
+      end
+      
+      def target_job_name
+        source_job_name
+      end
+
       def job_dir(path="")
-        File.join("jobs", job_name, path)
+        File.join("jobs", target_job_name, path)
       end
       
       def source_file(*path)

data/lib/bosh/gen/generators/extract_package_generator.rb

@@ -6,8 +6,7 @@ module Bosh::Gen
     class ExtractPackageGenerator < Thor::Group
       include Thor::Actions
 
-      argument :source_release_path
-      argument :source_package_name
+      argument :source_package_path
       
       def check_root_is_release
         unless File.exist?("jobs") && File.exist?("packages")
@@ -15,6 +14,21 @@ module Bosh::Gen
         end
       end
       
+      def check_package_path_within_release
+        FileUtils.chdir(source_release_path) do
+          unless File.exist?("jobs") && File.exist?("packages")
+            raise Thor::Error.new("source package path is not within a BOSH release project")
+          end
+        end
+      end
+      
+      def check_package_path_is_a_package
+        parent_dir = File.basename(File.dirname(source_package_path))
+        unless parent_dir == "packages"
+          raise Thor::Error.new("source package path is not a BOSH package")
+        end
+      end
+      
       def using_source_release_for_templates
         source_paths << File.join(source_release_path)
       end
@@ -22,7 +36,7 @@ module Bosh::Gen
       # Extract target package and all its dependencies
       def detect_dependent_packages
         spec = YAML.load_file(source_package_dir("spec"))
-        @packages = [target_package_name]
+        @packages = [source_package_name]
         @packages << spec["packages"] if spec["packages"]
       end
       
@@ -54,8 +68,12 @@ module Bosh::Gen
       end
       
       private
-      def target_package_name
-        source_package_name
+      def source_release_path
+        File.expand_path(File.join(source_package_path, "..", ".."))
+      end
+      
+      def source_package_name
+        File.basename(source_package_path)
       end
       
       def package_dir(path="")

data/lib/bosh/gen/generators/job_generator.rb

@@ -7,9 +7,8 @@ module Bosh::Gen
       include Thor::Actions
 
       argument :job_name
-      argument :command
       argument :dependencies, :type => :array
-      argument :flags, :type => :hash
+      argument :purpose
       
       def self.source_root
         File.join(File.dirname(__FILE__), "job_generator", "templates")
@@ -22,29 +21,44 @@ module Bosh::Gen
       end
       
       def check_name
-        raise Thor::Error.new("'#{job_name}' is not a vaild BOSH id") unless job_name.bosh_valid_id?
+        raise Thor::Error.new("'#{job_name}' is not a valid BOSH id") unless job_name.bosh_valid_id?
       end
-      
+
+      def check_purpose
+        unless valid_purposes.include?(purpose)
+          raise Thor::Error.new("'#{purpose}' is not a valid job purpose of #{valid_purposes.inspect}")
+        end
+      end
+
       def warn_missing_dependencies
         dependencies.each do |d|
-          raise Thor::Error.new("dependency '#{d}' is not a vaild BOSH id") unless d.bosh_valid_id?
+          raise Thor::Error.new("dependency '#{d}' is not a valid BOSH id") unless d.bosh_valid_id?
           unless File.exist?(File.join("packages", d))
             say_status "warning", "missing dependency '#{d}'", :yellow
           end
         end
       end
       
+      # copy the thor template files into the bosh release to be bosh templates
+      # that's right, templates (.tt) can become templates (.erb)
       def template_files
-        if ruby?
-          directory "jobs/%job_name%_rubyrack", "jobs/#{job_name}"
-        else
-          directory "jobs/%job_name%"
+        generator_job_templates_path = File.join(self.class.source_root, "jobs/%job_name%_#{purpose}")
+        directory "jobs/%job_name%_#{purpose}", "jobs/#{job_name}"
+
+        # build a hash of { 'bin/webapp_ctl.erb' => 'bin/webapp_ctl', ...} used in spec
+        # TODO: do I need to flatten it to { 'webapp_ctl' => 'bin/webapp_ctl' }?
+        @template_files = {}
+        FileUtils.chdir(File.join(generator_job_templates_path, "templates")) do
+          `ls */*`.split("\n").each do |template_file|
+            # clean up thor name convention
+            template_file.gsub!("%job_name%", job_name)
+            template_file.gsub!(".tt", "")
+            # strip erb from target file
+            target_template_file = template_file.gsub(/.erb/, '')
+
+            @template_files[template_file] = target_template_file
+          end
         end
-        @template_files = { "#{job_name}_ctl" => "bin/#{job_name}_ctl" }
-      end
-      
-      def ctl_executable
-        chmod "jobs/#{job_name}/templates/#{job_name}_ctl", 0755
       end
       
       def job_specification
@@ -52,6 +66,10 @@ module Bosh::Gen
         create_file job_dir("spec"), YAML.dump(config)
       end
       
+      def example
+        directory "examples/%job_name%_#{purpose}"
+      end
+
       private
       def filenames
         files.map {|f| File.basename(f) }
@@ -61,8 +79,8 @@ module Bosh::Gen
         File.join("jobs", job_name, path)
       end
       
-      def ruby?
-        flags[:ruby]
+      def valid_purposes
+        %w[simple]
       end
       
       # Run a command in git.

data/lib/bosh/gen/generators/job_generator/templates/examples/%job_name%_nginx_rack/nginx_puma_migrate.yml

@@ -0,0 +1,14 @@
+---
+jobs:
+  - <%= job_name %>
+properties:
+  webapp:
+    package: rackapp
+    use_nginx: 1
+    appstack: puma
+    puma:
+      threads:
+        min: 0
+        max: 20
+    migrate: 1
+    migration_command: rake db:migrate

data/lib/bosh/gen/generators/job_generator/templates/examples/%job_name%_nginx_rack/nginx_rackup.yml.tt

@@ -0,0 +1,8 @@
+---
+jobs:
+  - <%= job_name %>
+properties:
+  webapp:
+    package: rackapp
+    appstack: rackup
+    use_nginx: 1

data/lib/bosh/gen/generators/job_generator/templates/examples/%job_name%_simple/default.yml

@@ -0,0 +1,2 @@
+---
+properties: {}

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/monit.tt

@@ -0,0 +1,13 @@
+check process <%= job_name %>
+  with pidfile /var/vcap/sys/run/<%= job_name %>/<%= job_name %>.pid
+  start program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger <%= job_name %>_ctl '/var/vcap/jobs/<%= job_name %>/bin/<%= job_name %>_rack_ctl start'"
+  stop program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger <%= job_name %>_ctl '/var/vcap/jobs/<%= job_name %>/bin/<%= job_name %>_rack_ctl stop'"
+  group vcap
+
+<%% if properties.<%= job_name %> && properties.<%= job_name %>.use_nginx %>
+check process nginx
+  with pidfile /var/vcap/sys/run/<%= job_name %>/nginx.pid
+  start program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger nginx_ctl '/var/vcap/jobs/<%= job_name %>/bin/nginx_ctl start'"
+  stop program "/var/vcap/jobs/<%= job_name %>/bin/monit_debugger nginx_ctl '/var/vcap/jobs/<%= job_name %>/bin/nginx_ctl stop'"
+  group vcap
+<%% end %>

data/lib/bosh/gen/generators/job_generator/templates/jobs/%job_name%_nginx_rack/templates/bin/%job_name%_rack_ctl.tt

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/<%= job_name %>/bin/ctl_setup.sh
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+
+# Setup HTTP_PROXY, HTTPS_PROXY, NO_PROXY as necessary
+source $JOB_DIR/bin/ctl_nginx.sh
+
+# Setup for Rails application
+source $JOB_DIR/bin/rails_ctl_setup.sh
+
+# Helpers for PostgresDB
+source $JOB_DIR/bin/ctl_db_utils.sh
+
+# Helpers for Redis
+source $JOB_DIR/bin/ctl_redis_utils.sh
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+
+    cd $WEBAPP_DIR
+
+    # link database.yml config to application
+    link_sql_db_config
+
+    # if redis enabled, link it to application
+    link_redis_config yaml config/redis.yml
+
+    # Migrations, if requested
+    run_migrations $WEBAPP_DIR
+
+    # TODO what is 'ulimit -c unlimited' for; when not to use it?
+    ulimit -c unlimited
+
+    # Start the rack app using requested appstack
+    source $JOB_DIR/bin/ctl_start.sh
+
+    echo "Running $JOB_NAME on $PORT"
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+
+    ;;
+  *)
+    echo "Usage: <%= job_name %>_rack_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0