jets-fs 1.6.10

data/lib/jets/commands/clean.rb

@@ -0,0 +1,23 @@
+module Jets::Commands
+  class Clean < Jets::Commands::Base
+    autoload :Base, 'jets/commands/clean/base'
+    autoload :Log, 'jets/commands/clean/log'
+    autoload :Build, 'jets/commands/clean/build'
+
+    class_option :noop, type: :boolean, desc: "noop or dry-run mode"
+    class_option :mute, type: :boolean, desc: "mute output"
+    class_option :sure, type: :boolean, desc: "bypass are you sure prompt"
+
+    desc "log", "Cleans CloudWatch log groups assocated with app"
+    long_desc Help.text('clean:log')
+    def log
+      Log.new(options).clean
+    end
+
+    desc "build", "Cleans jets build"
+    long_desc Help.text('clean:build')
+    def build
+      Build.new(options).clean
+    end
+  end
+end

data/lib/jets/commands/clean/base.rb

@@ -0,0 +1,24 @@
+class Jets::Commands::Clean
+  class Base
+    def initialize(options={})
+      @options = options
+    end
+
+  private
+    def say(message)
+      prefix = 'NOOP ' if @options[:noop]
+      puts "#{prefix}#{message}" unless @options[:mute]
+    end
+
+    def are_you_sure?(message)
+      return true if @options[:sure]
+
+      puts "Are you sure that you want to #{message}? (y/N)"
+      yes = $stdin.gets.strip
+      unless yes =~ /^y/
+        puts "Phew that was close!"
+        exit 0
+      end
+    end
+  end
+end

data/lib/jets/commands/clean/build.rb

@@ -0,0 +1,13 @@
+require 'fileutils'
+
+class Jets::Commands::Clean
+  class Build < Base
+    def clean
+      are_you_sure?("delete /tmp/jets")
+
+      say "Removing /tmp/jets..."
+      FileUtils.rm_rf("/tmp/jets") unless @options[:noop]
+      say "Removed /tmp/jets"
+    end
+  end
+end

data/lib/jets/commands/clean/log.rb

@@ -0,0 +1,107 @@
+# The thing that limits this implementation is that there needs to be at least
+# one lambda function created from an internal jets function. Example:
+#
+#   /aws/lambda/demo-dev-2-jets-preheat_job-warm
+#   /aws/lambda/demo-dev-2-jets-public_controller-show
+#
+# We're doing this because JETS_ENV_EXTRA environments can create additional matching
+# log groups and we don't want to overly-aggressively delete them.
+#
+# The `keep_prefixes(log_group_names)` method calcuates the log groups to keep.
+class Jets::Commands::Clean
+  class Log < Base
+    extend Memoist
+    include Jets::AwsServices
+
+    def clean
+      are_you_sure?("delete CloudWatch logs")
+
+      say "Removing CloudWatch logs for #{prefix_guess}..."
+      log_groups.each do |g|
+        next if keep_log_group?(g.log_group_name)
+        logs.delete_log_group(log_group_name: g.log_group_name) unless @options[:noop]
+        say "Removed log group: #{g.log_group_name}"
+      end
+      say "Removed CloudWatch logs for #{prefix_guess}"
+    end
+
+    def clean_deploys
+      groups = deploy_log_groups.sort_by do |g|
+        g.log_group_name
+      end
+      # Keep the last 2 recent log groups so we can see the deleted logic
+      groups = groups[0..-3]
+      groups.each do |g|
+        logs.delete_log_group(log_group_name: g.log_group_name) unless @options[:noop]
+      end
+    end
+
+    def deploy_log_groups
+      log_groups.select do |g|
+        !keep_log_group?(g.log_group_name) &&
+        g.log_group_name.include?('jets-base-path')
+      end
+    end
+
+  private
+    def prefix_guess
+      Jets::Naming.parent_stack_name
+    end
+
+    def log_groups
+      groups, next_token = [], true
+      while next_token
+        next_token = nil if next_token == true # just at start the loop
+        resp = logs.describe_log_groups(
+          log_group_name_prefix: "/aws/lambda/#{prefix_guess}-",
+          next_token: next_token,
+        )
+        groups += resp.log_groups
+        next_token = resp.next_token
+      end
+      groups
+    end
+    memoize :log_groups
+
+    def log_group_names
+      log_groups.map(&:log_group_name)
+    end
+
+    def all_prefixes(log_group_names)
+      log_prefixes(log_group_names)
+    end
+    memoize :all_prefixes
+
+    # Check for the prefixes to keep. The slightly tricky thing to watch for is
+    # for the prefix matching addiitonal log groups that belong to other
+    # JETS_ENV_EXTRA=xxx created environments.
+    #
+    # We find and store the prefixes to keep so we don't over aggressively delete
+    # log groups.
+    def keep_prefixes(log_group_names)
+      names = log_group_names.reject do |name|
+        name =~ %r{/aws/lambda/#{prefix_guess}-jets}
+      end
+      log_prefixes(names)
+    end
+    memoize :keep_prefixes
+
+    # Strips -jets.* from the full log group name to leave only the prefix behind
+    def log_prefixes(names)
+      names = names.select do |name|
+        name.match(Regexp.new("#{prefix_guess}-.*jets"))
+      end
+      names.map do |name|
+        name.sub(/-jets.*/,'')
+      end.uniq.sort
+    end
+
+    # Check if it is safe to delete the log group
+    def keep_log_group?(log_group_name)
+      keep_prefixes = keep_prefixes(log_group_names)
+      !!keep_prefixes.detect do |keep_prefix|
+        log_group_name =~ Regexp.new(keep_prefix)
+      end
+    end
+  end
+end

data/lib/jets/commands/console.rb

@@ -0,0 +1,13 @@
+class Jets::Commands::Console
+  def self.run
+    puts Jets::Booter.message
+    Jets.boot
+
+    # Thanks: https://mutelight.org/bin-console
+    require "irb"
+    require "irb/completion"
+
+    ARGV.clear # https://stackoverflow.com/questions/33070092/irb-start-not-starting/33136762
+    IRB.start
+  end
+end

data/lib/jets/commands/db.rb

@@ -0,0 +1,15 @@
+module Jets::Commands
+  class Db < Jets::Commands::Base
+    autoload :Tasks, 'jets/commands/db/tasks'
+
+    desc "generate", "Creates a migration to change a db table"
+    long_desc Help.text('db:generate')
+    def generate(*args)
+      require "rails/generators"
+      require "rails/generators/active_record/migration/migration_generator"
+
+      generator = ActiveRecord::Generators::MigrationGenerator.new(args)
+      generator.create_migration_file
+    end
+  end
+end

data/lib/jets/commands/db/environment-task.rake

@@ -0,0 +1,3 @@
+task :environment do
+  Jets.boot
+end

data/lib/jets/commands/db/tasks.rb

@@ -0,0 +1,44 @@
+class Jets::Commands::Db::Tasks
+  # Ugly but it loads ActiveRecord database tasks
+  def self.load!
+    # Lazy require rails so Rails const is only defined in jets db:* tasks
+    require "rails"
+    require "active_record"
+    require "recursive-open-struct"
+
+    # Jets.boot # Jets.boot here screws up jets -h, the db_config doesnt seem to match exactly
+    # but seems to be working anyway.
+    db_configs = Jets.application.config.database
+    ActiveRecord::Tasks::DatabaseTasks.database_configuration = db_configs
+    ActiveRecord::Tasks::DatabaseTasks.migrations_paths = ["db/migrate"]
+    ActiveRecord::Tasks::DatabaseTasks.seed_loader = Seeder.new("#{Jets.root}/db/seeds.rb")
+
+    # Need to mock out the usage of Rails.application in:
+    # activerecord-5.1.4/lib/active_record/tasks/database_tasks.rb
+    Rails.application = RecursiveOpenStruct.new(
+      config: {
+        paths: {
+          db: ["db"],
+        }
+      },
+      paths: {
+        "db/migrate": ["db/migrate"]
+      }
+    )
+    load "active_record/railties/databases.rake"
+
+    load File.expand_path("../environment-task.rake", __FILE__)
+  end
+
+  # Thanks: https://stackoverflow.com/questions/19206764/how-can-i-load-activerecord-database-tasks-on-a-ruby-project-outside-rails/24840749
+  class Seeder
+    def initialize(seed_file)
+      @seed_file = seed_file
+    end
+
+    def load_seed
+      raise "Seed file '#{@seed_file}' does not exist" unless File.file?(@seed_file)
+      load @seed_file
+    end
+  end
+end

data/lib/jets/commands/dbconsole.rb

@@ -0,0 +1,131 @@
+# Thanks: Rails dbconsole_command.rb
+module Jets::Commands
+  class Dbconsole
+    def self.start(*args)
+      new(*args).start
+    end
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def start
+      ENV["RAILS_ENV"] ||= @options[:environment] || environment
+
+      case config["adapter"]
+      when /^(jdbc)?mysql/
+        args = {
+          "host"      => "--host",
+          "port"      => "--port",
+          "socket"    => "--socket",
+          "username"  => "--user",
+          "encoding"  => "--default-character-set",
+          "sslca"     => "--ssl-ca",
+          "sslcert"   => "--ssl-cert",
+          "sslcapath" => "--ssl-capath",
+          "sslcipher" => "--ssl-cipher",
+          "sslkey"    => "--ssl-key"
+        }.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
+
+        if config["password"] && @options["include_password"]
+          args << "--password=#{config['password']}"
+        elsif config["password"] && !config["password"].to_s.empty?
+          args << "-p"
+        end
+
+        args << config["database"]
+
+        find_cmd_and_exec(["mysql", "mysql5"], *args)
+
+      when /^postgres|^postgis/
+        ENV["PGUSER"]     = config["username"] if config["username"]
+        ENV["PGHOST"]     = config["host"] if config["host"]
+        ENV["PGPORT"]     = config["port"].to_s if config["port"]
+        ENV["PGPASSWORD"] = config["password"].to_s if config["password"] && @options["include_password"]
+        find_cmd_and_exec("psql", config["database"])
+
+      when "sqlite3"
+        args = []
+
+        args << "-#{@options['mode']}" if @options["mode"]
+        args << "-header" if @options["header"]
+        args << File.expand_path(config["database"], Jets.root)
+
+        find_cmd_and_exec("sqlite3", *args)
+
+      when "oracle", "oracle_enhanced"
+        logon = ""
+
+        if config["username"]
+          logon = config["username"]
+          logon << "/#{config['password']}" if config["password"] && @options["include_password"]
+          logon << "@#{config['database']}" if config["database"]
+        end
+
+        find_cmd_and_exec("sqlplus", logon)
+
+      when "sqlserver"
+        args = []
+
+        args += ["-D", "#{config['database']}"] if config["database"]
+        args += ["-U", "#{config['username']}"] if config["username"]
+        args += ["-P", "#{config['password']}"] if config["password"]
+
+        if config["host"]
+          host_arg = "#{config['host']}"
+          host_arg << ":#{config['port']}" if config["port"]
+          args += ["-S", host_arg]
+        end
+
+        find_cmd_and_exec("sqsh", *args)
+
+      else
+        abort "Unknown command-line client for #{config['database']}."
+      end
+    end
+
+    def config
+      @config ||= begin
+        if configurations[environment].blank?
+          raise ActiveRecord::AdapterNotSpecified, "'#{environment}' database is not configured. Available configuration: #{configurations.inspect}"
+        else
+          configurations[environment]
+        end
+      end
+    end
+
+    def environment
+      Jets.env.to_s
+    end
+
+    private
+      def configurations # :doc:
+        db_config = Jets.application.config.database
+        ActiveRecord::Base.configurations = db_config
+        ActiveRecord::Base.configurations
+      end
+
+      def find_cmd_and_exec(commands, *args) # :doc:
+        commands = Array(commands)
+
+        dirs_on_path = ENV["PATH"].to_s.split(File::PATH_SEPARATOR)
+        unless (ext = RbConfig::CONFIG["EXEEXT"]).empty?
+          commands = commands.map { |cmd| "#{cmd}#{ext}" }
+        end
+
+        full_path_command = nil
+        found = commands.detect do |cmd|
+          dirs_on_path.detect do |path|
+            full_path_command = File.join(path, cmd)
+            File.file?(full_path_command) && File.executable?(full_path_command)
+          end
+        end
+
+        if found
+          exec full_path_command, *args
+        else
+          abort("Couldn't find database client: #{commands.join(', ')}. Check your $PATH and try again.")
+        end
+      end
+  end
+end

data/lib/jets/commands/delete.rb

@@ -0,0 +1,143 @@
+class Jets::Commands::Delete
+  include Jets::AwsServices
+
+  def initialize(options)
+    @options = options
+  end
+
+  def run
+    puts("Deleting project...")
+    return if @options[:noop]
+
+    are_you_sure?
+
+    confirm_project_exists
+
+    # Must first remove all objects from s3 bucket in order to delete stack
+    puts "First, deleting objects in s3 bucket #{s3_bucket_name}" if s3_bucket_name
+    empty_s3_bucket
+
+    stack_in_progress?(parent_stack_name)
+
+    cfn.delete_stack(stack_name: parent_stack_name)
+    puts "Deleting #{Jets.config.project_namespace.color(:green)}..."
+
+    wait_for_stack if @options[:wait]
+
+    delete_logs
+
+    puts "Project #{Jets.config.project_namespace.color(:green)} deleted!"
+  end
+
+  def wait_for_stack
+    status = Jets::Cfn::Status.new(@options)
+    start_time = Time.now
+    status.wait
+    took = Time.now - start_time
+    puts "Time took for deletion: #{status.pretty_time(took).color(:green)}."
+  end
+
+  def delete_logs
+    puts "Deleting CloudWatch logs"
+    log = Jets::Commands::Clean::Log.new(mute: true, sure: true)
+    log.clean
+  end
+
+  def confirm_project_exists
+    begin
+      resp = cfn.describe_stacks(stack_name: parent_stack_name)
+    rescue Aws::CloudFormation::Errors::ValidationError
+      # Aws::CloudFormation::Errors::ValidationError is thrown when the stack
+      # does not exist
+      puts "The parent stack #{Jets.config.project_namespace.color(:green)} for the project #{Jets.config.project_name.color(:green)} does not exist. So it cannot be deleted."
+      exit 0
+    end
+  end
+
+  def empty_s3_bucket
+    return unless s3_bucket_name # Happens when minimal stack fails to build
+    return unless bucket_exists?(s3_bucket_name)
+
+    resp = s3.list_objects(bucket: s3_bucket_name)
+    if resp.contents.size > 0
+      # IE: objects = [{key: "objectkey1"}, {key: "objectkey2"}]
+      objects = resp.contents.map { |item| {key: item.key} }
+      s3.delete_objects(
+        bucket: s3_bucket_name,
+        delete: {
+          objects: objects,
+          quiet: false,
+        }
+      )
+      empty_s3_bucket # keep deleting objects until bucket is empty
+    end
+  end
+
+  def s3_bucket_name
+    return @s3_bucket_name if defined?(@s3_bucket_name)
+
+    resp = cfn.describe_stacks(stack_name: parent_stack_name)
+    outputs = resp.stacks[0].outputs
+    if outputs.empty?
+      @s3_bucket_name = false
+    else
+      @s3_bucket_name = outputs.find {|o| o.output_key == 'S3Bucket'}.output_value
+    end
+  end
+
+  # Thanks: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-example-does-bucket-exist.html
+  def bucket_exists?(bucket_name)
+    bucket_exists = false
+    begin
+      resp = s3.head_bucket(bucket: bucket_name, use_accelerate_endpoint: false)
+      bucket_exists = true
+    rescue
+    end
+    bucket_exists
+  end
+
+  def parent_stack_name
+    Jets::Naming.parent_stack_name
+  end
+
+  def are_you_sure?
+    if @options[:sure]
+      sure = 'y'
+    else
+      puts "Are you sure you want to want to delete the #{Jets.config.project_namespace.color(:green)} project? (y/N)"
+      sure = $stdin.gets
+    end
+
+    unless sure =~ /^y/
+      puts "Phew! Jets #{Jets.config.project_namespace.color(:green)} project was not deleted."
+      exit 0
+    end
+  end
+
+    # All CloudFormation states listed here: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-describing-stacks.html
+    #
+    # Returns resp so we can use it to grab data about the stack without calling api again.
+    def check_deleteable_status
+      return true if !stack_exists?(@parent_stack_name)
+
+      # Assumes stack exists
+      resp = cfn.describe_stacks(stack_name: @parent_stack_name)
+      status = resp.stacks[0].stack_status
+
+      return true if status == 'ROLLBACK_COMPLETE'
+
+      if status =~ /_IN_PROGRESS$/
+        puts "The '#{@parent_stack_name}' stack status is #{status}. " \
+             "It is not in an updateable status. Please wait until the stack is ready and try again.".color(:red)
+        exit 0
+      elsif resp.stacks[0].outputs.empty?
+        # This Happens when the miminal stack fails at the very beginning.
+        # There is no s3 bucket at all.  User should delete the stack.
+        puts "The minimal stack failed to create. Please delete the stack first and try again. " \
+        "You can delete the CloudFormation stack or use the `jets delete` command"
+        exit 0
+      else
+        true
+      end
+    end
+end