slugforge 4.0.0

data/lib/slugforge/commands/project.rb

@@ -0,0 +1,21 @@
+require 'slugforge/models/tag_manager'
+
+module Slugforge
+  module Commands
+    class Project < Slugforge::SubCommand
+      desc 'list', 'list available projects'
+      def list
+        logger.say "Available projects:"
+
+        tag_manager.projects.map do |project|
+          out = [set_color(project, :green)]
+          pc = tag_manager.slug_for_tag(project, 'production-current')
+          out << "(production-current: #{set_color(pc, :yellow)})" unless pc.nil?
+          out
+        end.sort.each do |o|
+          logger.say "  #{o * ' '}"
+        end
+      end
+    end
+  end
+end

data/lib/slugforge/commands/tag.rb

@@ -0,0 +1,148 @@
+module Slugforge
+  module Commands
+    class Tag < Slugforge::SubCommand
+      desc 'clean [ARGS]', 'remove tags that point to missing slugs (excluding production-current)'
+      def clean
+        verify_project_name!
+
+        tags = tag_manager.tags(project_name)
+        tags.delete('production-current')
+
+        bucket  # initialize value before using threads
+        logger.say_status :clean, "Reviewing #{tags.count} tags for #{project_name}", :cyan
+        results = tags.parallel_map do |tag|
+          begin
+            slug_file = tag_manager.slug_for_tag(project_name, tag)
+            if !slug_file.nil? && bucket.files.head(slug_file)
+              logger.say '.', :green, false
+              [tag, :valid]
+            else
+              if pretend?
+                [tag, :pretend]
+              else
+                tag_manager.delete_tag(project_name, tag)
+                logger.say '.', :red, false
+                [tag, :deleted]
+              end
+            end
+          rescue Excon::Errors::Forbidden
+            logger.say '.', :yellow, false
+            [tag, :forbidden]
+          end
+        end.sort {|a,b| b[1].to_s <=> a[1].to_s}
+
+        logger.say
+        results.each do |result|
+          tag, status = result
+          next if status == :valid
+          logger.say_status status, tag, (status == :deleted) ? :red : :yellow
+        end
+      end
+
+      desc 'clone <tag> <new_tag> [ARGS]', 'create a new tag with the same slug as an existing tag'
+      def clone(tag, new_tag)
+        verify_project_name!
+
+        slug_name = tag_manager.slug_for_tag(project_name, tag)
+        unless slug_name.nil?
+          tag_manager.clone_tag(project_name, tag, new_tag)
+          logger.say_json :project => project_name, :tag => new_tag, :slug => slug_name
+          logger.say_status :set, "#{project_name} #{new_tag} to Slug #{slug_name}"
+          true
+        else
+          logger.say_json :tag => tag, :exists => false
+          logger.say_status :clone, "could not find existing tag #{tag} for project '#{project_name}'", :red
+          false
+        end
+      end
+
+      desc 'history <tag> [ARGS]', 'show history of a project\'s tag'
+      def history(tag)
+        verify_project_name!
+
+        slug_names = tag_manager.slugs_for_tag(project_name, tag)
+        unless slug_names.empty?
+          logger.say_json :project => project_name, :tag => tag, :slug_names => slug_names, :exists => true
+          slug_names.each.with_index(0) do |slug_name, index|
+            logger.say_status (index == 0 ? 'current' : "-#{index}"), slug_name, :yellow
+          end
+        else
+          logger.say_json :tag => tag, :exists => false
+        end
+      end
+
+      desc 'list [ARGS]', 'list a project\'s tags'
+      def list
+        verify_project_name!
+
+        tags = tag_manager.tags(project_name)
+        pc = tags.delete('production-current')
+
+        if json?
+          logger.say_json tags
+        else
+          logger.say "Tags for #{project_name}"
+          logger.say_status :'production-current', tag_manager.slug_for_tag(project_name, 'production-current') unless pc.nil?
+
+          tag_list = tags.parallel_map do |tag|
+            [tag, tag_manager.slug_for_tag(project_name, tag)]
+          end
+          tag_list.sort {|a,b| b[1].to_s<=>a[1].to_s }.each do |tag, slug|
+            logger.say_status tag, slug, :yellow
+          end
+        end
+      end
+
+      desc 'migrate', 'migrate tags to new format', :hide => true
+      def migrate
+        metadata = JSON.parse(bucket.files.get('projects.json').body)
+        metadata.each do |project, data|
+          data['tags'].each do |tag, value|
+            puts "create_tag(#{project}, #{tag}, #{value['s3']})"
+            tag_manager.create_tag(project, tag, value['s3'])
+          end
+        end
+      end
+
+      desc 'show <tag> [ARGS]', 'show value of a project\'s tag'
+      def show(tag)
+        verify_project_name!
+
+        slug_name = tag_manager.slug_for_tag(project_name, tag)
+        unless slug_name.nil?
+          exists = !bucket.files.head(slug_name).nil?
+          logger.say_json :project => project_name, :tag => tag, :slug_name => slug_name, :exists => exists
+          logger.say_status tag, "#{slug_name} (#{exists ? "exists" : "missing"})", :yellow
+        else
+          logger.say_json :tag => tag, :exists => false
+        end
+      end
+
+      desc 'set <tag> <name_part>', 'update a tag to point to a slug in s3'
+      def set(tag, name_part)
+        verify_project_name!
+
+        slug = find_slug(name_part)
+
+        tag_manager.create_tag(project_name, tag, slug.key)
+        logger.say_json :project => project_name, :tag => tag, :slug => slug.key
+        logger.say_status :set, "#{project_name} #{tag} to Slug #{slug.key}"
+      end
+
+      desc 'delete <tag> [ARGS]', 'delete a tag'
+      option :yes, :type => :boolean, :aliases => '-y', :default => false,
+        :desc => 'answer "yes" to all questions'
+      def delete(tag)
+        verify_project_name!
+
+        if options[:yes] || (ask("Are you sure you wish to delete tag '#{tag}'? [Yn]").downcase != 'n')
+          tag_manager.delete_tag(project_name, tag)
+          logger.say_status :delete, "#{project_name} #{tag}"
+        else
+          logger.say_status :keep, "#{project_name} #{tag}"
+        end
+      end
+    end
+  end
+end
+

data/lib/slugforge/commands/wrangler.rb

@@ -0,0 +1,142 @@
+module Slugforge
+  module Commands
+    class Wrangler < Slugforge::SubCommand
+
+      desc 'push <file>', 'push a slug to S3'
+      option :tag, :type => :string, :aliases => '-t',
+        :desc => 'once pushed tag the slug with this tag.'
+      def push(file)
+        verify_project_name!
+        unless File.exist?(file)
+          raise error_class, "file does not exist"
+        end
+
+        dest = "#{project_name}/#{file}"
+        logger.say_status :upload, "slug #{file} to #{project_name}", :yellow
+        s3.put_object(aws_bucket, dest, File.read(file))
+        logger.say_status :uploaded, "slug saved"
+
+        if options[:tag]
+          logger.say_status :build, "applying tag '#{options[:tag]}' to your fancy new build", :green
+          invoke Slugforge::Commands::Tag, [:set, options[:tag], dest], []
+        end
+      end
+
+      desc 'pull [name_part]', 'pull a slug from S3 (most recent if no name part is specified)'
+      def pull(name_part)
+        verify_project_name!
+
+        slug = name_part ? find_slug(name_part) : find_latest_slug
+
+        logger.say_status :fetch, "#{slug.attributes[:name]} (#{slug.attributes[:pretty_length]})", :yellow
+        logger.say "Note: process will block until download completes."
+
+        # This should block until the body is downloaded.
+        # We open the file for writing afterwards to prevent creating empty files.
+        slug.body
+        File.open(slug.attributes[:name], 'w+') { |f| f.write(slug.body) }
+
+        logger.say_status :fetched, "pull complete, saved to #{slug.attributes[:name]}"
+      end
+
+      desc 'list [ARGS]', 'list published slugs for a project'
+      option :count, :type => :numeric, :aliases => '-c', :default => 10,
+        :desc => 'how many slugs to list'
+      option :sort, :type => :string, :aliases => '-s', :default => 'last_modified:desc',
+        :desc => 'change the sorting option (field:dir)'
+      option :all, :type => :boolean, :aliases => '-a', :default => false,
+        :desc => 'list all slugs'
+      def list
+        raise error_class, "count must be greater than 0" if !options[:all] && options[:count] <= 0
+
+        begin
+          project_name = self.project_name
+        rescue Thor::Error
+          # This is the only case where we don't care if project is not found.
+        end
+
+        slugs = (project_name.nil? ? self.files : self.slugs(project_name)).values
+        raise error_class, "No slugs found for #{project_name}" if slugs.first.nil?
+
+        total = slugs.length
+
+        sorting   = options[:sort].split(':')
+        field     = sorting.first.to_sym
+        direction = (sorting.last || 'desc')
+
+        unless slugs.first.class.attributes.include?(field)
+          raise error_class, "unknown attribute for sorting: #{field}. Available fields: #{slugs.first.class.attributes * ', '}"
+        end
+
+        slugs = slugs.sort_by { |f| f.attributes[field] }
+        slugs = slugs.reverse! if direction != 'asc'
+        slugs = slugs.slice(0...options[:count]) unless options[:all]
+
+        logger.say "Slugs for #{project_name} (#{slugs.size}", nil, false
+        logger.say " of #{total}", nil, false unless options[:all]
+        logger.say ")"
+
+        tag_manager.memoize_slugs_for_tags(project_name)
+        slugs.each do |slug|
+          tags = tag_manager.tags_for_slug(project_name, slug.key)
+          logger.say " #{tags.size > 0 ? ' (' + tags.join(', ') + ')' : ''} ", :yellow
+          logger.say "#{slug.attributes[:name]} ", :green
+          logger.say "- "
+          logger.say "#{slug.attributes[:pretty_age]} ago ", :cyan
+          logger.say "(#{set_color(slug.attributes[:pretty_length], :magenta)})"
+        end
+      end
+
+      desc 'delete <name_part>', 'delete a slug'
+      option :yes, :type => :boolean, :aliases => '-y', :default => false,
+        :desc => 'answer "yes" to all questions'
+      def delete(name_part)
+        slug = find_slug(name_part)
+        if options[:yes] || (ask("Are you sure you wish to delete '#{slug.attributes[:name]}'? [yN]").downcase == 'y')
+          slug.destroy
+          logger.say_status :destroy, slug.key, :red
+        else
+          logger.say_status :keep, slug.key, :green
+        end
+      end
+
+      desc 'purge', 'purge slugs for a project'
+      option :yes, :type => :boolean, :aliases => '-y', :default => false,
+        :desc => 'answer "yes" to all questions'
+      option :keep, :type => :numeric, :aliases => '-k', :default => 10,
+        :desc => 'number of slugs to keep'
+      option :all, :type => :boolean, :aliases => '-a', :default => false,
+        :desc => 'purge all slugs'
+      def purge
+        raise error_class, "keep must be greater than 0" if !options[:all] && options[:keep] <= 0
+
+        slugs = self.slugs(project_name).values
+
+        if options[:all]
+          if !options[:yes] && ask("Are you sure you wish to delete #{slugs.size} slugs for #{project_name}? [yN]").downcase == 'y'
+            options[:yes] = true
+          end
+
+          if options[:yes]
+            slugs.each do |slug|
+              slug.destroy
+              logger.say_status :destroy, slug.key, :red
+            end
+          end
+        elsif slugs.size > options[:keep]
+          slugs.slice(options[:keep]..-1).each do |slug|
+            if options[:yes] || (ask("Are you sure you wish to delete #{slug.attributes[:name]}? [Yn]").downcase != 'n')
+              slug.destroy
+              logger.say_status :destroy, slug.key, :red
+            else
+              logger.say_status :keep, slug.key, :green
+            end
+          end
+        else
+          logger.say "Aborting, only #{slugs.size} slugs for #{project_name} and we want to keep #{options[:keep]}"
+        end
+      end
+    end
+  end
+end
+

data/lib/slugforge/configuration.rb

@@ -0,0 +1,125 @@
+require 'singleton'
+require 'forwardable'
+require 'slugforge/slugins'
+
+module Slugforge
+  # Handles loading configuration data from files and the environment. Order of precedence:
+  #
+  # 1) ENV
+  # 2) `pwd`/.slugforge
+  # 3) $HOME/.slugforge
+  # 4) /etc/slugforge
+  #
+  # We load in reverse order, allowing us to simply overwrite values whenever found.
+  class Configuration
+    extend Forwardable
+
+    class << self
+      attr_accessor :configuration_files
+    end
+    self.configuration_files = [ '/etc/slugforge', File.join(ENV['HOME'], '.slugforge'), File.join(Dir.pwd, '.slugforge') ]
+
+    class <<self
+      def options
+        @options ||= {}
+      end
+
+      def option(name, config)
+        raise "configuration option #{name} has already been defined" if options.key?(name)
+
+        options[name] = config
+        define_method(name) { values[name] }
+      end
+    end
+
+    option :aws_access_key,    :key => 'aws.access_key',  :option => :'aws-access-key-id', :env => 'AWS_ACCESS_KEY_ID'
+    option :aws_secret_key,    :key => 'aws.secret_key',  :option => :'aws-secret-key',    :env => 'AWS_SECRET_ACCESS_KEY'
+    option :aws_region,        :key => 'aws.region',      :option => :'aws-region',        :env => 'AWS_DEFAULT_REGION', :default => 'us-east-1'
+    option :slug_bucket,       :key => 'aws.slug_bucket', :option => :'slug-bucket',       :env => 'SLUG_BUCKET'
+    option :aws_session_token, :option => :'aws-session-token'
+
+    option :project, :key => 'slugforge.project', :option => :project, :env => 'SLUGFORGE_PROJECT'
+
+    option :ssh_username, :key => 'ssh.username', :option => :'ssh-username', :env => 'SSH_USERNAME'
+
+    option :disable_slugins, :key => 'disable_slugins', :option => :'disable-slugins', :env => 'DISABLE_SLUGINS'
+
+    attr_reader :values
+
+    def initialize(options = {})
+      @slugin_manager = SluginManager.new
+      self.load
+      update_from_options options
+    end
+
+    def_delegators :@slugin_manager, :load_slugins, :locate_slugins, :slugins
+
+    # Get a hash of all options with default values. The list of values is initialized with the result.
+    def defaults
+      @values = Hash[self.class.options.select { |_, c| c.key?(:default) }.map { |n,c| [n, c[:default]] }].merge(@values)
+    end
+
+    def activate_slugins
+      @slugin_manager.activate_slugins(self) unless disable_slugins
+    end
+
+    protected
+    def load
+      # Read configuration files to load list of slugins. Load the slugin classes so that
+      # their configuration options are added and reload the configs to populate the new
+      # options.
+      @values = {}
+      load_configuration_files
+      defaults
+
+      locate_slugins
+      #TODO: disable individual slugins via configuration
+      load_slugins unless disable_slugins
+
+      load_configuration_files
+      read_env
+    end
+
+    def load_configuration_files
+      self.class.configuration_files.each { |f| read_yaml f }
+    end
+
+    # Attempt to read option keys from a YAML file
+    def read_yaml(path)
+      return unless File.exist?(path)
+      source = YAML.load_file(path)
+      return unless source.is_a?(Hash)
+
+      update_with { |config| read_yaml_key(source, config[:key]) }
+    end
+
+    # Split a dot-separated key and locate the value from a hash loaded by YAML.
+    #   eg. `aws.bucket` looks for `source['aws']['bucket']`.
+    def read_yaml_key(source, key)
+      return unless key.is_a?(String)
+      paths = key.split('.')
+      source = source[paths.shift] until paths.empty? || source.nil?
+      source
+    end
+
+    # Attempt to read option keys from the environment
+    def read_env
+      update_with { |config| config[:env] && ENV[config[:env]] }
+    end
+
+    # Update values with a hash of options.
+    def update_from_options(options={})
+      update_with { |config| config[:option] && options[config[:option]] }
+    end
+
+    # For every option we yield the configuration and expect a value back. If the block returns a value we set the
+    # option to it.
+    def update_with(&blk)
+      self.class.options.each do |name, config|
+        value = yield(config)
+        @values[name] = value unless value.nil?
+      end
+    end
+  end
+end
+

data/lib/slugforge/helper.rb

@@ -0,0 +1,186 @@
+require 'slugforge/helper/build'
+require 'slugforge/helper/config'
+require 'slugforge/helper/enumerable'
+require 'slugforge/helper/fog'
+require 'slugforge/helper/git'
+require 'slugforge/helper/path'
+require 'slugforge/helper/project'
+require 'slugforge/models/logger'
+
+module Slugforge
+  module Helper
+    def self.included(base)
+      base.send(:include, Slugforge::Helper::Build)
+      base.send(:include, Slugforge::Helper::Config)
+      base.send(:include, Slugforge::Helper::Fog)
+      base.send(:include, Slugforge::Helper::Git)
+      base.send(:include, Slugforge::Helper::Path)
+      base.send(:include, Slugforge::Helper::Project)
+    end
+
+    protected
+    def force?
+      options[:force] == true
+    end
+
+    def json?
+      options[:json] == true
+    end
+
+    def pretend?
+      test? || options[:pretend] == true
+    end
+
+    def test?
+      options[:test] == true
+    end
+
+    def notifications_enabled?
+      test? || !pretend?
+    end
+
+    def quiet?
+      options[:quiet] == true
+    end
+
+    def verbose?
+      options[:verbose] == true
+    end
+
+    def error_class
+      json? ? JsonError : Thor::Error
+    end
+
+    def elapsed_time
+      format_age @command_start_time
+    end
+
+    def logger
+      @logger ||= begin
+                    log_level = if quiet?
+                                  :quiet
+                                elsif verbose?
+                                  :verbose
+                                elsif json?
+                                  :json
+                                end
+                    Slugforge::Logger.new(self.shell, log_level)
+                  end
+    end
+
+    def execute(cmd)
+      unless pretend?
+        if ruby_version_specified?
+          cmd = "rvm #{options[:ruby]} do #{cmd}"
+        elsif has_ruby_version_file?
+          cmd = "rvm #{get_ruby_version_from_file} do #{cmd}"
+        end
+
+        # in thor, if capture is set, it uses backticks to run the command which returns a string.
+        # Otherwise they use `system` which returns true or nil if it worked. So check the return value
+        # and if it used backticks examine $? which keeps the result of the last command run to see
+        # if it worked.
+        returned = run(cmd, {:verbose => verbose?, :capture => verbose?})
+        if returned.is_a?(String)
+          process_status = $?
+          logger.say_status :run, "Command result #{process_status.to_s}. Command output: #{returned}", :green
+          return process_status.success?
+        end
+
+        return returned
+      end
+      true
+    end
+
+    def with_env(env={}, &blk)
+      original = ENV.to_hash
+      ENV.replace(original.merge(env))
+
+      # Ensure rbenv isn't locked into a version
+      if ENV['RBENV_VERSION']
+        ENV.delete('RBENV_VERSION')
+
+        # when you use a shim provided by rbenv the $PATH is modified to point to the proper ruby version so shims are
+        # bypassed. We need to remove those path entries to totally unset rbenv. We remove every .rbenv path _except_
+        # shims so it can still use the correct version defined by .ruby-version.
+        paths = ENV['PATH'].split(':').reject do |path|
+          path =~ /\.rbenv\/(\w+)/ && !%w(shims bin).include?($1)
+        end
+        ENV['PATH'] = paths * ':'
+      end
+
+      # Ensure RVM isn't locked into a version
+      ENV.delete('RUBY_VERSION')
+      yield
+    ensure
+      ENV.replace(original)
+    end
+
+    def with_gemfile(gemfile, &blk)
+      with_env('BUNDLE_GEMFILE' => gemfile) do
+        ENV.delete('GEM_HOME')
+        ENV.delete('GEM_PATH')
+        ENV.delete('RUBYOPT')
+        yield
+      end
+    end
+
+    def delete_option(options, option)
+      result = options.dup
+      index = result.index(option)
+      if index
+        result.delete_at(index)
+        result.delete_at(index)
+      end
+      result
+    end
+
+    def delete_switch(options, switch)
+      result = options.dup
+      index = result.index(option)
+      result.delete_at(index) if index
+      result
+    end
+
+    def format_size(size)
+      units = %w(B KB MB GB TB)
+      size, unit = units.reduce(size.to_f) do |(fsize, _), utype|
+        fsize > 512 ? [fsize / 1024, utype] : (break [fsize, utype])
+      end
+
+      "#{size > 9 || size.modulo(1) < 0.1 ? '%d' : '%.1f'} %s" % [size, unit]
+    end
+
+    SEGMENTS = {
+      :year   => 60 * 60 * 24 * 365,
+      :month  => 60 * 60 * 24 * 7 * 4,
+      :week   => 60 * 60 * 24 * 7,
+      :day    => 60 * 60 * 24,
+      :hour   => 60 * 60
+    }
+
+    def format_age(age)
+      age = Time.now - age
+      segments = {}
+
+      SEGMENTS.each do |segment, length|
+        next unless age >= length
+
+        segments[segment] = (age / length).floor
+        age = age % length
+      end
+
+      # We only show Minutes or Seconds if there is no other scope
+      if segments.empty?
+        segments[:minute] = (age / 60).floor if age >= 60
+        segments[:second] = (age % 60).floor
+      end
+
+      segments.map do |seg, size|
+        plural = 's' if size != 1
+        "#{size} #{seg}#{plural}"
+      end.join(', ')
+    end
+  end
+end
+