mamiya 0.0.1.alpha24 → 0.0.1.beta1

data/lib/mamiya/cli.rb

@@ -47,11 +47,6 @@ module Mamiya
       end
     end
 
-    desc "status", "Show status of servers"
-    def status
-      # TODO:
-    end
-
     desc "list-packages", "List packages in storage"
     method_option :name_only, aliases: '-n'
     def list_packages
@@ -88,14 +83,6 @@ module Mamiya
 
     # ---
 
-    desc "deploy PACKAGE", "Run build->push->distribute->prepare->finalize"
-    def deploy
-    end
-
-    desc "rollback", "Switch back to previous release then finalize"
-    def rollback
-    end
-
     desc "build", "Build package."
     method_option :build_from, aliases: %w(--source -f), type: :string
     method_option :build_to, aliases: %w(--destination -t), type: :string
@@ -216,12 +203,6 @@ module Mamiya
       @agent.run!
     end
 
-    # def worker
-    # end
-
-    # def event_handler
-    # end
-
     private
 
     def prepare_agent_behavior!
@@ -247,7 +228,7 @@ module Mamiya
 
     def config(dont_raise_error = false)
       return @config if @config
-      path = [options[:config], './mamiya.conf.rb', './config.rb'].compact.find { |_| File.exists?(_) }
+      path = [options[:config], './mamiya.conf.rb', './config.rb', '/etc/mamiya/config.rb'].compact.find { |_| File.exists?(_) }
 
       if path
         logger.debug "Using configuration: #{path}"

data/lib/mamiya/cli/client.rb

@@ -28,7 +28,7 @@ module Mamiya
       desc "show-package", "show package meta data"
       method_option :format, aliases: %w(-f), type: :string, default: 'pp'
       def show_package(package)
-        meta =  master_get("/packages/#{application}/#{package}")
+        meta = @meta =  master_get("/packages/#{application}/#{package}")
 
         case options[:format]
         when 'pp'
@@ -77,6 +77,7 @@ module Mamiya
         end
       end
 
+      # TODO: Deprecated. Remove this
       desc "show-distribution package", "Show package distribution status"
       method_option :format, aliases: %w(-f), type: :string, default: 'text'
       method_option :verbose, aliases: %w(-v), type: :boolean
@@ -127,6 +128,19 @@ not distributed: #{dist['not_distributed_count']} agents
         end
       end
 
+      desc "status [PACKAGE]", "Show application or package status"
+      method_option :format, aliases: %w(-f), type: :string, default: 'text'
+      method_option :labels, type: :string
+      method_option :show_done, type: :boolean, default: false
+      def status(package=nil)
+        if package
+          pkg_status package
+        else
+          app_status
+        end
+      end
+
+          
       desc "distribute package", "order distributing package to agents"
       method_option :labels, type: :string
       def distribute(package)
@@ -145,17 +159,136 @@ not distributed: #{dist['not_distributed_count']} agents
         p master_post("/packages/#{application}/#{package}/prepare", params.merge(type: :json))
       end
 
+      desc "switch PACKAGE", "order switching package to agents"
+      method_option :labels, type: :string
+      method_option :no_release, type: :boolean, default: false
+      def switch(package)
+        params = {no_release: options[:no_release]}
+        if options[:labels]
+          params[:labels] = Mamiya::Util::LabelMatcher.parse_string_expr(options[:labels])
+        end
+
+        p master_post("/packages/#{application}/#{package}/switch", params.merge(type: :json))
+      end
+
       desc "refresh", "order refreshing agent status"
       def refresh
         p master_post('/agents/refresh')
       end
 
-      desc "deploy PACKAGE", "Run distribute->prepare->finalize"
-      def deploy
+      desc "deploy PACKAGE", "Prepare, then switch"
+      method_option :labels, type: :string
+      method_option :no_release, type: :boolean, default: false
+      method_option :config, aliases: '-C', type: :string
+      method_option :no_switch, type: :boolean, default: false
+      def deploy(package)
+        @deploy_exception = nil
+        # TODO: move this run on master node side
+        puts "=> Deploying #{application}/#{package}"
+        puts " * with labels: #{options[:labels].inspect}" if options[:labels] && !options[:labels].empty?
+
+        show_package(package)
+
+        if config
+          config.set :application, application
+          config.set :package_name, package
+          config.set :package, @meta
+
+          config.before_deploy_or_rollback[]
+          config.before_deploy[]
+        end
+
+        do_prep = -> do
+          puts "=> Preparing..."
+          prepare(package)
+        end
+
+        do_prep[]
+
+        puts " * Wait until prepared"
+        puts ""
+
+        i = 0
+        loop do
+          i += 1
+          do_prep[] if i % 25 == 0
+
+          s = pkg_status(package, :short)
+          puts ""
+          break if 0 < s['participants_count'] && s['participants_count'] == s['prepare']['done'].size
+          sleep 2
+        end
+
+        ###
+
+        unless options[:no_switch]
+          puts "=> Switching..."
+          switch(package)
+
+          puts " * Wait until switch"
+          puts ""
+          loop do
+            s = pkg_status(package, :short)
+            puts ""
+            break if s['participants_count'] == s['switch']['done'].size
+            sleep 2
+          end
+        end
+      rescue Exception => e
+        @deploy_exception = e
+        $stderr.puts "ERROR: #{e.inspect}"
+        $stderr.puts "\t#{e.backtrace("\n\t")}"
+      ensure
+        config.after_deploy[@deploy_exception] if config
+        config.after_deploy_or_rollback[@deploy_exception] if config
+        puts "=> Done."
       end
 
       desc "rollback", "Switch back to previous release then finalize"
+      method_option :labels, type: :string
+      method_option :no_release, type: :boolean, default: false
+      method_option :config, aliases: '-C', type: :string
       def rollback
+        @deploy_exception = nil
+        # TODO: move this run on master node side
+        appstatus = master_get("/applications/#{application}/status", options[:labels] ? {labels: options[:labels]} : {})
+        package = appstatus['common_previous_release']
+
+        unless package
+          raise 'there is no common_previous_release for specified application'
+        end
+
+        puts "=> Rolling back #{application} to #{package}"
+        puts " * with labels: #{options[:labels].inspect}" if options[:labels] && !options[:labels].empty?
+
+        show_package(package)
+
+        if config
+          config.set :application, application
+          config.set :package_name, package
+          config.set :package, @meta
+
+          config.before_deploy_or_rollback[]
+          config.before_rollback[]
+        end
+
+        switch(package)
+
+        puts " * Wait until switch"
+        puts ""
+        loop do
+          s = pkg_status(package, :short)
+          puts ""
+          break if 0 < s['participants_count'] && s['participants_count'] == s['switch']['done'].size
+          sleep 2
+        end
+      rescue Exception => e
+        @deploy_exception = e
+        raise e
+      ensure
+        config.after_rollback[@deploy_exception] if config
+        config.after_deploy_or_rollback[@deploy_exception] if config
+        puts "=> Done."
       end
 
       desc "join HOST", "let serf to join to HOST"
@@ -174,6 +307,17 @@ not distributed: #{dist['not_distributed_count']} agents
         options[:application] or fatal!('specify application')
       end
 
+      def config
+        return @config if @config
+        path = [options[:config], './mamiya.conf.rb', './config.rb', '/etc/mamiya/config.rb'].compact.find { |_| File.exists?(_) }
+
+        if path
+          @config = Mamiya::Configuration.new.load!(File.expand_path(path))
+        end
+
+        @config
+      end
+
       def master_get(path, params={})
         path += "?#{Rack::Utils.build_query(params)}" unless params.empty?
         master_http.start do |http|
@@ -217,6 +361,84 @@ not distributed: #{dist['not_distributed_count']} agents
         fatal! 'specify master URL via --master(-u) option or $MAMIYA_MASTER_URL' unless url
         URI.parse(url)
       end
+
+      def app_status
+        params = options[:labels] ? {labels: options[:labels]} : {}
+        status = master_get("/applications/#{application}/status", params)
+
+        case options[:format]
+        when 'json'
+          require 'json'
+          puts status.to_json
+          return
+
+        when 'yaml'
+          require 'yaml'
+          puts status.to_yaml
+          return
+
+        end
+
+        puts <<-EOF
+at: #{Time.now.inspect}
+application: #{application}
+participants: #{status['participants_count']} agents
+
+major_current: #{status['major_current']}
+currents:
+#{status['currents'].sort_by { |pkg, as| -(as.size) }.flat_map { |pkg, as|
+["  - #{pkg} (#{as.size} agents)"] + (pkg == status['major_current'] ? ["    + (omitted)"] : as.map{ |_| "    * #{_['name']}" })
+}.join("\n")}
+
+common_previous_release: #{status['common_previous_release']}
+common_releases:
+#{status['common_releases'].map { |_| _.prepend('  - ') }.join("\n")}
+        EOF
+      end
+
+      def pkg_status(package, short=false)
+        params = options[:labels] ? {labels: options[:labels]} : {}
+        status = master_get("/packages/#{application}/#{package}/status", params)
+
+        case options[:format]
+        when 'json'
+          require 'json'
+          puts status.to_json
+          return
+
+        when 'yaml'
+          require 'yaml'
+          puts status.to_yaml
+          return
+
+        end
+
+        total = status['participants_count']
+
+        if short
+          puts "#{Time.now.strftime("%H:%M:%S")}  app:#{application} pkg:#{package}  agents:#{total}"
+        else
+          puts <<-EOF
+at: #{Time.now.inspect}
+package: #{application}/#{package}
+status: #{status['status'].join(',')}
+
+participants: #{total} agents
+
+          EOF
+        end
+
+          %w(fetch prepare switch).each do |key|
+            status[key].tap do |st|
+              puts "#{key}: queued=#{st['queued'].size}, working=#{st['working'].size}, done=#{st['done'].size}"
+              puts "  * queued:  #{st['queued'].join(', ')}" if !st['queued'].empty? && st['queued'].size != total
+              puts "  * working: #{st['working'].join(', ')}" if !st['working'].empty? && st['working'].size != total
+              puts "  * done:    #{st['done'].join(', ')}" if !st['done'].empty? && options[:show_done] && st['done'].size != total
+            end
+          end
+
+        status
+      end
     end
 
    desc "client", "client for master"

data/lib/mamiya/configuration.rb

@@ -20,6 +20,8 @@ module Mamiya
     set_default :fetch_sleep, 12
     set_default :keep_packages, 3
     set_default :keep_prereleases, 3
+    set_default :keep_releases, 3
+    set_default :applications, {}
 
     # master
     set_default :master, {monitor: {refresh_interval: nil}} # TODO: don't nest
@@ -27,6 +29,15 @@ module Mamiya
 
     add_hook :labels, chain: true
 
+    add_hook :before_deploy_or_rollback
+    add_hook :after_deploy_or_rollback
+
+    add_hook :before_deploy
+    add_hook :after_deploy
+
+    add_hook :before_rollback
+    add_hook :after_rollback
+
     def storage_class
       Storages.find(self[:storage][:type])
     end
@@ -38,6 +49,13 @@ module Mamiya
     def prereleases_dir
       self[:prereleases_dir] && Pathname.new(self[:prereleases_dir])
     end
+
+    # XXX: `config.app(app_name).deploy_to` form is better?
+    def deploy_to_for(app)
+      # XXX: to_sym
+      application = applications[app.to_sym] || applications[app.to_s]
+      application && application[:deploy_to] && Pathname.new(application[:deploy_to])
+    end
   end
 end
 

data/lib/mamiya/master/agent_monitor.rb

@@ -4,6 +4,8 @@ require 'thread'
 
 require 'mamiya/master'
 require 'mamiya/master/agent_monitor_handlers'
+require 'mamiya/master/package_status'
+require 'mamiya/master/application_status'
 
 module Mamiya
   class Master
@@ -17,6 +19,8 @@ module Mamiya
       PACKAGES_QUERY = 'mamiya:packages'.freeze
       DEFAULT_INTERVAL = 60
 
+      PACKAGE_STATUS_KEYS = %w(packages prereleases releases currents).map(&:freeze).freeze
+
       def initialize(master, raise_exception: false)
         @master = master
         @interval = (master.config[:master] && 
@@ -47,6 +51,14 @@ module Mamiya
         end
       end
 
+      def package_status(app, pkg, labels: nil)
+        PackageStatus.new(self, app, pkg, labels: labels)
+      end
+
+      def application_status(app, labels: nil)
+        ApplicationStatus.new(self, app, labels: labels)
+      end
+
       def start!
         @thread ||= Thread.new do
           loop do
@@ -140,8 +152,9 @@ module Mamiya
             begin
               resp = JSON.parse(json)
 
-              new_statuses[name]['packages'] = resp['packages']
-              new_statuses[name]['prereleases'] = resp['prereleases']
+              PACKAGE_STATUS_KEYS.each do |k|
+                new_statuses[name][k] = resp[k]
+              end
             rescue JSON::ParserError => e
               logger.warn "Failed to parse packages from #{name}: #{e.message}"
               next
@@ -149,8 +162,10 @@ module Mamiya
           end
 
           (new_statuses.keys - packages_response["Responses"].keys).each do |name|
-            if @statuses[name] && @statuses[name]['packages']
-              new_statuses[name]['packages'] = @statuses[name]['packages']
+            PACKAGE_STATUS_KEYS.each do |k|
+              if @statuses[name] && @statuses[name][k]
+                new_statuses[name][k] = @statuses[name][k]
+              end
             end
           end
 

data/lib/mamiya/master/agent_monitor_handlers.rb

@@ -72,6 +72,10 @@ module Mamiya
         end
       end
 
+      def task___switch__finish(status, task)
+        status['currents'] ||= {}
+        status['currents'][task['app']] = task['pkg']
+      end
 
 
       def pkg__remove(status, payload, event)
@@ -85,6 +89,12 @@ module Mamiya
         prereleases = status['prereleases'][payload['app']]
         prereleases.delete(payload['pkg']) if prereleases
       end
+
+      def release__remove(status, payload, event)
+        status['releases'] ||= {}
+        releases = status['releases'][payload['app']]
+        releases.delete(payload['pkg']) if releases
+      end
     end
   end
 end

data/lib/mamiya/master/application_status.rb

@@ -0,0 +1,72 @@
+require 'mamiya/master'
+
+module Mamiya
+  class Master
+    ##
+    # This class determines application cluster's status (what's majority package active, etc).
+    class ApplicationStatus
+      def initialize(agent_monitor, application, labels: nil)
+        @application = application
+        @agents = agent_monitor.statuses(labels: @labels).reject { |_, s| s['master'] }
+        @labels = labels
+      end
+
+      attr_reader :labels, :application, :agents
+
+      def to_hash
+        {
+          application: application,
+          labels: labels,
+          participants_count: participants.size,
+
+          major_current: major_current,
+          currents: currents,
+
+          common_releases: common_releases,
+          common_previous_release: common_previous_release,
+        }
+      end
+
+      def participants
+        @participants ||= Hash[agents.select do |name, status|
+          (status['currents']    && status['currents'][application])    || \
+          (status['releases']    && status['releases'][application])    || \
+          (status['prereleases'] && status['prereleases'][application]) || \
+          (status['packages']    && status['packages'][application])    || \
+
+          status['queues'].any? { |_, q|
+            (q['working'] && q['working']['app'] == application) || \
+            (q['queue']   && q['queue'].any? { |t| t['app'] == application })
+          }
+        end]
+    end
+
+      def currents
+        @currents ||= Hash[participants.group_by do |name, status|
+          status['currents'] && status['currents'][application]
+        end.map { |package, as| [package, as.map(&:first).sort] }]
+      end
+
+      def major_current
+        @major_current ||= currents.max_by { |package, as| as.size }[0]
+      end
+
+      def common_releases
+        #@common_releases ||= participants.map { |_| _[].select { |package, as| as.size > 2 }.map(&:first).compact.sort
+        @common_releases ||= participants.
+          map { |name, agent| agent['releases'] && agent['releases'][application] }.
+          compact.
+          inject(:&).
+          sort
+      end
+
+      def common_previous_release
+        idx = common_releases.index(major_current)
+        return if !idx || idx < 1
+
+        common_releases[idx-1]
+      end
+    end
+  end
+end
+