mamiya 0.0.1.alpha21 → 0.0.1.alpha22

data/lib/mamiya/cli/client.rb

@@ -7,6 +7,8 @@ require 'uri'
 require 'json'
 require 'thor'
 
+require 'mamiya/util/label_matcher'
+
 module Mamiya
   class CLI < Thor
     class Client < Thor
@@ -42,8 +44,10 @@ module Mamiya
       end
 
       desc "list-agents", 'list agents'
+      method_option :labels, type: :string
       def list_agents
-        payload = master_get("/agents")
+        params = options[:labels] ? {labels: options[:labels]} : {}
+        payload = master_get("/agents", params)
 
         agents = payload["agents"].keys
 
@@ -76,8 +80,10 @@ module Mamiya
       desc "show-distribution package", "Show package distribution status"
       method_option :format, aliases: %w(-f), type: :string, default: 'text'
       method_option :verbose, aliases: %w(-v), type: :boolean
+      method_option :labels, type: :string
       def show_distribution(package)
-        dist = master_get("/packages/#{application}/#{package}/distribution")
+        params = options[:labels] ? {labels: options[:labels]} : {}
+        dist = master_get("/packages/#{application}/#{package}/distribution", params)
 
         case options[:format]
         when 'json'
@@ -122,8 +128,21 @@ not distributed: #{dist['not_distributed_count']} agents
       end
 
       desc "distribute package", "order distributing package to agents"
+      method_option :labels, type: :string
       def distribute(package)
-        p master_post("/packages/#{application}/#{package}/distribute")
+        params = options[:labels] ?
+          {labels: Mamiya::Util::LabelMatcher.parse_string_expr(options[:labels])} : {}
+
+        p master_post("/packages/#{application}/#{package}/distribute", params, type: :json)
+      end
+
+      desc "prepare PACKAGE", "order preparing package to agents"
+      method_option :labels, type: :string
+      def prepare(package)
+        params = options[:labels] ?
+          {labels: Mamiya::Util::LabelMatcher.parse_string_expr(options[:labels])} : {}
+
+        p master_post("/packages/#{application}/#{package}/prepare", params, type: :json)
       end
 
       desc "refresh", "order refreshing agent status"
@@ -155,20 +174,29 @@ not distributed: #{dist['not_distributed_count']} agents
         options[:application] or fatal!('specify application')
       end
 
-      def master_get(path)
+      def master_get(path, params={})
+        path += "?#{Rack::Utils.build_query(params)}" unless params.empty?
         master_http.start do |http|
           JSON.parse http.get(path).tap(&:value).body
         end
       end
 
-      def master_post(path, data='')
+      def master_post(path, data='', type: :text)
         response = nil
         master_http.start do |http|
+          headers = {}
+
           if Hash === data
-            data = Rack::Utils.build_nested_query(data)
+            case type
+            when :json
+              data = data.to_json
+              headers['Content-Type'] = 'application/json'
+            when :text
+              data = Rack::Utils.build_nested_query(data)
+            end
           end
 
-          response = http.post(path, data)
+          response = http.post(path, data, headers)
           response.value
           response.code == '204' ? true : JSON.parse(response.body)
         end

data/lib/mamiya/cli.rb

@@ -6,8 +6,11 @@ require 'mamiya/logger'
 
 require 'mamiya/steps/build'
 require 'mamiya/steps/push'
+
 require 'mamiya/steps/fetch'
 require 'mamiya/steps/extract'
+require 'mamiya/steps/prepare'
+require 'mamiya/steps/switch'
 
 require 'mamiya/agent'
 require 'mamiya/master'
@@ -159,15 +162,30 @@ module Mamiya
     def distribute
     end
 
-    desc "prepare", "Prepare package on clients."
-    def prepare
+    desc "prepare TARGET", "Prepare package."
+    method_option :labels, type: :string
+    def prepare(target)
+      Mamiya::Steps::Prepare.new(
+        script: nil,
+        config: config,
+        target: target,
+        labels: labels,
+      ).run!
     end
 
-    desc "finalize", "Finalize (start) prepared package on clients."
-    def finalize
+    desc "switch TARGET", "Switch current dir then release."
+    method_option :no_release, type: :boolean, default: false
+    method_option :labels, type: :string
+    def switch(target)
+      Mamiya::Steps::Switch.new(
+        script: nil,
+        config: config,
+        target: target,
+        labels: labels,
+        no_release: options[:no_release],
+      ).run!
     end
 
-
     # ---
 
     desc "agent", "Start agent."
@@ -175,9 +193,11 @@ module Mamiya
     method_option :daemonize, aliases: '-D', type: :boolean, default: false
     method_option :log, aliases: '-l', type: :string
     method_option :pidfile, aliases: '-p', type: :string
+    method_option :labels, type: :array
     def agent
       prepare_agent_behavior!
       merge_serf_option!
+      override_labels!
 
       @agent = Agent.new(config, logger: logger)
       @agent.run!
@@ -256,6 +276,11 @@ module Mamiya
       end
     end
 
+    def labels
+      c = config(:no_error)
+      options[:labels] ? options[:labels].split(/,/).map(&:to_sym) : (c ? c.labels[[]] : [])
+    end
+
     def fatal!(message)
       logger.fatal message
       exit 1
@@ -272,6 +297,20 @@ module Mamiya
       end
     end
 
+    def override_labels!
+      return unless config(:no_error)
+      return unless options[:labels]
+
+      labels = options[:labels].flat_map{ |_| _.split(/,/) }.map(&:to_sym)
+      return if labels.empty?
+
+      config.labels do
+        labels
+      end
+
+      logger.info "Overriding labels: #{labels.inspect}"
+    end
+
     def application
       options[:application] || config[:application] || script.application
     end

data/lib/mamiya/configuration.rb

@@ -14,16 +14,28 @@ module Mamiya
 
     # agent
     set_default :packages_dir, nil
+    set_default :prereleases_dir, nil
     set_default :fetch_sleep, 12
     set_default :keep_packages, 3
+    set_default :keep_prereleases, 3
 
     # master
     set_default :master, {monitor: {refresh_interval: nil}} # TODO: don't nest
     set_default :web, {port: 7761, bind: '0.0.0.0', environment: :development} # TODO: IPv6
 
+    add_hook :labels, chain: true
+
     def storage_class
       Storages.find(self[:storage][:type])
     end
+
+    def packages_dir
+      self[:packages_dir] && Pathname.new(self[:packages_dir])
+    end
+
+    def prereleases_dir
+      self[:prereleases_dir] && Pathname.new(self[:prereleases_dir])
+    end
   end
 end
 

data/lib/mamiya/dsl.rb

@@ -1,5 +1,6 @@
 require 'mamiya/util/label_matcher'
 require 'thread'
+require 'pathname'
 
 module Mamiya
   class DSL
@@ -40,6 +41,8 @@ module Mamiya
       self.define_variable_accessor(k)
     end
 
+    set_default :_file, nil
+
     ##
     # Add hook point with name +name+.
     # This defines method with same name in class to call and define hooks.
@@ -101,9 +104,9 @@ module Mamiya
             @file = filename if filename
 
             if str && filename && lineno
-              self.instance_eval(str, filename, lineno)
+              self.instance_eval(str, filename.to_s, lineno)
             elsif str && filename
-              self.instance_eval(str, filename)
+              self.instance_eval(str, filename.to_s)
             elsif str
               self.instance_eval(str)
             end
@@ -118,6 +121,7 @@ module Mamiya
     ##
     # Evaluates specified file +file+ in DSL environment.
     def load!(file)
+      set :_file, Pathname.new(file)
       evaluate! File.read(file), file, 1
     end
 

data/lib/mamiya/helpers/git.rb

@@ -3,6 +3,10 @@ require 'time'
 set_default :git_remote, 'origin'
 set_default :commit, "#{self.git_remote}/HEAD"
 
+def git_managed?
+  system *%w(git rev-parse --git-dir), err: File::NULL, out: File::NULL
+end
+
 def git_ignored_files
   git_clean_out = `git clean -ndx`.lines
   prefix = /^Would (?:remove|skip repository) /
@@ -73,3 +77,23 @@ if options[:include_head_commit_to_meta]
     candidate.merge(git: git_head())
   end
 end
+
+options[:manage_script] = true unless options.key?(:manage_script)
+options[:script_auto_additionals] = true unless options.key?(:script_auto_additionals)
+if options[:manage_script] && _file
+  Dir.chdir(File.dirname(_file)) do
+    break unless git_managed?
+
+    script_git_head = git_head()
+    package_meta do |candidate|
+      candidate.merge(script_git: script_git_head)
+    end
+
+    if options[:script_auto_additionals]
+      files = `git ls-files`.lines.map(&:chomp).reject { |_| _ == File.basename(_file) }
+      set :script_additionals, (script_additionals || []) + files
+    end
+  end
+end
+
+

data/lib/mamiya/master/agent_monitor.rb

@@ -30,9 +30,22 @@ module Mamiya
         @failed_agents = [].freeze
         @statuses = {}
         @commit_lock = Mutex.new
+        @last_refresh_at = nil
       end
 
-      attr_reader :statuses, :agents, :failed_agents
+      attr_reader :agents, :failed_agents, :last_refresh_at
+
+      def statuses(labels: nil)
+        if labels
+          @statuses.select { |name, status|
+            status['labels'] &&
+              Mamiya::Util::LabelMatcher::Simple.new(status['labels']).
+              match?(labels)
+          }
+        else
+          @statuses
+        end
+      end
 
       def start!
         @thread ||= Thread.new do
@@ -119,7 +132,13 @@ module Mamiya
             next unless new_statuses[name]
 
             begin
-              new_statuses[name]['packages'] = JSON.parse(json)
+              resp = JSON.parse(json)
+              if resp.keys.sort == ['packages', 'prereleases']
+                new_statuses[name]['packages'] = resp['packages']
+                new_statuses[name]['prereleases'] = resp['prereleases']
+              else # TODO: Compatibility; should remove soon
+                new_statuses[name]['packages'] = resp
+              end
             rescue JSON::ParserError => e
               logger.warn "Failed to parse packages from #{name}: #{e.message}"
               next
@@ -153,6 +172,7 @@ module Mamiya
           @agents = new_agents.freeze
           @failed_agents = new_failed_agents.freeze
           @statuses = new_statuses
+          @last_refresh_at = Time.now
         }
 
         self

data/lib/mamiya/master/agent_monitor_handlers.rb

@@ -63,11 +63,28 @@ module Mamiya
         end
       end
 
+      def task___prepare__finish(status, task)
+        status['prereleases'] ||= {}
+        status['prereleases'][task['app']] ||= []
+
+        unless status['prereleases'][task['app']].include?(task['pkg'])
+          status['prereleases'][task['app']] << task['pkg']
+        end
+      end
+
+
+
       def pkg__remove(status, payload, event)
         status['packages'] ||= {}
         packages = status['packages'][payload['application']]
         packages.delete(payload['package']) if packages
       end
+
+      def prerelease__remove(status, payload, event)
+        status['prereleases'] ||= {}
+        prereleases = status['prereleases'][payload['app']]
+        prereleases.delete(payload['pkg']) if prereleases
+      end
     end
   end
 end

data/lib/mamiya/master/web.rb

@@ -1,6 +1,7 @@
 require 'mamiya/version'
 require 'mamiya/agent'
 require 'sinatra/base'
+require 'mamiya/util/label_matcher'
 require 'json'
 
 module Mamiya
@@ -18,6 +19,20 @@ module Mamiya
         def storage(app)
           master.storage(app)
         end
+
+        def parse_label_matcher_expr(str)
+          Mamiya::Util::LabelMatcher.parse_string_expr(str)
+        end
+      end
+
+      before do
+        if request.content_type == 'application/json'
+          begin
+            params.merge! JSON.parse(request.body.read)
+          rescue JSON::ParserError
+            halt :bad_request
+          end
+        end
       end
 
       get '/' do
@@ -58,7 +73,19 @@ module Mamiya
         # TODO: filter with label
         if storage(params[:application]).meta(params[:package])
           status 204
-          master.distribute(params[:application], params[:package])
+          master.distribute(params[:application], params[:package], labels: params['labels'])
+        else
+          status 404
+          content_type :json
+          {error: 'not found'}.to_json
+        end
+      end
+
+      post '/packages/:application/:package/prepare' do
+        # TODO: filter with label
+        if storage(params[:application]).meta(params[:package])
+          status 204
+          master.prepare(params[:application], params[:package], labels: params['labels'])
         else
           status 404
           content_type :json
@@ -83,7 +110,12 @@ module Mamiya
           queued: [],
           not_distributed: []
         }
-        statuses = agent_monitor.statuses
+        if params[:labels]
+          expr = parse_label_matcher_expr(params[:labels])
+          statuses = agent_monitor.statuses(labels: expr)
+        else
+          statuses = agent_monitor.statuses
+        end
 
         pkg_array = [params[:application], params[:package]]
 
@@ -148,7 +180,10 @@ module Mamiya
       end
 
       get '/agents' do
-        statuses = agent_monitor.statuses
+        expr = params[:labels] ? parse_label_matcher_expr(params[:labels]) : nil
+
+        last_refresh_at = agent_monitor.last_refresh_at
+        statuses = agent_monitor.statuses(labels: expr)
         members = agent_monitor.agents
         failed_agents = agent_monitor.failed_agents
 
@@ -170,10 +205,14 @@ module Mamiya
           agents[name]["status"] = status
         end
 
+        if params[:labels]
+          agents.select! { |k,v| v['status'] }
+        end
 
         content_type :json
 
         {
+          last_refresh_at: agent_monitor.last_refresh_at,
           agents: agents,
           failed_agents: failed_agents,
         }.to_json

data/lib/mamiya/master.rb

@@ -56,6 +56,10 @@ module Mamiya
       {name: serf.name, master: true}
     end
 
+    def labels
+      []
+    end
+
     private
 
     def init_serf

data/lib/mamiya/script.rb

@@ -12,16 +12,16 @@ module Mamiya
     add_hook :build
     add_hook :after_build
 
-    add_hook :before_distribute
-    add_hook :after_distribute
+    #add_hook :before_distribute
+    #add_hook :after_distribute
 
     add_hook :before_prepare
     add_hook :prepare
     add_hook :after_prepare
 
-    add_hook :before_finalize
-    add_hook :finalize
-    add_hook :after_finalize
+    add_hook :before_switch
+    add_hook :release
+    add_hook :after_switch
 
     add_hook :before_rollback
     add_hook :rollback
@@ -43,16 +43,16 @@ module Mamiya
     set_default :exclude_from_package, []
     set_default :dereference_symlinks, true
 
-    set_default :package_to, nil
-
     # TODO: use variable in config.yml
     set_default :deploy_to, nil
-    set_default :prepare_to, nil
 
     set_default :logger, Mamiya::Logger.new(outputs: [])
 
     set_default :skip_prepare_build, false
 
+    set_default :script_file, nil
+    set_default :script_additionals, []
+
     def run(*args, allow_failure: false)
       # TODO: Stop when fail
       actual = -> do
@@ -113,5 +113,25 @@ module Mamiya
       logger.info "$ cd #{args[0]}"
       Dir.chdir *args
     end
+
+    def build_from
+      self[:build_from] && Pathname.new(self[:build_from])
+    end
+
+    def deploy_to
+      self[:deploy_to] && Pathname.new(self[:deploy_to])
+    end
+
+    def release_path
+      self[:release_path] && Pathname.new(self[:release_path])
+    end
+
+    def shared_path
+      deploy_to && deploy_to.join('shared')
+    end
+
+    def current_path
+      deploy_to && deploy_to.join('current')
+    end
   end
 end

data/lib/mamiya/steps/abstract.rb

@@ -1,5 +1,6 @@
 require 'mamiya/script'
 require 'mamiya/logger'
+require 'mamiya/configuration'
 
 module Mamiya
   module Steps