mamiya 0.0.1.alpha24 → 0.0.1.beta1

data/lib/mamiya/master/package_status.rb

@@ -0,0 +1,178 @@
+require 'mamiya/master'
+
+module Mamiya
+  class Master
+    ##
+    # This class determines distribution and releasing status of given package using given AgentMonitor.
+    class PackageStatus
+      def initialize(agent_monitor, application, package, labels: nil)
+        @application = application
+        @package = package
+        @agents = agent_monitor.statuses(labels: @labels).reject { |_, s| s['master'] }
+        @labels = labels
+      end
+
+      attr_reader :labels, :application, :package, :agents
+
+      def to_hash
+        {
+          application: application,
+          package: package,
+          labels: labels,
+          status: status,
+          participants_count: participants.size,
+          non_participants: non_participants,
+          active: current_agents,
+          fetch: {
+            queued: fetch_queued_agents,
+            working: fetching_agents,
+            done: fetched_agents,
+          },
+          prepare: {
+            queued: prepare_queued_agents,
+            working: preparing_agents,
+            done: prepared_agents,
+          },
+          switch: {
+            queued: switch_queued_agents,
+            working: switching_agents,
+            done: current_agents,
+          },
+        }
+      end
+
+      def status
+        [].tap do |s|
+          working = false
+
+          case
+          when fetched_agents == agents.keys
+            s << :distributed
+          when !fetching_agents.empty? || !fetch_queued_agents.empty?
+            working = true
+            s << :distributing
+          end
+          if fetched_agents != agents.keys && !fetched_agents.empty?
+            s << :partially_distributed
+          end
+
+          # TODO: FIXME: tests
+          case
+          when prepared_agents == agents.keys
+            s << :prepared
+          when !preparing_agents.empty? || !prepare_queued_agents.empty?
+            working = true
+            s << :preparing
+          end
+          if prepared_agents != agents.keys && !prepared_agents.empty?
+            s << :partially_prepared
+          end
+
+          # TODO: FIXME: tests
+          case
+          when current_agents == agents.keys
+            s << :active
+          when !current_agents.empty? || !switch_queued_agents.empty?
+            working = true
+            s << :switching
+          end
+          if current_agents != agents.keys && !current_agents.empty?
+            s << :partially_active
+          end
+
+          s << :unknown if s.empty?
+          s << :working if working
+        end
+      end
+
+      def fetch_queued_agents
+        @fetch_queued_agents ||= agents.select do |name, agent|
+          queue = agent['queues'] && agent['queues']['fetch'] && agent['queues']['fetch']['queue']
+          queue && queue.any? { |task|
+            app_and_pkg == task.values_at('app', 'pkg')
+          }
+        end.keys - fetched_agents
+      end
+
+      def fetching_agents
+        @fetching_agents ||= agents.select do |name, agent|
+          task = agent['queues'] && agent['queues']['fetch'] && agent['queues']['fetch'] && agent['queues']['fetch']['working']
+          task && app_and_pkg == task.values_at('app', 'pkg')
+        end.keys - fetched_agents
+      end
+
+      def fetched_agents
+        @fetched_agents ||= agents.select do |name, agent|
+          packages = agent['packages'] && agent['packages'][application]
+          packages && packages.include?(package)
+        end.keys
+      end
+
+      def prepare_queued_agents
+        @prepare_queued_agents ||= agents.select do |name, agent|
+          queue = agent['queues'] && agent['queues']['prepare'] && agent['queues']['prepare']['queue']
+          queue && queue.any? { |task|
+            app_and_pkg == task.values_at('app', 'pkg')
+          }
+        end.keys - prepared_agents
+      end
+
+      def preparing_agents
+        @preparing_agents ||= agents.select do |name, agent|
+          task = agent['queues'] && agent['queues']['prepare'] && agent['queues']['prepare'] && agent['queues']['prepare']['working']
+          task && app_and_pkg == task.values_at('app', 'pkg')
+        end.keys - prepared_agents
+      end
+
+      def prepared_agents
+        @prepare_agents ||= agents.select do |name, agent|
+          packages = agent['prereleases'] && agent['prereleases'][application]
+          packages && packages.include?(package)
+        end.keys
+      end
+
+      def switch_queued_agents
+        @switch_queued_agents ||= agents.select do |name, agent|
+          queue = agent['queues'] && agent['queues']['switch'] && agent['queues']['switch']['queue']
+          queue && queue.any? { |task|
+            app_and_pkg == task.values_at('app', 'pkg')
+          }
+        end.keys - current_agents
+      end
+
+      def switching_agents
+        @switching_agents ||= agents.select do |name, agent|
+          task = agent['queues'] && agent['queues']['switch'] && agent['queues']['switch'] && agent['queues']['switch']['working']
+          task && app_and_pkg == task.values_at('app', 'pkg')
+        end.keys - current_agents
+      end
+
+      def current_agents
+        @current_agents ||= agents.select do |name, agent|
+          current = agent['currents'] && agent['currents'][application]
+          current == package
+        end.keys
+      end
+
+      def participants
+        (fetch_queued_agents   + fetching_agents  + fetched_agents  + \
+        prepare_queued_agents + preparing_agents + prepared_agents + \
+        switch_queued_agents  + switching_agents + current_agents).uniq
+      end
+
+      def non_participants
+        agents.keys - participants
+      end
+
+      def reload
+        @agents = nil
+      end
+
+      private
+
+      def app_and_pkg
+        @app_and_pkg ||= [application, package]
+      end
+    end
+  end
+end

data/lib/mamiya/master/web.rb

@@ -40,6 +40,20 @@ module Mamiya
         "mamiya v#{Mamiya::VERSION}\n"
       end
 
+      get '/applications/:application/status' do
+        content_type :json
+
+        expr = params[:labels] && parse_label_matcher_expr(params[:labels])
+        appstatus = agent_monitor.application_status(params[:application], labels: expr)
+
+        if appstatus.participants.empty?
+          status 404
+          {error: 'not found'}.to_json
+        else
+          appstatus.to_hash.to_json
+        end
+      end
+
       get '/packages/:application' do
         content_type :json
         packages = storage(params[:application]).packages
@@ -93,8 +107,33 @@ module Mamiya
         end
       end
 
+      post '/packages/:application/:package/switch' do
+        if storage(params[:application]).meta(params[:package])
+          status 204
+          master.switch(params[:application], params[:package], labels: params['labels'], no_release: !!(params['no_release'] || params[:no_release]))
+        else
+          status 404
+          content_type :json
+          {error: 'not found'}.to_json
+        end
+      end
+
+      get '/packages/:application/:package/status' do
+        content_type :json
+        meta = storage(params[:application]).meta(params[:package])
+        unless meta
+          status 404
+          next {error: 'not found'}.to_json
+        end
+
+        expr = params[:labels] && parse_label_matcher_expr(params[:labels])
+        pkgstatus = agent_monitor.package_status(params[:application], params[:package], labels: expr)
+
+        pkgstatus.to_hash.to_json
+      end
+
       get '/packages/:application/:package/distribution' do
-        # TODO: filter with label
+        # TODO: Deprecated. Remove this
         content_type :json
         meta = storage(params[:application]).meta(params[:package])
         unless meta
@@ -102,59 +141,24 @@ module Mamiya
           next {error: 'not found'}.to_json
         end
 
+        expr = params[:labels] && parse_label_matcher_expr(params[:labels])
+        pkgstatus = agent_monitor.package_status(params[:application], params[:package], labels: expr)
+
         result = {
           application: params[:application],
           package: params[:package],
-          distributed: [],
-          fetching: [],
-          queued: [],
-          not_distributed: []
+          distributed: pkgstatus.fetched_agents,
+          fetching: pkgstatus.fetching_agents,
+          queued: pkgstatus.fetch_queued_agents,
+          not_distributed: pkgstatus.non_participants,
         }
-        if params[:labels]
-          expr = parse_label_matcher_expr(params[:labels])
-          statuses = agent_monitor.statuses(labels: expr)
-        else
-          statuses = agent_monitor.statuses
-        end
-
-        pkg_array = [params[:application], params[:package]]
-
-        statuses.each do |name, status|
-          next if status["master"]
-          queue = status["queues"] && status["queues"]["fetch"]
-          packages = status["packages"] && status["packages"][params[:application]]
-
-          task_matcher = -> (task) do
-             task["task"] == "fetch" &&
-             task["app"] == params[:application] &&
-             task["pkg"] == params[:package]
-          end
-
-
-          case
-          when packages && packages.include?(params[:package])
-
-            result[:distributed] << name
-
-          when queue && queue["working"] && task_matcher.call(queue['working'])
-
-            result[:fetching] << name
-
-          when queue['queue'].any?(&task_matcher)
-
-            result[:queued] << name
-
-          else
-            result[:not_distributed] << name
-          end
-        end
 
         result[:distributed_count] = result[:distributed].size
         result[:fetching_count] = result[:fetching].size
         result[:not_distributed_count] = result[:not_distributed].size
         result[:queued_count] = result[:queued].size
 
-        total = statuses.size
+        total = agent_monitor.statuses.size
 
         case
         when 0 < result[:queued_count] || 0 < result[:fetching_count]

data/lib/mamiya/steps/build.rb

@@ -128,7 +128,7 @@ module Mamiya
           logger.debug "Determining package name..."
           name = Dir.chdir(script.build_from) {
             script.package_name[
-              [Time.now.strftime("%Y-%m-%d_%H.%M.%S"), script.application]
+              [Time.now.strftime("%Y%m%d%H%M%S"), script.application]
             ].join('-')
           }
           logger.info "Package name determined: #{name}"

data/lib/mamiya/steps/prepare.rb

@@ -35,7 +35,7 @@ module Mamiya
       def script
         @target_script ||= Mamiya::Script.new.load!(
           target.join('.mamiya.script', target_meta['script'])).tap do |script|
-          # XXX: release_path is set by options[:target] but deploy_to is set by script?
+          script.set(:deploy_to, config.deploy_to_for(script.application))
           script.set(:release_path, target)
           script.set(:logger, logger)
         end

data/lib/mamiya/steps/switch.rb

@@ -45,10 +45,11 @@ module Mamiya
       # This class see target_dir's script
       alias given_script script 
 
+      # XXX: modulize?
       def script
         @target_script ||= Mamiya::Script.new.load!(
           target.join('.mamiya.script', target_meta['script'])).tap do |script|
-          # XXX: release_path is set by options[:target] but deploy_to is set by script?
+          script.set(:deploy_to, config.deploy_to_for(script.application))
           script.set(:release_path, target)
           script.set(:logger, logger)
         end

data/lib/mamiya/storages/filesystem.rb

@@ -42,7 +42,7 @@ module Mamiya
         package_path = File.join(destination, "#{package_name}.tar.gz")
         meta_path = File.join(destination, "#{package_name}.json")
 
-        if File.exists?(package_path) || File.exists?(meta_path)
+        if File.exists?(package_path) && File.exists?(meta_path)
           raise AlreadyFetched
         end
 

data/lib/mamiya/storages/mock.rb

@@ -41,7 +41,7 @@ module Mamiya
         package_path = File.join(destination, "#{package_name}.tar.gz")
         meta_path = File.join(destination, "#{package_name}.json")
 
-        if File.exists?(package_path) || File.exists?(meta_path)
+        if File.exists?(package_path) && File.exists?(meta_path)
           raise AlreadyFetched
         end
 

data/lib/mamiya/storages/s3.rb

@@ -48,7 +48,7 @@ module Mamiya
         package_path = File.join(dir, File.basename(package_key))
         meta_path = File.join(dir, File.basename(meta_key))
 
-        if File.exists?(package_path) || File.exists?(meta_path)
+        if File.exists?(package_path) && File.exists?(meta_path)
           raise AlreadyFetched
         end
 

data/lib/mamiya/storages/s3_proxy.rb

@@ -18,7 +18,7 @@ module Mamiya
         package_path = File.join(dir, File.basename(package_key))
         meta_path = File.join(dir, File.basename(meta_key))
 
-        if File.exists?(package_path) || File.exists?(meta_path)
+        if File.exists?(package_path) && File.exists?(meta_path)
           raise AlreadyFetched
         end
 

data/lib/mamiya/version.rb

@@ -1,3 +1,3 @@
 module Mamiya
-  VERSION = "0.0.1.alpha24"
+  VERSION = "0.0.1.beta1"
 end

data/spec/agent/actions_spec.rb

@@ -36,7 +36,6 @@ describe Mamiya::Agent::Actions do
         agent.distribute('myapp', 'mypkg', labels: ['foo'])
       end
     end
-
   end
 
   describe "#prepare" do
@@ -54,4 +53,30 @@ describe Mamiya::Agent::Actions do
       end
     end
   end
+
+
+  describe "#switch" do
+    it "sends switch request" do
+      expect(agent).to receive(:trigger).with('task', task: 'switch', app: 'myapp', pkg: 'mypkg', coalesce: false, no_release: false)
+
+      agent.switch('myapp', 'mypkg')
+    end
+
+    context "with no_release" do
+      it "sends switch request" do
+        expect(agent).to receive(:trigger).with('task', task: 'switch', app: 'myapp', pkg: 'mypkg', coalesce: false, no_release: true)
+
+        agent.switch('myapp', 'mypkg', no_release: true)
+      end
+
+    end
+
+    context "with labels" do
+      it "adds _labels on task" do
+        expect(agent).to receive(:trigger).with('task', task: 'switch', app: 'myapp', pkg: 'mypkg', _labels: ['foo'], coalesce: false, no_release: false)
+
+        agent.switch('myapp', 'mypkg', labels: ['foo'])
+      end
+    end
+  end
 end

data/spec/agent/tasks/clean_spec.rb

@@ -3,6 +3,7 @@ require 'spec_helper'
 require 'mamiya/agent/tasks/clean'
 require 'mamiya/agent/tasks/abstract'
 require 'mamiya/steps/fetch'
+require 'mamiya/configuration'
 
 describe Mamiya::Agent::Tasks::Clean do
   let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-tasks-clean-spec') }
@@ -10,10 +11,21 @@ describe Mamiya::Agent::Tasks::Clean do
 
   let(:packages_dir) { Pathname.new(tmpdir).join('packages').tap(&:mkdir) }
   let(:prereleases_dir) { Pathname.new(tmpdir).join('prereleases').tap(&:mkdir) }
+  
+  let(:deploy_to_a) {  Pathname.new(tmpdir).join('app_a').tap(&:mkdir)  }
+  let(:deploy_to_b) {  Pathname.new(tmpdir).join('app_b').tap(&:mkdir)  }
 
   let(:config) do
-    {packages_dir: packages_dir, keep_packages: 2,
-     prereleases_dir: prereleases_dir, keep_prereleases: 2,}
+    Mamiya::Configuration.new.tap do |c|
+      c.set :packages_dir, packages_dir
+      c.set :prereleases_dir, prereleases_dir 
+      c.set :keep_packages, 2
+      c.set :keep_prereleases, 2
+      c.set :keep_releases, 2
+
+      c.applications[:app_a] = {deploy_to: deploy_to_a}
+      c.applications[:app_b] = {deploy_to: deploy_to_b}
+    end
   end
 
   let(:agent) { double('agent', config: config) }
@@ -117,5 +129,79 @@ describe Mamiya::Agent::Tasks::Clean do
         )
       end
     end
+
+    describe "releases_dir" do
+      before do
+        # TODO: XXX: this may remove ongoing preparing somewhat
+        deploy_to_a.join('releases').tap do |path|
+          path.mkdir
+          path.join('1').mkdir
+          path.join('2').mkdir
+          path.join('3').mkdir
+        end
+
+        deploy_to_b.join('releases').tap do |path|
+          path.mkdir
+          path.join('1').mkdir
+          path.join('2').mkdir
+        end
+      end
+
+      it "cleans up" do
+        expect(agent).to receive(:trigger).with('release', action: 'remove', app: :app_a, pkg: '1', coalesce: false)
+
+        task.execute
+
+        existences = Hash[
+          [
+            deploy_to_a.join('releases', '1'),
+            deploy_to_a.join('releases', '2'),
+            deploy_to_a.join('releases', '3'),
+            deploy_to_b.join('releases', '1'),
+            deploy_to_b.join('releases', '2'),
+          ].map { |file|
+            [file, file.exist?]
+          }
+        ]
+
+        expect(existences).to eq(
+            deploy_to_a.join('releases', '1') => false,
+            deploy_to_a.join('releases', '2') => true,
+            deploy_to_a.join('releases', '3') => true,
+            deploy_to_b.join('releases', '1') => true,
+            deploy_to_b.join('releases', '2') => true,
+        )
+      end
+
+      context "with current release" do
+        before do
+          deploy_to_a.join('current').make_symlink deploy_to_a.join('releases', '1')
+        end
+
+        it "cleans up" do
+          task.execute
+
+          existences = Hash[
+            [
+              deploy_to_a.join('releases', '1'),
+              deploy_to_a.join('releases', '2'),
+              deploy_to_a.join('releases', '3'),
+              deploy_to_b.join('releases', '1'),
+              deploy_to_b.join('releases', '2'),
+            ].map { |file|
+              [file, file.exist?]
+            }
+          ]
+
+          expect(existences).to eq(
+              deploy_to_a.join('releases', '1') => true,
+              deploy_to_a.join('releases', '2') => true,
+              deploy_to_a.join('releases', '3') => true,
+              deploy_to_b.join('releases', '1') => true,
+              deploy_to_b.join('releases', '2') => true,
+          )
+        end
+      end
+    end
   end
 end