mamiya 0.0.1.alpha21 → 0.0.1.alpha22

data/spec/agent/tasks/clean_spec.rb

@@ -8,11 +8,18 @@ describe Mamiya::Agent::Tasks::Clean do
   let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-tasks-clean-spec') }
   after { FileUtils.remove_entry_secure(tmpdir) if File.exist?(tmpdir) }
 
-  let(:config) { {packages_dir: tmpdir, keep_packages: 2} }
+  let(:packages_dir) { Pathname.new(tmpdir).join('packages').tap(&:mkdir) }
+  let(:prereleases_dir) { Pathname.new(tmpdir).join('prereleases').tap(&:mkdir) }
+
+  let(:config) do
+    {packages_dir: packages_dir, keep_packages: 2,
+     prereleases_dir: prereleases_dir, keep_prereleases: 2,}
+  end
 
   let(:agent) { double('agent', config: config) }
   let(:task_queue) { double('task_queue') }
 
+
   subject(:task) { described_class.new(task_queue, {}, agent: agent, raise_error: true) }
 
   it 'inherits abstract task' do
@@ -21,52 +28,94 @@ describe Mamiya::Agent::Tasks::Clean do
 
 
   describe "#execute" do
-    before do
-      path = Pathname.new(tmpdir)
-
-      path.join('a').mkdir
-      File.write path.join('a', "a.tar.gz"), "\n"
-      File.write path.join('a', "a.json"), "\n"
-      File.write path.join('a', "b.json"), "\n"
-      File.write path.join('a', "b.tar.gz"), "\n"
-      File.write path.join('a', "c.json"), "\n"
-      File.write path.join('a', "c.tar.gz"), "\n"
-      path.join('b').mkdir
-      File.write path.join('b', "a.tar.gz"), "\n"
-      File.write path.join('b', "a.json"), "\n"
-
-      path.join('c').mkdir
-      File.write path.join('c', "a.tar.gz"), "\n"
-      File.write path.join('c', "b.json"), "\n"
+    describe "packages" do
+      before do
+        path = packages_dir
+
+        path.join('a').mkdir
+        File.write path.join('a', "a.tar.gz"), "\n"
+        File.write path.join('a', "a.json"), "\n"
+        File.write path.join('a', "b.json"), "\n"
+        File.write path.join('a', "b.tar.gz"), "\n"
+        File.write path.join('a', "c.json"), "\n"
+        File.write path.join('a', "c.tar.gz"), "\n"
+        path.join('b').mkdir
+        File.write path.join('b', "a.tar.gz"), "\n"
+        File.write path.join('b', "a.json"), "\n"
+
+        path.join('c').mkdir
+        File.write path.join('c', "a.tar.gz"), "\n"
+        File.write path.join('c', "b.json"), "\n"
+      end
+
+      it "cleans up" do
+        expect(agent).to receive(:trigger).with('pkg', action: 'remove', application: 'a', package: 'a', coalesce: false)
+
+        task.execute
+
+        existences = Hash[
+          [
+            packages_dir.join('a', 'a.tar.gz'),
+            packages_dir.join('a', 'a.json'),
+            packages_dir.join('a', 'b.tar.gz'),
+            packages_dir.join('a', 'b.json'),
+            packages_dir.join('a', 'c.tar.gz'),
+            packages_dir.join('a', 'c.json'),
+          ].map { |file|
+            [file, file.exist?]
+          }
+        ]
+
+        expect(existences).to eq(
+          packages_dir.join('a', 'a.tar.gz') => false,
+          packages_dir.join('a', 'a.json') => false,
+          packages_dir.join('a', 'b.tar.gz') => true,
+          packages_dir.join('a', 'b.json') => true,
+          packages_dir.join('a', 'c.tar.gz') => true,
+          packages_dir.join('a', 'c.json') => true,
+        )
+      end
     end
 
-    it "cleans up" do
-      expect(agent).to receive(:trigger).with('pkg', action: 'remove', application: 'a', package: 'a', coalesce: false)
-
-      task.execute
-
-      path = Pathname.new(tmpdir)
-      existences = Hash[
-        [
-          path.join('a', 'a.tar.gz'),
-          path.join('a', 'a.json'),
-          path.join('a', 'b.tar.gz'),
-          path.join('a', 'b.json'),
-          path.join('a', 'c.tar.gz'),
-          path.join('a', 'c.json'),
-        ].map { |file|
-          [file, file.exist?]
-        }
-      ]
-
-      expect(existences).to eq(
-        path.join('a', 'a.tar.gz') => false,
-        path.join('a', 'a.json') => false,
-        path.join('a', 'b.tar.gz') => true,
-        path.join('a', 'b.json') => true,
-        path.join('a', 'c.tar.gz') => true,
-        path.join('a', 'c.json') => true,
-      )
+    describe "prereleases_dir" do
+      before do
+        path = prereleases_dir
+
+        # TODO: XXX: this may remove ongoing preparing somewhat
+        path.join('a').mkdir
+        path.join('a', '1').mkdir
+        path.join('a', '2').mkdir
+        path.join('a', '3').mkdir
+        path.join('b').mkdir
+        path.join('b', '1').mkdir
+        path.join('b', '2').mkdir
+      end
+
+      it "cleans up" do
+        expect(agent).to receive(:trigger).with('prerelease', action: 'remove', app: 'a', pkg: '1', coalesce: false)
+
+        task.execute
+
+        existences = Hash[
+          [
+            prereleases_dir.join('a', '1'),
+            prereleases_dir.join('a', '2'),
+            prereleases_dir.join('a', '3'),
+            prereleases_dir.join('b', '1'),
+            prereleases_dir.join('b', '2'),
+          ].map { |file|
+            [file, file.exist?]
+          }
+        ]
+
+        expect(existences).to eq(
+          prereleases_dir.join('a', '1') => false,
+          prereleases_dir.join('a', '2') => true,
+          prereleases_dir.join('a', '3') => true,
+          prereleases_dir.join('b', '1') => true,
+          prereleases_dir.join('b', '2') => true,
+        )
+      end
     end
   end
 end

data/spec/agent/tasks/fetch_spec.rb

@@ -36,6 +36,7 @@ describe Mamiya::Agent::Tasks::Fetch do
         package: 'mypkg',
         destination: app_destination,
         config: config,
+        logger: task.logger,
       ).and_return(step)
 
       allow(agent).to receive(:trigger)

data/spec/agent/tasks/prepare_spec.rb

@@ -0,0 +1,127 @@
+require 'spec_helper'
+require 'tmpdir'
+require 'pathname'
+
+require 'mamiya/agent/tasks/notifyable'
+require 'mamiya/agent/tasks/prepare'
+
+require 'mamiya/steps/extract'
+require 'mamiya/steps/prepare'
+
+describe Mamiya::Agent::Tasks::Prepare do
+  let!(:tmpdir) { Pathname.new Dir.mktmpdir("mamiya-agent-tasks-prepare-spec") }
+  after { FileUtils.remove_entry_secure tmpdir }
+
+  let(:packages_dir) { tmpdir.join('packages').tap(&:mkdir) }
+  let(:prereleases_dir) { tmpdir.join('prereleases').tap(&:mkdir) }
+
+  let(:config) do
+    _pkg, _pre = packages_dir, prereleases_dir
+    Mamiya::Configuration.new.evaluate! do
+      set :packages_dir, _pkg
+      set :prereleases_dir, _pre
+    end
+  end
+
+  let(:agent) { double('agent', config: config, trigger: nil, labels: [:foo, :bar]) }
+  let(:task_queue) { double('task_queue', enqueue: nil) }
+
+  let(:extract_step) { double('extract step', run!: nil) }
+  let(:prepare_step) { double('prepare step', run!: nil) }
+
+  let(:job) { {'app' => 'myapp', 'pkg' => 'mypkg'} }
+
+  subject(:task) { described_class.new(task_queue, job, agent: agent, raise_error: true) }
+
+  it 'inherits notifyable task' do
+    expect(described_class.ancestors).to include(Mamiya::Agent::Tasks::Notifyable)
+  end
+
+  describe "#execute" do
+    context "when package not fetched" do
+      before do
+        expect(Mamiya::Steps::Extract).not_to receive(:new)
+        expect(Mamiya::Steps::Prepare).not_to receive(:new)
+      end
+
+      it "enqueues fetch task and finish" do
+        expect(task_queue).to receive(:enqueue).with(
+          :fetch, job.merge('task' => 'prepare', '_chain' => ['prepare'])
+        )
+
+        task.execute
+      end
+
+      context "with _chain-ed job" do
+        let(:job) { {'app' => 'myapp', 'pkg' => 'mypkg', '_chain' => ['next']} }
+
+        it "enqueues fetch task and finish" do
+          expect(task_queue).to receive(:enqueue).with(
+            :fetch, job.merge('task' => 'prepare', '_chain' => ['prepare', 'next'])
+          )
+
+          task.execute
+        end
+      end
+    end
+
+    context "when package fetched" do
+      before do
+        packages_dir.join('myapp').mkdir
+        File.write packages_dir.join('myapp', 'mypkg.tar.gz'), "\n"
+
+        allow(Mamiya::Steps::Extract).to receive(:new).with(
+          package: packages_dir.join('myapp', 'mypkg.tar.gz'),
+          destination: prereleases_dir.join('myapp', 'mypkg'),
+          config: config,
+          logger: task.logger,
+        ).and_return(extract_step)
+
+        allow(Mamiya::Steps::Prepare).to receive(:new).with(
+          target: prereleases_dir.join('myapp', 'mypkg'),
+          labels: [:foo, :bar],
+          config: config,
+          script: nil,
+          logger: task.logger,
+        ).and_return(prepare_step)
+      end
+
+      it "prepares extracted release" do
+        expect(extract_step).to receive(:run!).ordered
+        expect(prepare_step).to receive(:run!).ordered
+
+        task.execute
+      end
+
+      context "when prepared package exists" do
+        before do
+          prereleases_dir.join('myapp', 'mypkg').mkpath
+          File.write prereleases_dir.join('myapp', 'mypkg', '.mamiya.prepared'),
+            "#{Time.now.to_i}\n"
+        end
+
+        it "does nothing" do
+          expect(extract_step).not_to receive(:run!)
+          expect(prepare_step).not_to receive(:run!)
+
+          task.execute
+        end
+      end
+
+      context "when extracted but non-prepared release exists" do
+        before do
+          prereleases_dir.join('myapp', 'mypkg').mkpath
+        end
+
+        it "removes existing release then prepare" do
+          expect(extract_step).to receive(:run!).ordered do
+            expect(prereleases_dir.join('myapp', 'mypkg')).not_to be_exist
+          end
+          expect(prepare_step).to receive(:run!).ordered
+
+          task.execute
+        end
+      end
+    end
+  end
+end

data/spec/agent_spec.rb

@@ -10,6 +10,8 @@ require 'mamiya/agent'
 require 'mamiya/agent/task_queue'
 require 'mamiya/agent/actions'
 
+require 'mamiya/configuration'
+
 require_relative './support/dummy_serf.rb'
 
 describe Mamiya::Agent do
@@ -21,7 +23,9 @@ describe Mamiya::Agent do
   end
 
   let(:config) do
-    {serf: {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}}}
+    Mamiya::Configuration.new.evaluate! do
+      set :serf, {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}}
+    end
   end
 
   before do
@@ -84,33 +88,11 @@ describe Mamiya::Agent do
     end
   end
 
-  describe "#update_tags!" do
-    describe "(status)" do
-      context "when it is not busy" do
-        it "shows ready" do
-          agent.update_tags!
-
-          expect(serf.tags['mamiya']).to eq ',ready,'
-        end
-      end
-
-      context "when it is running multiple jobs" do
-        pending
-      end
-    end
-
-    describe "(prepared)" do
-      pending
-    end
-
-    describe "(current)" do
-      pending
-    end
-  end
-
   describe "#status" do
     before do
       allow(agent).to receive(:existing_packages).and_return("app" => ["pkg"])
+      allow(agent).to receive(:existing_prereleases).and_return("app" => ["pkg"])
+      allow(agent).to receive(:labels).and_return([:foo,:bar])
 
       allow(task_queue).to receive(:status).and_return({a: {working: nil, queue: []}})
     end
@@ -133,12 +115,24 @@ describe Mamiya::Agent do
       expect(status[:packages]).to eq agent.existing_packages
     end
 
+    it "includes prereleases" do
+      expect(status[:prereleases]).to eq agent.existing_prereleases
+    end
+
+    it "includes status" do
+      expect(status[:labels]).to eq agent.labels
+    end
+
     context "with packages=false" do
       subject(:status) { agent.status(packages: false) }
 
       it "doesn't include existing packages" do
         expect(status.has_key?(:packages)).to be_false
       end
+
+      it "doesn't include existing packages" do
+        expect(status.has_key?(:prereleases)).to be_false
+      end
     end
 
     describe "(task queue)" do
@@ -176,6 +170,51 @@ describe Mamiya::Agent do
     end
   end
 
+  describe "#existing_prereleases" do
+    let!(:prereleases_dir) { Pathname.new Dir.mktmpdir('mamiya-agent-spec') }
+    after { FileUtils.remove_entry_secure(prereleases_dir) }
+
+    let(:config) { {prereleases_dir: prereleases_dir} }
+
+    subject(:existing_prereleases) { agent.existing_prereleases }
+
+    before do
+      prereleases_dir.join('a').mkdir
+      prereleases_dir.join('a', '1').mkdir
+      File.write prereleases_dir.join('a', '1', '.mamiya.prepared'), "#{Time.now.to_s}\n"
+      prereleases_dir.join('a', '2').mkdir
+      File.write prereleases_dir.join('a', '2', '.mamiya.prepared'), "#{Time.now.to_s}\n"
+      prereleases_dir.join('a', '3').mkdir
+    end
+
+    it "returns prepared prereleases" do
+      expect(existing_prereleases).to eq('a' => ['1', '2'])
+    end
+  end
+
+
+  describe "#labels" do
+    subject(:labels) { agent.labels }
+
+    context "with config.labels" do
+      before do
+        config.evaluate! do
+          labels { [:foo, :bar, :baz] }
+        end
+      end
+
+      it "retrieves label from configuration" do
+        expect(labels).to eq [:foo, :bar, :baz]
+      end
+    end
+
+    context "without config.labels" do
+      it "returns []" do
+        expect(labels).to eq []
+      end
+    end
+  end
+
   describe "query responder" do
     it "responds to 'mamiya:status'" do
       allow(agent).to receive(:status).with(packages: false).and_return("my" => "status")
@@ -185,10 +224,19 @@ describe Mamiya::Agent do
     end
 
     it "responds to 'mamiya:packages'" do
-      allow(agent).to receive(:existing_packages).and_return(%w(pkg1 pkg2))
+      allow(agent).to receive(:existing_packages).and_return("app" => %w(pkg1 pkg2))
+      allow(agent).to receive(:existing_prereleases).and_return("app" => %w(pkg2))
 
       response = serf.trigger_query('mamiya:packages', '')
-      expect(JSON.parse(response)).to eq(%w(pkg1 pkg2))
+
+      expect(JSON.parse(response)).to eq(
+        "packages" => {
+          "app" => %w(pkg1 pkg2)
+        },
+        "prereleases" => {
+          "app" => %w(pkg2)
+        },
+      )
     end
   end
 

data/spec/dsl_spec.rb

@@ -50,6 +50,12 @@ describe Mamiya::DSL do
         dsl.load! "#{__dir__}/fixtures/dsl_test_load.rb"
       }.to change { dsl.foo }.from(42).to(72)
     end
+
+    it "sets _file" do
+      expect {
+        dsl.load! "#{__dir__}/fixtures/dsl_test_load.rb"
+      }.to change { dsl._file }.from(nil).to(Pathname.new("#{__dir__}/fixtures/dsl_test_load.rb"))
+    end
   end
 
   describe "#use" do
@@ -284,12 +290,4 @@ describe Mamiya::DSL do
       end
     end
   end
-
-  describe "#servers" do
-    pending "Hey!"
-  end
-
-  describe "#use_servers" do
-    pending "Hey!"
-  end
 end

data/spec/master/agent_monitor_spec.rb

@@ -3,6 +3,8 @@ require 'json'
 
 require 'villein/event'
 
+require 'mamiya/util/label_matcher'
+
 require 'mamiya/master/agent_monitor'
 
 describe Mamiya::Master::AgentMonitor do
@@ -30,7 +32,7 @@ describe Mamiya::Master::AgentMonitor do
     {
       "Acks" => ['a'],
       "Responses" => {
-        'a' => {"foo" => "bar", 'packages' => ['pkg1']}.to_json,
+        'a' => {"foo" => "bar", 'packages' => {"app" => ['pkg1']}, 'prereleases' => {"app" => ['pkg2']}}.to_json,
       },
     }
   end
@@ -39,11 +41,50 @@ describe Mamiya::Master::AgentMonitor do
     {
       "Acks" => ['a'],
       "Responses" => {
-        'a' => ['pkg1','pkg2'].to_json,
+        'a' => {"packages" => {"app" => ['pkg1','pkg2']}, "prereleases" => {"app" => ['pkg2']}}.to_json,
       },
     }
   end
 
+  describe "#statuses" do
+    let(:members) do
+      [
+        {
+          "name"=>"a", "status"=>"alive",
+          "addr"=>"x.x.x.x:7676", "port"=>7676,
+          "protocol"=>{"max"=>4, "min"=>2, "version"=>4},
+          "tags"=>{},
+        },
+      ]
+    end
+
+    let(:status_query_response) do
+      {
+        "Acks" => ['a','b'],
+        "Responses" => {
+          'a' => {'packages' => {}, 'prereleases' => {}, "labels" => ['foo','bar']}.to_json,
+          'b' => {'packages' => {}, 'prereleases' => {}, "labels" => ['baz']}.to_json,
+        },
+      }
+    end
+
+    before do
+      stub_serf_queries()
+      allow(serf).to receive(:members).and_return(members)
+
+      agent_monitor.refresh
+    end
+
+    context "with labels" do
+      it "can filter agents by label" do
+        # FIXME: stub label matcher
+        expect(agent_monitor.statuses.keys.sort).to eq ['a', 'b']
+        expect(agent_monitor.statuses(labels: ['foo']).keys.sort).to eq ['a']
+        expect(agent_monitor.statuses(labels: ['baz']).keys.sort).to eq ['b']
+      end
+    end
+  end
+
   describe "#refresh" do
     let(:members) do
       [
@@ -69,12 +110,32 @@ describe Mamiya::Master::AgentMonitor do
       }.to('bar')
     end
 
+    it "updates #last_refresh_at" do
+      agent_monitor.refresh
+      expect(agent_monitor.last_refresh_at).to be_a_kind_of(Time)
+
+      expect {
+        agent_monitor.refresh
+      }.to change {
+        agent_monitor.last_refresh_at
+      }
+    end
+
+
     it "updates status .packages by packages query" do
       expect {
         agent_monitor.refresh
       }.to change {
         agent_monitor.statuses["a"] && agent_monitor.statuses["a"]['packages']
-      }.to(%w(pkg1 pkg2))
+      }.to("app" => %w(pkg1 pkg2))
+    end
+
+    it "updates status .prereleases by packages query" do
+      expect {
+        agent_monitor.refresh
+      }.to change {
+        agent_monitor.statuses["a"] && agent_monitor.statuses["a"]['prereleases']
+      }.to("app" => %w(pkg2))
     end
 
     context "when packages query unavailable, but available in status query" do
@@ -91,7 +152,15 @@ describe Mamiya::Master::AgentMonitor do
           agent_monitor.refresh
         }.to change {
           agent_monitor.statuses["a"] && agent_monitor.statuses["a"]['packages']
-        }.to(%w(pkg1))
+        }.to("app" => %w(pkg1))
+      end
+
+      it "updates status .prereleases from status" do
+        expect {
+          agent_monitor.refresh
+        }.to change {
+          agent_monitor.statuses["a"] && agent_monitor.statuses["a"]['prereleases']
+        }.to("app" => %w(pkg2))
       end
     end
 
@@ -393,6 +462,47 @@ describe Mamiya::Master::AgentMonitor do
         end
       end
 
+      describe "prerelease" do
+        describe ":remove" do
+          let(:status) do
+            {prereleases: {'myapp' => ['pkg1']}}
+          end
+
+          it "removes removed release from prereleases" do
+            commit('mamiya:prerelease:remove',
+                   app: 'myapp', pkg: 'pkg1')
+
+            expect(new_status["prereleases"]['myapp']).to eq []
+          end
+
+          context "with existing packages" do
+            let(:status) do
+              {prereleases: {'myapp' => ['pkg1', 'pkg2']}}
+            end
+
+            it "removes removed release from prereleases" do
+              commit('mamiya:prerelease:remove',
+                     app: 'myapp', pkg: 'pkg1')
+
+              expect(new_status["prereleases"]['myapp']).to eq ['pkg2']
+            end
+          end
+
+          context "with inexist package" do
+            let(:status) do
+              {prereleases: {'myapp' => ['pkg1', 'pkg3']}}
+            end
+
+            it "removes removed release from packages" do
+              commit('mamiya:prerelease:remove',
+                     app: 'myapp', pkg: 'pkg2')
+
+              expect(new_status["prereleases"]['myapp']).to eq ['pkg1', 'pkg3']
+            end
+          end
+        end
+      end
+
       describe "fetch" do
         describe "success" do
           let(:status) do
@@ -420,6 +530,34 @@ describe Mamiya::Master::AgentMonitor do
           end
         end
       end
+
+      describe "prepare" do
+        describe "success" do
+          let(:status) do
+            {prereleases: {}}
+          end
+
+          it "updates prereleases" do
+            commit('mamiya:task:finish',
+                   task: {task: 'prepare', app: 'myapp', pkg: 'pkg'})
+
+            expect(new_status["prereleases"]['myapp']).to eq ["pkg"]
+          end
+
+          context "with existing prerelease" do
+            let(:status) do
+              {prereleases: {'myapp' => ['pkg1']}}
+            end
+
+            it "updates prereleases" do
+              commit('mamiya:task:finish',
+                     task: {task: 'prepare', app: 'myapp', pkg: 'pkg2'})
+
+              expect(new_status["prereleases"]['myapp']).to eq %w(pkg1 pkg2)
+            end
+          end
+        end
+      end
     end
   end
 end