mamiya 0.0.1.alpha19 → 0.0.1.alpha20

data/spec/agent/tasks/abstract_spec.rb

@@ -0,0 +1,58 @@
+require 'spec_helper'
+require 'mamiya/agent/tasks/abstract'
+
+describe Mamiya::Agent::Tasks::Abstract do
+  let(:queue) { double('task_queue') }
+
+  let(:job) { {'foo' => 'bar'} }
+  subject(:task) { described_class.new(queue, job) }
+
+  describe "#task" do
+    before do
+      allow(described_class).to receive(:identifier).and_return('ident')
+    end
+    it "includes job specification and class' identifier" do
+      expect(task.task['foo']).to eq 'bar'
+      expect(task.task['task']).to eq described_class.identifier
+    end
+  end
+
+  describe "#execute" do
+    it "calls before, then run" do
+      expect(task).to receive(:before).ordered
+      expect(task).to receive(:run).ordered
+      task.execute
+    end
+    it "calls after" do
+      expect(task).to receive(:run).ordered
+      expect(task).to receive(:after).ordered
+      expect(task).not_to receive(:error)
+      task.execute
+    end
+
+    it "handles error" do
+      expect(task).to receive(:after)
+      expect(task).to receive(:errored)
+
+      err = RuntimeError.new
+      allow(task).to receive(:run).and_raise(err)
+
+      task.execute
+
+      expect(task.error).to eq err
+    end
+  end
+
+  describe ".identifier" do
+    it "returns class name without module name" do
+      allow(described_class).to receive(:name).and_return('Mamiya::Agent::Tasks::Foo')
+      expect(described_class.identifier).to eq 'foo'
+    end
+
+    it "returns camelcased class name" do
+      allow(described_class).to receive(:name).and_return('Mamiya::Agent::Tasks::FooBar')
+      expect(described_class.identifier).to eq 'foo_bar'
+    end
+  end
+end
+

data/spec/agent/tasks/clean_spec.rb

@@ -0,0 +1,72 @@
+require 'spec_helper'
+
+require 'mamiya/agent/tasks/clean'
+require 'mamiya/agent/tasks/abstract'
+require 'mamiya/steps/fetch'
+
+describe Mamiya::Agent::Tasks::Clean do
+  let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-tasks-clean-spec') }
+  after { FileUtils.remove_entry_secure(tmpdir) if File.exist?(tmpdir) }
+
+  let(:config) { {packages_dir: tmpdir, keep_packages: 2} }
+
+  let(:agent) { double('agent', config: config) }
+  let(:task_queue) { double('task_queue') }
+
+  subject(:task) { described_class.new(task_queue, {}, agent: agent, raise_error: true) }
+
+  it 'inherits abstract task' do
+    expect(described_class.ancestors).to include(Mamiya::Agent::Tasks::Abstract)
+  end
+
+
+  describe "#execute" do
+    before do
+      path = Pathname.new(tmpdir)
+
+      path.join('a').mkdir
+      File.write path.join('a', "a.tar.gz"), "\n"
+      File.write path.join('a', "a.json"), "\n"
+      File.write path.join('a', "b.json"), "\n"
+      File.write path.join('a', "b.tar.gz"), "\n"
+      File.write path.join('a', "c.json"), "\n"
+      File.write path.join('a', "c.tar.gz"), "\n"
+      path.join('b').mkdir
+      File.write path.join('b', "a.tar.gz"), "\n"
+      File.write path.join('b', "a.json"), "\n"
+
+      path.join('c').mkdir
+      File.write path.join('c', "a.tar.gz"), "\n"
+      File.write path.join('c', "b.json"), "\n"
+    end
+
+    it "cleans up" do
+      expect(agent).to receive(:trigger).with('pkg', action: 'remove', application: 'a', package: 'a', coalesce: false)
+
+      task.execute
+
+      path = Pathname.new(tmpdir)
+      existences = Hash[
+        [
+          path.join('a', 'a.tar.gz'),
+          path.join('a', 'a.json'),
+          path.join('a', 'b.tar.gz'),
+          path.join('a', 'b.json'),
+          path.join('a', 'c.tar.gz'),
+          path.join('a', 'c.json'),
+        ].map { |file|
+          [file, file.exist?]
+        }
+      ]
+
+      expect(existences).to eq(
+        path.join('a', 'a.tar.gz') => false,
+        path.join('a', 'a.json') => false,
+        path.join('a', 'b.tar.gz') => true,
+        path.join('a', 'b.json') => true,
+        path.join('a', 'c.tar.gz') => true,
+        path.join('a', 'c.json') => true,
+      )
+    end
+  end
+end

data/spec/agent/tasks/fetch_spec.rb

@@ -0,0 +1,56 @@
+require 'spec_helper'
+
+require 'mamiya/agent/tasks/fetch'
+require 'mamiya/agent/tasks/notifyable'
+require 'mamiya/steps/fetch'
+
+describe Mamiya::Agent::Tasks::Fetch do
+  let(:config) { {packages_dir: File::NULL} }
+  let(:agent) { double('agent', config: config) }
+  let(:task_queue) { double('task_queue', enqueue: nil) }
+
+  let(:step) { double('step', run!: nil) }
+
+  let(:job) { {'app' => 'myapp', 'pkg' => 'mypkg'} }
+
+  subject(:task) { described_class.new(task_queue, job, agent: agent, raise_error: true) }
+
+  it 'inherits notifyable task' do
+    expect(described_class.ancestors).to include(Mamiya::Agent::Tasks::Notifyable)
+  end
+
+  describe "#execute" do
+    before do
+      allow(Mamiya::Steps::Fetch).to receive(:new).with(
+        application: 'myapp',
+        package: 'mypkg',
+        destination: File.join(File::NULL, 'myapp'),
+        config: config,
+      ).and_return(step)
+
+      allow(agent).to receive(:trigger)
+    end
+
+    it "calls fetch step" do
+      expect(step).to receive(:run!)
+
+      task.execute
+    end
+
+    it "enqueues clean task" do
+      expect(task_queue).to receive(:enqueue).with(:clean, {})
+
+      task.execute
+    end
+
+    context "when already fetched" do
+      it "does nothing" do
+        allow(step).to receive(:run!).and_raise(Mamiya::Storages::Abstract::AlreadyFetched)
+
+        expect {
+          task.execute
+        }.not_to raise_error
+      end
+    end
+  end
+end

data/spec/agent/tasks/notifyable_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+require 'mamiya/agent/tasks/notifyable'
+
+describe Mamiya::Agent::Tasks::Notifyable do
+  let(:queue) { double('task_queue') }
+  let(:agent) { double('agent', trigger: nil) }
+
+  # specifying 'task' key that Tasks::Abstract assigns,
+  # to make expecting message easier
+  let(:job) { {'foo' => 'bar', 'task' => 'notifyable'} }
+  subject(:task) { described_class.new(queue, job, agent: agent) }
+
+  describe "#execute" do
+    it "notifies first, then :run" do
+      expect(agent).to receive(:trigger).with('task', action: 'start', task: job).ordered
+      expect(task).to receive(:before).ordered
+      expect(task).to receive(:run).ordered
+      task.execute
+    end
+
+    it "calls after, then notify" do
+      expect(task).to receive(:run).ordered
+      expect(task).to receive(:after).ordered
+      expect(agent).to receive(:trigger).with('task', action: 'finish', task: job).ordered
+      task.execute
+    end
+
+    it "handles error" do
+      expect(agent).to receive(:trigger).with('task', action: 'error', task: job, error: RuntimeError.name)
+      err = RuntimeError.new
+      allow(task).to receive(:run).and_raise(err)
+      task.execute
+      expect(task.error).to eq err
+    end
+  end
+end
+

data/spec/agent_spec.rb

@@ -7,7 +7,7 @@ require 'villein/event'
 
 require 'mamiya/version'
 require 'mamiya/agent'
-require 'mamiya/agent/fetcher'
+require 'mamiya/agent/task_queue'
 require 'mamiya/agent/actions'
 
 require_relative './support/dummy_serf.rb'
@@ -15,12 +15,9 @@ require_relative './support/dummy_serf.rb'
 describe Mamiya::Agent do
 
   let(:serf) { DummySerf.new }
-  let(:fetcher) do
-    double('fetcher', start!: nil, working?: false).tap do |f|
-      cleanup_hook = nil
-      allow(f).to receive(:cleanup_hook=) { |_| cleanup_hook = _ }
-      allow(f).to receive(:cleanup_hook)  { cleanup_hook }
-    end
+
+  let(:task_queue) do
+    double('task_queue', start!: nil)
   end
 
   let(:config) do
@@ -29,7 +26,7 @@ describe Mamiya::Agent do
 
   before do
     allow(Villein::Agent).to receive(:new).and_return(serf)
-    allow(Mamiya::Agent::Fetcher).to receive(:new).and_return(fetcher)
+    allow(Mamiya::Agent::TaskQueue).to receive(:new).and_return(task_queue)
   end
 
   subject(:agent) { described_class.new(config) }
@@ -38,32 +35,47 @@ describe Mamiya::Agent do
     expect(described_class.ancestors).to include(Mamiya::Agent::Actions)
   end
 
-  describe "fetcher" do
-    it "sends events on cleanup hook" do
+  describe "#trigger" do
+    it "sends serf event" do
       expect(serf).to receive(:event).with(
-        'mamiya:fetch-result:remove',
+        'mamiya:foo',
         {
-          name: serf.name, application: 'foo', package: 'bar',
+          a: 'b',
+          name: 'my-name',
         }.to_json,
-        coalesce: false,
+        coalesce: true,
       )
 
-      agent.fetcher.cleanup_hook.call('foo', 'bar')
+      agent.trigger(:foo, a: 'b')
+    end
+
+    it "sends serf event with action" do
+      expect(serf).to receive(:event).with(
+        'mamiya:foo:bar',
+        {
+          a: 'b',
+          name: 'my-name',
+        }.to_json,
+        coalesce: true,
+      )
+
+      agent.trigger(:foo, a: 'b', action: 'bar')
     end
   end
 
   describe "#run!" do
-    it "starts serf and fetcher" do
+    it "starts serf, and task_queue" do
       begin
         flag = false
 
-        expect(fetcher).to receive(:start!)
+        expect(task_queue).to receive(:start!)
         expect(serf).to receive(:start!)
         expect(serf).to receive(:auto_stop) do
           flag = true
         end
 
         th = Thread.new { agent.run! }
+        th.abort_on_exception = true
 
         10.times { break if flag; sleep 0.1 }
       ensure
@@ -82,18 +94,6 @@ describe Mamiya::Agent do
         end
       end
 
-      context "when it is fetching" do
-        before do
-          allow(fetcher).to receive(:working?).and_return(true)
-        end
-
-        it "shows fetching" do
-          agent.update_tags!
-
-          expect(serf.tags['mamiya']).to eq ',fetching,'
-        end
-      end
-
       context "when it is running multiple jobs" do
         pending
       end
@@ -111,10 +111,8 @@ describe Mamiya::Agent do
   describe "#status" do
     before do
       allow(agent).to receive(:existing_packages).and_return("app" => ["pkg"])
-      allow(fetcher).to receive(:queue_size).and_return(42)
-      allow(fetcher).to receive(:working?).and_return(false)
-      allow(fetcher).to receive(:current_job).and_return(nil)
-      allow(fetcher).to receive(:pending_jobs).and_return([['app', 'pkg2', nil, nil]])
+
+      allow(task_queue).to receive(:status).and_return({a: {working: nil, queue: []}})
     end
 
     subject(:status) { agent.status }
@@ -131,28 +129,9 @@ describe Mamiya::Agent do
       expect(status[:packages]).to eq agent.existing_packages
     end
 
-    describe "(fetcher)" do
-      it "includes queue_size as pending" do
-        expect(status[:fetcher][:pending]).to eq 42
-      end
-
-      it "includes pendings job" do
-        expect(status[:fetcher][:pending_jobs]).to eq([['app', 'pkg2']])
-      end
-
-      it "shows fetching status" do
-        expect(status[:fetcher][:fetching]).to be_nil
-      end
-
-      context "when it's fetching" do
-        before do
-          allow(fetcher).to receive(:working?).and_return(true)
-          allow(fetcher).to receive(:current_job).and_return(%w(foo bar))
-        end
-
-        it "shows fetching true" do
-          expect(status[:fetcher][:fetching]).to eq ['foo', 'bar']
-        end
+    describe "(task queue)" do
+      it "includes task_queue" do
+        expect(status[:queues]).to eq({a: {working: nil, queue: []}})
       end
     end
   end

data/spec/master/agent_monitor_spec.rb

@@ -150,119 +150,205 @@ describe Mamiya::Master::AgentMonitor do
 
     subject(:new_status) { agent_monitor.statuses["a"] }
 
-    describe "fetch-result" do
-      describe ":ack" do
-        let(:status) do
-          {fetcher: {fetching: nil, pending: 0}}
-        end
+    describe "task" do
+      describe ":start" do
+        context "if task is in the queue" do
+          let(:status) do
+            {queues: {
+              a: {queue: [{task: 'a', foo: 'bar'}], working: nil}
+            }}
+          end
 
-        it "updates pending" do
-          commit('mamiya:fetch-result:ack', pending: 72, application: 'foo', package: 'bar')
-          expect(new_status["fetcher"]["pending"]).to eq 72
-          expect(new_status["fetcher"]["pending_jobs"]).to eq [%w(foo bar)]
-        end
-      end
+          it "removes from queue, set in working" do
+            commit('mamiya:task:start',
+                   task: {task: 'a', foo: 'bar'})
 
-      describe ":start" do
-        let(:status) do
-          {fetcher: {fetching: nil, pending: 1, pending_jobs: [%w(app pkg)]}}
+            expect(new_status['queues']['a']['queue']).to be_empty
+            expect(new_status['queues']['a']['working']).to eq('task' => 'a', 'foo' => 'bar')
+          end
         end
 
-        it "updates fetching" do
-          commit('mamiya:fetch-result:start',
-                 application: 'app', package: 'pkg', pending: 0)
-          expect(new_status["fetcher"]["fetching"]).to eq ['app', 'pkg']
-          expect(new_status["fetcher"]["pending_jobs"]).to be_empty
+        context "if task is not in the queue" do
+          let(:status) do
+            {queues: {
+              a: {queue: [], working: nil}
+            }}
+          end
+
+          it "set in working" do
+            commit('mamiya:task:start',
+                   task: {task: 'a', foo: 'bar'})
+
+            expect(new_status['queues']['a']['working']).to eq('task' => 'a', 'foo' => 'bar')
+          end
         end
       end
 
-      describe ":error" do
-        let(:status) do
-          {fetcher: {fetching: ['app', 'pkg'], pending: 0}}
+      describe ":finish" do
+        context "if task is working" do
+          let(:status) do
+            {queues: {
+              a: {queue: [], working: {task: 'a', foo: 'bar'}}
+            }}
+          end
+
+          it "removes from working" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
+
+            expect(new_status['queues']['a']['working']).to be_nil
+          end
         end
 
-        it "updates fetching" do
-          commit('mamiya:fetch-result:error',
-                 application: 'app', package: 'pkg', pending: 0)
+        context "if task is in queue" do
+          let(:status) do
+            {queues: {
+              a: {queue: [{task: 'a', foo: 'bar'}, {task: 'a', bar: 'baz'}], working: nil}
+            }}
+          end
+
+          it "removes from queue" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
 
-          expect(new_status["fetcher"]["fetching"]).to eq nil
+            expect(new_status['queues']['a']['working']).to be_nil
+            expect(new_status['queues']['a']['queue']).to eq [{'task' => 'a', 'bar' => 'baz'}]
+          end
         end
 
-        context "when package doesn't match with present state" do
-          it "doesn't updates fetching" do
-            commit('mamiya:fetch-result:error',
-                   application: 'app', package: 'pkg2', pending: 0)
+        context "if task is not working" do
+          let(:status) do
+            {queues: {
+              a: {queue: [], working: {task: 'a', foo: 'baz'}}
+            }}
+          end
+
+          it "does nothing" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
 
-            expect(new_status["fetcher"]["fetching"]).to \
-              eq(['app', 'pkg'])
+            expect(new_status['queues']['a']['working']).to eq('task' => 'a', 'foo' => 'baz')
+            expect(new_status['queues']['a']['queue']).to eq []
           end
         end
       end
 
-      describe ":success" do
-        let(:status) do
-          {fetcher: {fetching: ['app', 'pkg'], pending: 0},
-           packages: {}}
-        end
+      describe ":error" do
+        context "if task is working" do
+          let(:status) do
+            {queues: {
+              a: {queue: [], working: {task: 'a', foo: 'bar'}}
+            }}
+          end
 
-        it "updates fetching" do
-          commit('mamiya:fetch-result:success',
-                 application: 'app', package: 'pkg', pending: 0)
+          it "removes from working" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
 
-          expect(new_status["fetcher"]["fetching"]).to eq nil
+            expect(new_status['queues']['a']['working']).to be_nil
+          end
         end
 
-        it "updates packages" do
-          commit('mamiya:fetch-result:success',
-                 application: 'app', package: 'pkg', pending: 0)
+        context "if task is in queue" do
+          let(:status) do
+            {queues: {
+              a: {queue: [{task: 'a', foo: 'bar'}, {task: 'a', bar: 'baz'}], working: nil}
+            }}
+          end
 
-          expect(new_status["packages"]["app"]).to eq ["pkg"]
+          it "removes from queue" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
+
+            expect(new_status['queues']['a']['working']).to be_nil
+            expect(new_status['queues']['a']['queue']).to eq [{'task' => 'a', 'bar' => 'baz'}]
+          end
         end
 
-        context "with existing packages" do
+        context "if task is not working" do
           let(:status) do
-            {fetcher: {fetching: ['app', 'pkg2'], pending: 0},
-             packages: {"app" => ['pkg1']}}
+            {queues: {
+              a: {queue: [], working: {task: 'a', foo: 'baz'}}
+            }}
           end
 
-          it "updates packages" do
-            commit('mamiya:fetch-result:success',
-                   application: 'app', package: 'pkg2', pending: 0)
+          it "does nothing" do
+            commit('mamiya:task:finish',
+                   task: {task: 'a', foo: 'bar'})
 
-            expect(new_status["packages"]["app"]).to eq %w(pkg1 pkg2)
+            expect(new_status['queues']['a']['working']).to eq('task' => 'a', 'foo' => 'baz')
+            expect(new_status['queues']['a']['queue']).to eq []
           end
         end
+      end
+    end
 
-        context "when package doesn't match with present state" do
-          it "doesn't updates fetching" do
-            commit('mamiya:fetch-result:success',
-                   application: 'app', package: 'pkg2', pending: 0)
+    describe "(task handling)" do
+      describe "pkg" do
+        describe ":remove" do
+          let(:status) do
+            {packages: {'myapp' => ['pkg1']}}
+          end
+
+          it "removes removed package from packages" do
+            commit('mamiya:pkg:remove',
+                   application: 'myapp', package: 'pkg1')
 
-            expect(agent_monitor.statuses["a"]["fetcher"]["fetching"]).to \
-              eq(['app', 'pkg'])
+            expect(new_status["packages"]['myapp']).to eq []
           end
 
-          it "updates packages" do
-            commit('mamiya:fetch-result:success',
-                   application: 'app', package: 'pkg', pending: 0)
+          context "with existing packages" do
+            let(:status) do
+              {packages: {'myapp' => ['pkg1', 'pkg2']}}
+            end
+
+            it "removes removed package from packages" do
+              commit('mamiya:pkg:remove',
+                     application: 'myapp', package: 'pkg1')
+
+              expect(new_status["packages"]['myapp']).to eq ['pkg2']
+            end
+          end
 
-            expect(new_status["packages"]["app"]).to eq ["pkg"]
+          context "with inexist package" do
+            let(:status) do
+              {packages: {'myapp' => ['pkg1', 'pkg3']}}
+            end
+
+            it "removes removed package from packages" do
+              commit('mamiya:pkg:remove',
+                     application: 'myapp', package: 'pkg2')
+
+              expect(new_status["packages"]['myapp']).to eq ['pkg1', 'pkg3']
+            end
           end
         end
       end
 
-      describe ":remove" do
-        context "with existing packages" do
+      describe "fetch" do
+        describe "success" do
           let(:status) do
-            {fetcher: {fetching: ['app', 'pkg2'], pending: 0},
-             packages: {"app" => ['pkg1']}}
+            {packages: {}}
           end
 
           it "updates packages" do
-            commit('mamiya:fetch-result:remove',
-                   application: 'app', package: 'pkg1', pending: 0)
+            commit('mamiya:task:finish',
+                   task: {task: 'fetch', app: 'myapp', pkg: 'pkg'})
+
+            expect(new_status["packages"]['myapp']).to eq ["pkg"]
+          end
+
+          context "with existing packages" do
+            let(:status) do
+              {packages: {'myapp' => ['pkg1']}}
+            end
+
+            it "updates packages" do
+              commit('mamiya:task:finish',
+                     task: {task: 'fetch', app: 'myapp', pkg: 'pkg2'})
 
-            expect(new_status["packages"]["app"]).to eq []
+              expect(new_status["packages"]['myapp']).to eq %w(pkg1 pkg2)
+            end
           end
         end
       end