mamiya 0.0.1.alpha19 → 0.0.1.alpha20

data/lib/mamiya/agent/tasks/fetch.rb

@@ -0,0 +1,60 @@
+require 'mamiya/agent/tasks/notifyable'
+require 'mamiya/steps/fetch'
+require 'mamiya/storages/abstract'
+
+module Mamiya
+  class Agent
+    module Tasks
+      class Fetch < Notifyable
+        def run
+          logger.info "Fetching #{application}/#{package}"
+
+          take_interval
+          step.run!
+          order_cleaning
+        rescue Mamiya::Storages::Abstract::AlreadyFetched
+          logger.info "It has already fetched; skipping."
+        end
+
+        private
+
+        def take_interval
+          fetch_sleep = config[:fetch_sleep]
+          wait = rand(fetch_sleep)
+
+          @logger.info "Sleeping #{wait} sec before starting fetch"
+          rand(wait)
+        end
+
+        def order_cleaning
+          task_queue.enqueue(:clean, {})
+        end
+
+        def application
+          task['app']
+        end
+
+        def package
+          task['pkg']
+        end
+
+        def destination
+          @destination ||= File.join(packages_dir, application)
+        end
+
+        def packages_dir
+          @packages_dir ||= config && config[:packages_dir]
+        end
+
+        def step
+          @step ||= Mamiya::Steps::Fetch.new(
+            application: application,
+            package: package,
+            destination: destination,
+            config: config,
+          )
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/agent/tasks/notifyable.rb

@@ -0,0 +1,30 @@
+require 'mamiya/agent'
+require 'mamiya/agent/tasks/abstract'
+
+module Mamiya
+  class Agent
+    module Tasks
+      class Notifyable < Abstract
+        def execute
+          agent.trigger('task', action: 'start',
+            task: task
+          )
+
+          super
+
+        ensure
+          if error
+            agent.trigger('task', action: 'error',
+              error: error.class.name,
+              task: task,
+            )
+          else
+            agent.trigger('task', action: 'finish',
+              task: task,
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mamiya/cli/client.rb

@@ -10,7 +10,7 @@ require 'thor'
 module Mamiya
   class CLI < Thor
     class Client < Thor
-      class_option :master, aliases: '-u', type: :string
+      class_option :master, aliases: '-u', type: :string, default: 'http://localhost:7761/'
       class_option :application, aliases: %w(-a --app), type: :string
 
       desc "list-applications", "list applications"

data/lib/mamiya/master.rb

@@ -27,7 +27,7 @@ module Mamiya
     end
 
     def start
-      # Override and stop starting fetcher
+      # Override and stop starting task_queue
       web_start
       serf_start
       monitor_start
@@ -38,11 +38,6 @@ module Mamiya
       super
     end
 
-    # XXX: dupe? Mamiya::Agent::Actions#distribute
-    def distribute(application, package)
-      trigger(:fetch, coalesce: false, application: application, package: package)
-    end
-
     def storage(app)
       config.storage_class.new(
         config[:storage].merge(
@@ -89,9 +84,9 @@ module Mamiya
         options = config[:web] || {}
         rack_options = {
           app: self.web,
-          Port: options[:port].to_i,
-          Host: options[:bind],
-          environment: options[:environment],
+          Port: options[:port] ? options[:port].to_i : 7761,
+          Host: options[:bind] || '0.0.0.0', # TODO: IPv6
+          environment: options[:environment] || :development,
           server: options[:server],
           Logger: logger['web']
         }

data/lib/mamiya/master/agent_monitor_handlers.rb

@@ -2,55 +2,72 @@ require 'mamiya/master'
 
 module Mamiya
   class Master
+    # XXX: TODO:
     module AgentMonitorHandlers
-      def fetch_result__ack(status, payload, event)
-        status['fetcher'] ||= {}
-        status['fetcher']['pending'] = payload['pending']
+      def task__start(status, payload, event)
+        task = payload['task']
 
-        status['fetcher']['pending_jobs'] ||= []
-        status['fetcher']['pending_jobs'] << [payload['application'], payload['package']]
+        status['queues'] ||= {}
+        status['queues'][task['task']] ||= {'queue' => [], 'working' => nil}
+
+        status['queues'][task['task']]['working'] = task
+        status['queues'][task['task']]['queue'].delete task
       end
 
-      def fetch_result__start(status, payload, event)
-        status['fetcher'] ||= {}
-        status['fetcher']['fetching'] = [payload['application'], payload['package']]
+      def task__finalize(status, payload, event)
+        task = payload['task']
 
-        logger.debug "#{status['name']} started to fetch #{payload['application']}/#{payload['package']}"
+        status['queues'] ||= {}
+        status['queues'][task['task']] ||= {'queue' => [], 'working' => nil}
 
-        status['fetcher']['pending_jobs'] ||= []
-        status['fetcher']['pending_jobs'].delete [payload['application'], payload['package']]
+        s = status['queues'][task['task']]
+        if s['working'] == task
+          s['working'] = nil
+        end
+        status['queues'][task['task']]['queue'].delete task
       end
 
-      def fetch_result__error(status, payload, event)
-        status['fetcher'] ||= {}
+      def task__finish(status, payload, event)
+        task = payload['task']
+        logger.error "#{status['name']} finished task #{task['task']}: #{payload['error']}"
 
-        logger.error "#{status['name']} failed to fetch #{payload['application']}/#{payload['package']}: #{payload['error']}"
+        task__finalize(status, payload, event)
 
-        if status['fetcher']['fetching'] == [payload['application'], payload['package']]
-          status['fetcher']['fetching'] = nil
+        method_name = "task___#{task['task']}__finish"
+        if self.respond_to?(method_name)
+          __send__ method_name, status, task
         end
       end
 
-      def fetch_result__success(status, payload, event)
-        status['fetcher'] ||= {}
+      def task__error(status, payload, event)
+        task = payload['task']
+        logger.error "#{status['name']} failed task #{task['task']}: #{payload['error']}"
 
-        logger.info "#{status['name']} fetched #{payload['application']}/#{payload['package']}"
+        task__finalize(status, payload, event)
 
-        if status['fetcher']['fetching'] == [payload['application'], payload['package']]
-          status['fetcher']['fetching'] = nil
+        method_name = "task___#{task['task']}__error"
+        if self.respond_to?(method_name)
+          __send__ method_name, status, task, error
         end
+      end
+
 
+
+      # XXX: move task finish handlers into tasks/
+      def task___fetch__finish(status, task)
         status['packages'] ||= {}
-        status['packages'][payload['application']] ||= []
-        status['packages'][payload['application']] << payload['package']
+        status['packages'][task['app']] ||= []
+
+        unless status['packages'][task['app']].include?(task['pkg'])
+          status['packages'][task['app']] << task['pkg']
+        end
       end
 
-      def fetch_result__remove(status, payload, event)
+      def pkg__remove(status, payload, event)
         status['packages'] ||= {}
         packages = status['packages'][payload['application']]
         packages.delete(payload['package']) if packages
       end
-
     end
   end
 end

data/lib/mamiya/master/web.rb

@@ -21,7 +21,8 @@ module Mamiya
       end
 
       get '/' do
-        "mamiya v#{Mamiya::VERSION}"
+        content_type 'text/plain'
+        "mamiya v#{Mamiya::VERSION}\n"
       end
 
       get '/packages/:application' do
@@ -88,15 +89,29 @@ module Mamiya
 
         statuses.each do |name, status|
           next if status["master"]
-          if status["packages"] && status["packages"][params[:application]] &&
-            status["packages"][params[:application]].include?(params[:package])
+          queue = status["queues"] && status["queues"]["fetch"]
+          packages = status["packages"] && status["packages"][params[:application]]
+
+          task_matcher = -> (task) do
+             task["task"] == "fetch" &&
+             task["app"] == params[:application] &&
+             task["pkg"] == params[:package]
+          end
+
+
+          case
+          when packages && packages.include?(params[:package])
 
             result[:distributed] << name
-          elsif status["fetcher"] && status["fetcher"]["fetching"] && status["fetcher"]["fetching"] == pkg_array
+
+          when queue && queue["working"] && task_matcher.call(queue['working'])
+
             result[:fetching] << name
-          elsif status["fetcher"]["pending_jobs"] &&
-                status["fetcher"]["pending_jobs"].include?(pkg_array)
+
+          when queue['queue'].any?(&task_matcher)
+
             result[:queued] << name
+
           else
             result[:not_distributed] << name
           end
@@ -113,7 +128,7 @@ module Mamiya
         when 0 < result[:queued_count] || 0 < result[:fetching_count]
           status = :distributing
         when 0 < result[:distributed_count] && result[:distributed_count] < total
-          status = :well_distributed
+          status = :partially_distributed
         when result[:distributed_count] == total
           status = :distributed
         else

data/lib/mamiya/version.rb

@@ -1,3 +1,3 @@
 module Mamiya
-  VERSION = "0.0.1.alpha19"
+  VERSION = "0.0.1.alpha20"
 end

data/mamiya.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency "thor", ">= 0.18.1"
-  spec.add_runtime_dependency "aws-sdk-core", "2.0.0.rc14"
+  spec.add_runtime_dependency "aws-sdk-core", "2.0.0.rc15"
   spec.add_runtime_dependency "term-ansicolor", ">= 1.3.0"
   unless ENV["MAMIYA_VILLEIN_PATH"]
     spec.add_runtime_dependency "villein", ">= 0.3.2"

data/spec/agent/actions_spec.rb

@@ -10,7 +10,6 @@ require_relative '../support/dummy_serf.rb'
 
 describe Mamiya::Agent::Actions do
   let(:serf) { DummySerf.new }
-  let(:fetcher) { double('fetcher', start!: nil) }
 
   let(:config) do
     {serf: {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}}}
@@ -18,7 +17,6 @@ describe Mamiya::Agent::Actions do
 
   before do
     allow(Villein::Agent).to receive(:new).and_return(serf)
-    allow(Mamiya::Agent::Fetcher).to receive(:new).and_return(fetcher)
   end
 
   subject(:agent) { Mamiya::Agent.new(config) }
@@ -26,13 +24,9 @@ describe Mamiya::Agent::Actions do
 
   describe "#distribute" do
     it "sends fetch request" do
-      expect(serf).to receive(:event).with(
-        'mamiya:fetch',
-        {application: 'app', package: 'pkg'}.to_json,
-        coalesce: false
-      )
+      expect(agent).to receive(:trigger).with('task', task: 'fetch', app: 'myapp', pkg: 'mypkg', coalesce: false)
 
-      agent.distribute('app', 'pkg')
+      agent.distribute('myapp', 'mypkg')
     end
   end
 end

data/spec/agent/handlers/task_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+
+require 'json'
+require 'villein/event'
+
+require 'mamiya/agent/handlers/task'
+
+describe Mamiya::Agent::Handlers::Task do
+  let(:event) do
+    Villein::Event.new(
+      {
+        'SERF_EVENT' => 'user',
+        'SERF_USER_EVENT' => 'mamiya:task',
+      },
+      payload: {
+        task: 'fetch',
+        application: 'app',
+        package: 'package',
+      }.to_json,
+    )
+  end
+
+  let(:task_queue) { double('task_queue', enqueue: nil) }
+
+  let(:agent) do
+    double('agent', task_queue: task_queue)
+  end
+
+  subject(:handler) { described_class.new(agent, event) }
+
+  before do
+  end
+
+  it "enqueue a request" do
+    expect(task_queue).to receive(:enqueue).with(:fetch, 'task' => 'fetch', 'application' => 'app', 'package' => 'package')
+
+    handler.run!
+  end
+end

data/spec/agent/task_queue_spec.rb

@@ -0,0 +1,246 @@
+require 'spec_helper'
+require 'thread'
+require 'mamiya/agent/tasks/abstract'
+require 'mamiya/agent/task_queue'
+
+describe Mamiya::Agent::TaskQueue do
+  let(:agent) do
+    double('agent')
+  end
+
+  let(:task_class_root) do
+    Class.new(Mamiya::Agent::Tasks::Abstract) do
+      def self.runs
+        @runs ||= []
+      end
+
+      def self.locks
+        @locks ||= {}
+      end
+
+      def self.locks_lock
+        @locks_lock ||= Mutex.new
+      end
+
+      def execute
+        self.class.runs << task.dup
+        if task['wait']
+          begin
+            queue = Queue.new
+            self.class.locks_lock.synchronize do
+              self.class.locks[self.task] = queue
+            end
+            queue.pop
+          ensure
+            self.class.locks_lock.synchronize do
+              self.class.locks.delete self.task
+            end
+          end
+        end
+      end
+    end
+  end
+
+
+  let(:task_class_a) do
+    Class.new(task_class_root) do
+      def self.identifier
+        'a'
+      end
+    end
+  end
+
+  let(:task_class_b) do
+    Class.new(task_class_root) do
+      def self.identifier
+        'b'
+      end
+    end
+  end
+
+  subject(:queue) do
+    described_class.new(agent, task_classes: [task_class_a, task_class_b])
+  end
+
+  describe "lifecycle (#start!, #stop!)" do
+    it "can start and stop" do
+      expect(queue).not_to be_running
+      expect(queue.worker_threads).to be_nil
+
+      queue.start!
+
+      expect(queue).to be_running
+
+      expect(queue.worker_threads).to be_a_kind_of(Hash)
+      expect(queue.worker_threads.values.all? { |v| v.kind_of?(Thread) }).to be_true
+      expect(queue.worker_threads.values.all? { |v| v.alive? }).to be_true
+      threads = queue.worker_threads.dup
+
+      queue.stop!
+
+      expect(queue).not_to be_running
+      expect(queue.worker_threads).to be_nil
+      expect(threads.each_value.all? { |v| !v.alive? }).to be_true
+    end
+
+    it "can stop gracefully"
+  end
+
+  describe "work loop (#enqueue, #running?, #working, #status)" do
+    after do
+      queue.stop! if queue.running?
+    end
+
+    it "run enqueued task" do
+      queue.start!
+
+      queue.enqueue(:a, 'foo' => '1')
+      queue.enqueue(:a, 'foo' => '2')
+      100.times { break if task_class_a.runs.size == 2; sleep 0.01 }
+
+      expect(task_class_a.runs.size).to eq 2
+      expect(task_class_a.runs[0]['foo']).to eq '1'
+      expect(task_class_a.runs[1]['foo']).to eq '2'
+    end
+
+    describe "#working?" do
+      it "returns true if there're any working tasks" do
+        queue.start!
+
+        expect(queue).not_to be_working
+
+        queue.enqueue(:a, 'wait' => true, 'id' => 1)
+        100.times { break unless task_class_a.locks.empty?; sleep 0.01 }
+        expect(task_class_a.locks).not_to be_empty
+        expect(queue).to be_working
+
+        task_class_a.locks.values.last << true
+
+        100.times { break unless queue.working?; sleep 0.01 }
+        expect(queue).not_to be_working
+      end
+    end
+
+
+    describe "#status" do
+      it "shows status" do
+        queue.start!
+
+        expect(queue.status[:a][:working]).to be_nil
+        expect(queue.status[:a][:queue]).to be_a_kind_of(Array)
+        expect(queue.status[:a][:queue]).to be_empty
+
+        queue.enqueue(:a, 'wait' => true, 'id' => 1)
+
+        100.times { break unless task_class_a.locks.empty?; sleep 0.01 }
+        expect(task_class_a.locks).not_to be_empty
+        expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1)
+
+        queue.enqueue(:a, 'id' => 2)
+        100.times { break unless queue.status[:a][:queue].empty?; sleep 0.01 }
+        expect(queue.status[:a][:queue].size).to eq 1
+        expect(queue.status[:a][:queue].first).to eq('id' => 2)
+
+        task_class_a.locks.values.last << true
+
+        100.times { break unless queue.status[:a][:working]; sleep 0.01 }
+        expect(queue.status[:a][:working]).to be_nil
+        expect(queue.status[:a][:queue]).to be_empty
+      end
+    end
+
+    context "with multiple task classes" do
+      it "run enqueued task" do
+        queue.start!
+
+        queue.enqueue(:a, 'foo' => '1')
+        queue.enqueue(:b, 'foo' => '2')
+        100.times { break if task_class_a.runs.size == 1 && task_class_b.runs.size == 1; sleep 0.01 }
+
+        expect(task_class_a.runs.size).to eq 1
+        expect(task_class_b.runs.size).to eq 1
+        expect(task_class_a.runs[0]['foo']).to eq '1'
+        expect(task_class_b.runs[0]['foo']).to eq '2'
+      end
+
+      it "run enqueued task parallel" do
+        queue.start!
+
+        queue.enqueue(:a, 'foo' => '1', 'wait' => true)
+        queue.enqueue(:b, 'foo' => '2', 'wait' => true)
+        100.times { break if task_class_a.locks.size == 1 && task_class_b.locks.size == 1; sleep 0.01 }
+
+        expect(task_class_a.locks.size).to eq 1
+        expect(task_class_b.locks.size).to eq 1
+        task_class_a.locks.each_value.first << true
+        task_class_b.locks.each_value.first << true
+
+        expect(task_class_a.runs.size).to eq 1
+        expect(task_class_b.runs.size).to eq 1
+        expect(task_class_a.runs[0]['foo']).to eq '1'
+        expect(task_class_b.runs[0]['foo']).to eq '2'
+      end
+
+      describe "#status" do
+        it "shows status for each task class" do
+          queue.start!
+
+          expect(queue.status[:a][:working]).to be_nil
+          expect(queue.status[:a][:queue]).to be_a_kind_of(Array)
+          expect(queue.status[:a][:queue]).to be_empty
+
+          expect(queue.status[:b][:working]).to be_nil
+          expect(queue.status[:b][:queue]).to be_a_kind_of(Array)
+          expect(queue.status[:b][:queue]).to be_empty
+
+          queue.enqueue(:a, 'wait' => true, 'id' => 1)
+          queue.enqueue(:b, 'wait' => true, 'id' => 2)
+
+          100.times { break if !task_class_a.locks.empty? && !task_class_a.locks.empty?; sleep 0.01 }
+          expect(task_class_a.locks).not_to be_empty
+          expect(task_class_b.locks).not_to be_empty
+
+          expect(queue.status[:a][:working]).to eq('wait' => true, 'id' => 1)
+          expect(queue.status[:b][:working]).to eq('wait' => true, 'id' => 2)
+
+          queue.enqueue(:a, 'id' => 3)
+          queue.enqueue(:b, 'id' => 4)
+          100.times { break if !queue.status[:a][:queue].empty? && !queue.status[:b][:queue].empty?; sleep 0.01 }
+          expect(queue.status[:a][:queue].size).to eq 1
+          expect(queue.status[:a][:queue].first).to eq('id' => 3)
+          expect(queue.status[:b][:queue].size).to eq 1
+          expect(queue.status[:b][:queue].first).to eq('id' => 4)
+
+          task_class_a.locks.values.last << true
+          task_class_b.locks.values.last << true
+
+          100.times { break if !queue.status[:a][:working] && !queue.status[:b][:working]; sleep 0.01 }
+          expect(queue.status[:a][:working]).to be_nil
+          expect(queue.status[:a][:queue]).to be_empty
+          expect(queue.status[:b][:working]).to be_nil
+          expect(queue.status[:b][:queue]).to be_empty
+
+        end
+      end
+
+      describe "#working?" do
+        it "returns true if there're any working tasks" do
+          queue.start!
+
+          expect(queue).not_to be_working
+
+          queue.enqueue(:a, 'wait' => true, 'id' => 1)
+          100.times { break unless task_class_a.locks.empty?; sleep 0.01 }
+          expect(task_class_a.locks).not_to be_empty
+          expect(queue).to be_working
+
+          task_class_a.locks.values.last << true
+
+          100.times { break unless queue.working?; sleep 0.01 }
+          expect(queue).not_to be_working
+        end
+      end
+
+    end
+  end
+end