mamiya 0.0.1.alpha19 → 0.0.1.alpha20

data/lib/mamiya/agent/fetcher.rb

@@ -1,165 +0,0 @@
-require 'thread'
-require 'mamiya/steps/fetch'
-
-require 'mamiya/storages/abstract'
-
-module Mamiya
-  class Agent
-    ##
-    # This class has a queue for fetching packages.
-    class Fetcher
-      GRACEFUL_TIMEOUT = 60
-
-      def initialize(config, logger: Mamiya::Logger.new)
-        @worker_thread = nil
-        @queueing_thread = nil
-        @external_queue = Queue.new
-        @internal_queue = Queue.new
-
-        @config = config
-        @destination = config[:packages_dir]
-        @keep_packages = config[:keep_packages]
-        @current_job = nil
-        @pending_jobs = []
-
-        @logger = logger['fetcher']
-      end
-
-      attr_reader :worker_thread
-      attr_reader :queueing_thread
-      attr_reader :current_job
-      attr_reader :pending_jobs
-      attr_writer :cleanup_hook
-
-      def enqueue(app, package, before: nil, &callback)
-        @external_queue << [app, package, before, callback]
-      end
-
-      def queue_size
-        @internal_queue.size
-      end
-
-      def start!
-        stop!
-        @logger.info 'Starting...'
-
-        @worker_thread = Thread.new(&method(:main_loop))
-        @worker_thread.abort_on_exception = true
-
-        @queueing_thread = Thread.new(&method(:queueing_loop))
-        @queueing_thread.abort_on_exception = true
-      end
-
-      def stop!(graceful = false)
-        {@external_queue => @queueing_thread, @internal_queue => @worker_thread}.each do |q, th|
-          next unless th
-          if graceful
-            q << :suicide
-            th.join(GRACEFUL_TIMEOUT)
-          end
-
-          th.kill if th.alive?
-        end
-      ensure
-        @worker_thread = nil
-        @queueing_thread = nil
-      end
-
-      def running?
-        @worker_thread && @worker_thread.alive? && \
-        @queueing_thread && @queueing_thread.alive?
-      end
-
-      def working?
-        !!@current_job
-      end
-
-      def cleanup
-        Dir[File.join(@destination, '*')].each do |app|
-          packages = Dir[File.join(app, "*.tar.gz")]
-          packages.sort_by! { |_| [File.mtime(_), _] }
-          packages[0...-@keep_packages].each do |victim|
-            @logger.info "Cleaning up: remove #{victim}"
-            File.unlink(victim) if File.exist?(victim)
-
-            meta_victim = victim.sub(/\.tar\.gz\z/, '.json')
-            if File.exist?(meta_victim)
-              @logger.info "Cleaning up: remove #{meta_victim}"
-              File.unlink(meta_victim)
-            end
-
-            package_name = File.basename(victim, '.tar.gz')
-            if @cleanup_hook
-              @cleanup_hook.call(File.basename(app), package_name)
-            end
-          end
-        end
-      end
-
-      private
-
-      def main_loop
-        while order = @internal_queue.pop
-          break if order == :suicide
-          @pending_jobs.delete(order)
-          handle_order(*order)
-        end
-      end
-
-      def queueing_loop
-        while order = @external_queue.pop
-          break if order == :suicide
-          @pending_jobs << order
-          @internal_queue << order
-        end
-      end
-
-      def handle_order(app, package, before_hook = nil, callback = nil)
-        @current_job = [app, package]
-        @logger.info "fetching #{app}:#{package}"
-
-        if @config[:fetch_sleep]
-          wait = rand(@config[:fetch_sleep])
-          @logger.debug "Sleeping #{wait} before starting fetch"
-          sleep wait
-        end
-
-        # TODO: Limit apps by configuration
-
-        destination = File.join(@destination, app)
-
-        Dir.mkdir(destination) unless File.exist?(destination)
-
-        before_hook.call if before_hook
-
-        # TODO: before run hook for agent.update_tags!
-        Mamiya::Steps::Fetch.new(
-          application: app,
-          package: package,
-          destination: destination,
-          config: @config,
-        ).run!
-
-        @current_job = nil
-        callback.call if callback
-
-        @logger.info "fetched #{app}:#{package}"
-
-        cleanup
-
-      rescue Mamiya::Storages::Abstract::AlreadyFetched => e
-        @logger.info "skipped #{app}:#{package} (already fetched)"
-        callback.call(e) if callback
-      rescue Exception => e
-        @logger.fatal "fetch failed (#{app}:#{package}): #{e.inspect}"
-        e.backtrace.each do |line|
-          @logger.fatal "\t#{line}"
-        end
-
-        callback.call(e) if callback
-      ensure
-        @current_job = nil
-      end
-    end
-  end
-end

data/lib/mamiya/agent/handlers/fetch.rb

@@ -1,78 +0,0 @@
-require 'mamiya/agent/handlers/abstract'
-require 'mamiya/storages/abstract'
-
-module Mamiya
-  class Agent
-    module Handlers
-      class Fetch < Abstract
-        FETCH_ACK_EVENT = 'mamiya:fetch-result:ack'
-        FETCH_START_EVENT = 'mamiya:fetch-result:start'
-        FETCH_SUCCESS_EVENT = 'mamiya:fetch-result:success'
-        FETCH_ERROR_EVENT = 'mamiya:fetch-result:error'
-
-        IGNORED_ERRORS = [
-          Mamiya::Storages::Abstract::AlreadyFetched.new(''),
-        ].freeze
-
-        def run!
-          # XXX: Why don't use agent#trigger ...
-          agent.serf.event(FETCH_ACK_EVENT,
-            {
-              name: agent.serf.name,
-              application: payload['application'],
-              package: payload['package'],
-              pending: agent.fetcher.queue_size.succ,
-            }.to_json,
-            coalesce: false,
-          )
-
-          agent.fetcher.enqueue(
-            payload['application'], payload['package'],
-            before: proc {
-              agent.serf.event(FETCH_START_EVENT,
-                {
-                  name: agent.serf.name,
-                  application: payload['application'],
-                  package: payload['package'],
-                  pending: agent.fetcher.queue_size.succ,
-                }.to_json,
-                coalesce: false,
-              )
-              agent.update_tags!
-            }
-          ) do |error|
-            if error && IGNORED_ERRORS.lazy.grep(error.class).none?
-              # FIXME: TODO: may exceed 256
-              begin
-                agent.serf.event(FETCH_ERROR_EVENT,
-                  {
-                    name: agent.serf.name,
-                    application: payload['application'],
-                    package: payload['package'],
-                    error: error.class,
-                    pending: agent.fetcher.queue_size,
-                  }.to_json,
-                  coalesce: false,
-                )
-              rescue Villein::Client::SerfError => e
-                agent.logger.error "error sending fetch error event: #{e.inspect}"
-              end
-            else
-              agent.serf.event(FETCH_SUCCESS_EVENT,
-                {
-                  name: agent.serf.name,
-                  application: payload['application'],
-                  package: payload['package'],
-                  pending: agent.fetcher.queue_size,
-                }.to_json,
-                coalesce: false,
-              )
-            end
-
-            agent.update_tags!
-          end
-        end
-      end
-    end
-  end
-end

data/spec/agent/fetcher_spec.rb

@@ -1,237 +0,0 @@
-require 'spec_helper'
-require 'pathname'
-require 'tmpdir'
-require 'fileutils'
-
-require 'mamiya/agent/fetcher'
-require 'mamiya/steps/fetch'
-
-
-describe Mamiya::Agent::Fetcher do
-  let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-fetcher-spec') }
-  after { FileUtils.remove_entry_secure(tmpdir) if File.exist?(tmpdir) }
-
-  let(:config) do
-    {packages_dir: tmpdir, keep_packages: 2}
-  end
-
-  subject(:fetcher) { described_class.new(config) }
-
-  describe "lifecycle" do
-    it "can start and stop" do
-      expect(fetcher.worker_thread).to be_nil
-      expect(fetcher).not_to be_running
-
-      fetcher.start!
-
-      expect(fetcher).to be_running
-      expect(fetcher.worker_thread).to be_a(Thread)
-      expect(fetcher.worker_thread).to be_alive
-      th = fetcher.worker_thread
-
-      fetcher.stop!
-
-      20.times { break unless th.alive?; sleep 0.1 }
-      expect(th).not_to be_alive
-
-      expect(fetcher.worker_thread).to be_nil
-      expect(fetcher).not_to be_running
-    end
-
-    it "can graceful stop"
-  end
-
-  describe "#cleanup" do
-    before do
-      path = Pathname.new(tmpdir)
-
-      path.join('a').mkdir
-      File.write path.join('a', "a.tar.gz"), "\n"
-      File.write path.join('a', "a.json"), "\n"
-      File.write path.join('a', "b.json"), "\n"
-      File.write path.join('a', "b.tar.gz"), "\n"
-      File.write path.join('a', "c.json"), "\n"
-      File.write path.join('a', "c.tar.gz"), "\n"
-      path.join('b').mkdir
-      File.write path.join('b', "a.tar.gz"), "\n"
-      File.write path.join('b', "a.json"), "\n"
-
-      path.join('c').mkdir
-      File.write path.join('c', "a.tar.gz"), "\n"
-      File.write path.join('c', "b.json"), "\n"
-    end
-
-    it "cleans up" do
-      called = []
-      fetcher.cleanup_hook = proc { |a,b| called << [a,b] }
-
-      fetcher.cleanup
-
-      path = Pathname.new(tmpdir)
-      existences = Hash[
-        [
-          path.join('a', 'a.tar.gz'),
-          path.join('a', 'a.json'),
-          path.join('a', 'b.tar.gz'),
-          path.join('a', 'b.json'),
-          path.join('a', 'c.tar.gz'),
-          path.join('a', 'c.json'),
-        ].map { |file|
-          [file, file.exist?]
-        }
-      ]
-
-      expect(called).to eq([['a', 'a']])
-      expect(existences).to eq(
-        path.join('a', 'a.tar.gz') => false,
-        path.join('a', 'a.json') => false,
-        path.join('a', 'b.tar.gz') => true,
-        path.join('a', 'b.json') => true,
-        path.join('a', 'c.tar.gz') => true,
-        path.join('a', 'c.json') => true,
-      )
-    end
-  end
-
-  describe "#pending_jobs" do
-    before do
-      step = double('fetch-step')
-      allow(step).to receive(:run!)
-      allow(Mamiya::Steps::Fetch).to receive(:new).with(
-        application: 'myapp',
-        package: 'package',
-        destination: File.join(tmpdir, 'myapp'),
-        config: config,
-      ).and_return(step)
-    end
-
-    it "shows remaining jobs" do
-      fetcher.start!; fetcher.worker_thread.kill
-
-      expect {
-        fetcher.enqueue('myapp', 'package')
-        fetcher.stop!(:graceful)
-      }.to change { fetcher.pending_jobs } \
-        .from([]).to([['myapp', 'package', nil, nil]])
-
-      fetcher.start!; fetcher.stop!(:graceful)
-
-      expect(fetcher.pending_jobs).to be_empty
-    end
-  end
-
-  describe "mainloop" do
-    before do
-      allow(step).to receive(:run!)
-      allow(Mamiya::Steps::Fetch).to receive(:new).with(
-        application: 'myapp',
-        package: 'package',
-        destination: File.join(tmpdir, 'myapp'),
-        config: config,
-      ).and_return(step)
-
-      fetcher.start!
-    end
-
-    let(:step) { double('fetch-step') }
-
-    it "starts fetch step for each order" do
-      flag = false
-
-      expect(step).to receive(:run!) do
-        flag = true
-      end
-
-      fetcher.enqueue('myapp', 'package')
-      fetcher.stop!(:graceful)
-    end
-
-    it "calls callback" do
-      received = true
-
-      fetcher.enqueue('myapp', 'package') do |succeeded|
-        expect(fetcher.working?).to be_false
-        received = succeeded
-      end
-
-      fetcher.stop!(:graceful)
-
-      expect(received).to be_nil
-    end
-
-    it "calls cleanup" do
-      expect(fetcher).to receive(:cleanup)
-      fetcher.enqueue('myapp', 'package')
-      fetcher.stop!(:graceful)
-    end
-
-    it "claims itself as working" do
-      expect(fetcher.working?).to be_false
-      expect(fetcher.current_job).to be_nil
-
-      received = false
-      fetcher.enqueue 'myapp', 'package', before: proc { |error|
-        expect(fetcher.working?).to be_true
-        expect(fetcher.current_job).to eq %w(myapp package)
-        received = true
-      }
-
-      fetcher.stop!(:graceful)
-      expect(received).to be_true
-      expect(fetcher.working?).to be_false
-      expect(fetcher.current_job).to be_nil
-    end
-
-    context "with config.fetch_sleep" do
-      it "calls sleep" do
-        config[:fetch_sleep] = 1
-        expect(fetcher).to receive(:sleep)
-        fetcher.enqueue 'myapp', 'package'
-        fetcher.stop!(:graceful)
-      end
-    end
-
-    context "with before hook" do
-      it "calls callback" do
-        run = false
-        received = false
-
-        allow(step).to receive(:run!) do
-          run = true
-        end
-
-        fetcher.enqueue('myapp', 'package', before: proc {
-          received = true
-          expect(run).to be_false
-        })
-        fetcher.stop!(:graceful)
-
-        expect(received).to be_true
-      end
-    end
-
-    context "when fetch step raised error" do
-      let(:exception) { Exception.new("he he...") }
-
-      before do
-        allow(step).to receive(:run!).and_raise(exception)
-      end
-
-      it "calls callback with error" do
-        received = nil
-
-        fetcher.enqueue('myapp', 'package') do |error|
-          received = error
-        end
-
-        fetcher.stop!(:graceful)
-
-        expect(received).to eq exception
-      end
-    end
-
-    after do
-      fetcher.stop!
-    end
-  end
-end