mamiya 0.0.1.alpha2

data/lib/mamiya/version.rb

@@ -0,0 +1,3 @@
+module Mamiya
+  VERSION = "0.0.1.alpha2"
+end

data/mamiya.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mamiya/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mamiya"
+  spec.version       = Mamiya::VERSION
+  spec.authors       = ["Shota Fukumori (sora_h)"]
+  spec.email         = ["her@sorah.jp"]
+  spec.summary       = %q{Fast deploy tool using tarballs and serf}
+  spec.description   = %q{Deploy tool using tarballs and serf for lot of servers}
+  spec.homepage      = "https://github.com/sorah/mamiya"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency "thor", "~> 0.18.1"
+  spec.add_runtime_dependency "aws-sdk-core", "2.0.0.rc6"
+  spec.add_runtime_dependency "term-ansicolor", ">= 1.3.0"
+  unless ENV["MAMIYA_VILLEIN_PATH"]
+    spec.add_runtime_dependency "villein", ">= 0.3.1"
+  end
+
+  spec.add_runtime_dependency "sinatra", ">= 1.4.5"
+
+  spec.add_development_dependency "rspec", "2.14.1"
+  spec.add_development_dependency 'rack-test', '~> 0.6.2'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

data/misc/logger_test.rb

@@ -0,0 +1,12 @@
+require 'bundler/setup'
+$:.unshift File.join(__dir__, '..' ,'lib')
+require 'mamiya/logger'
+
+[true,false].each {|color|
+  l = Mamiya::Logger.new(level: Logger::DEBUG, color: color)
+  [nil, "app"].each {|n|
+    (0..5).each {|s|
+      l.log s, "hi", n
+    }
+  }
+}

data/spec/agent/actions_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+require 'json'
+require 'villein/event'
+
+require 'mamiya/agent'
+require 'mamiya/agent/actions'
+
+require_relative '../support/dummy_serf.rb'
+
+describe Mamiya::Agent::Actions do
+  let(:serf) { DummySerf.new }
+  let(:fetcher) { double('fetcher', start!: nil) }
+
+  let(:config) do
+    {serf: {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}}}
+  end
+
+  before do
+    allow(Villein::Agent).to receive(:new).and_return(serf)
+    allow(Mamiya::Agent::Fetcher).to receive(:new).and_return(fetcher)
+  end
+
+  subject(:agent) { Mamiya::Agent.new(config) }
+
+
+  describe "#distribute" do
+    it "sends fetch request" do
+      expect(serf).to receive(:event).with(
+        'mamiya:fetch',
+        {application: 'app', package: 'pkg'}.to_json
+      )
+
+      agent.distribute('app', 'pkg')
+    end
+  end
+end

data/spec/agent/fetcher_spec.rb

@@ -0,0 +1,199 @@
+require 'spec_helper'
+require 'pathname'
+require 'tmpdir'
+require 'fileutils'
+
+require 'mamiya/agent/fetcher'
+require 'mamiya/steps/fetch'
+
+describe Mamiya::Agent::Fetcher do
+  let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-fetcher-spec') }
+  after { FileUtils.remove_entry_secure(tmpdir) if File.exist?(tmpdir) }
+
+  let(:config) do
+    {packages_dir: tmpdir, keep_packages: 2}
+  end
+
+  subject(:fetcher) { described_class.new(config) }
+
+  describe "lifecycle" do
+    it "can start and stop" do
+      expect(fetcher.thread).to be_nil
+      expect(fetcher).not_to be_running
+
+      fetcher.start!
+
+      expect(fetcher).to be_running
+      expect(fetcher.thread).to be_a(Thread)
+      expect(fetcher.thread).to be_alive
+      th = fetcher.thread
+
+      fetcher.stop!
+
+      10.times { break unless th.alive?; sleep 0.1 }
+      expect(th).not_to be_alive
+
+      expect(fetcher.thread).to be_nil
+      expect(fetcher).not_to be_running
+    end
+
+    it "can graceful stop"
+  end
+
+  describe "#cleanup" do
+    before do
+      path = Pathname.new(tmpdir)
+
+      path.join('a').mkdir
+      File.write path.join('a', "a.tar.gz"), "\n"
+      File.write path.join('a', "a.json"), "\n"
+      File.write path.join('a', "b.json"), "\n"
+      File.write path.join('a', "b.tar.gz"), "\n"
+      File.write path.join('a', "c.json"), "\n"
+      File.write path.join('a', "c.tar.gz"), "\n"
+      path.join('b').mkdir
+      File.write path.join('b', "a.tar.gz"), "\n"
+      File.write path.join('b', "a.json"), "\n"
+
+      path.join('c').mkdir
+      File.write path.join('c', "a.tar.gz"), "\n"
+      File.write path.join('c', "b.json"), "\n"
+    end
+
+    it "cleans up" do
+      called = []
+      fetcher.cleanup_hook = proc { |a,b| called << [a,b] }
+
+      fetcher.cleanup
+
+      path = Pathname.new(tmpdir)
+      existences = Hash[
+        [
+          path.join('a', 'a.tar.gz'),
+          path.join('a', 'a.json'),
+          path.join('a', 'b.tar.gz'),
+          path.join('a', 'b.json'),
+          path.join('a', 'c.tar.gz'),
+          path.join('a', 'c.json'),
+        ].map { |file|
+          [file, file.exist?]
+        }
+      ]
+
+      expect(called).to eq([['a', 'a']])
+      expect(existences).to eq(
+        path.join('a', 'a.tar.gz') => false,
+        path.join('a', 'a.json') => false,
+        path.join('a', 'b.tar.gz') => true,
+        path.join('a', 'b.json') => true,
+        path.join('a', 'c.tar.gz') => true,
+        path.join('a', 'c.json') => true,
+      )
+    end
+  end
+
+  describe "mainloop" do
+    before do
+      allow(step).to receive(:run!)
+      allow(Mamiya::Steps::Fetch).to receive(:new).with(
+        application: 'myapp',
+        package: 'package',
+        destination: File.join(tmpdir, 'myapp'),
+        config: config,
+      ).and_return(step)
+
+      fetcher.start!
+    end
+
+    let(:step) { double('fetch-step') }
+
+    it "starts fetch step for each order" do
+      flag = false
+
+      expect(step).to receive(:run!) do
+        flag = true
+      end
+
+      fetcher.enqueue('myapp', 'package')
+      fetcher.stop!(:graceful)
+    end
+
+    it "calls callback" do
+      received = true
+
+      fetcher.enqueue('myapp', 'package') do |succeeded|
+        received = succeeded
+      end
+
+      fetcher.stop!(:graceful)
+
+      expect(received).to be_nil
+    end
+
+    it "calls cleanup" do
+      expect(fetcher).to receive(:cleanup)
+      fetcher.enqueue('myapp', 'package')
+      fetcher.stop!(:graceful)
+    end
+
+    it "claims itself as working" do
+      expect(fetcher.working?).to be_false
+      expect(fetcher.current_job).to be_nil
+
+      received = false
+      fetcher.enqueue('myapp', 'package') do |error|
+        received = true
+        expect(fetcher.working?).to be_true
+        expect(fetcher.current_job).to eq %w(myapp package)
+      end
+
+      fetcher.stop!(:graceful)
+      expect(received).to be_true
+      expect(fetcher.working?).to be_false
+      expect(fetcher.current_job).to be_nil
+    end
+
+    context "with before hook" do
+      it "calls callback" do
+        run = false
+        received = false
+
+        allow(step).to receive(:run!) do
+          run = true
+        end
+
+        fetcher.enqueue('myapp', 'package', before: proc {
+          received = true
+          expect(run).to be_false
+        })
+        fetcher.stop!(:graceful)
+
+        expect(received).to be_true
+      end
+    end
+
+    context "when fetch step raised error" do
+      let(:exception) { Exception.new("he he...") }
+
+      before do
+        allow(step).to receive(:run!).and_raise(exception)
+      end
+
+      it "calls callback with error" do
+        received = nil
+
+        fetcher.enqueue('myapp', 'package') do |error|
+          received = error
+        end
+
+        fetcher.stop!(:graceful)
+
+        expect(received).to eq exception
+      end
+    end
+
+    after do
+      fetcher.stop!
+    end
+  end
+end

data/spec/agent/handlers/fetch_spec.rb

@@ -0,0 +1,121 @@
+require 'spec_helper'
+
+require 'json'
+require 'villein/event'
+
+require 'mamiya/agent/handlers/fetch'
+
+describe Mamiya::Agent::Handlers::Fetch do
+  let(:event) do
+    Villein::Event.new(
+      {
+        'SERF_EVENT' => 'user',
+        'SERF_USER_EVENT' => 'mamiya:fetch',
+      },
+      payload: {
+        application: 'app',
+        package: 'package',
+      }.to_json,
+    )
+  end
+
+  let(:fetcher) { double('fetcher', start!: nil) }
+  let(:serf) { double('serf', name: 'myname', event: nil) }
+
+  let(:agent) do
+    double('agent', fetcher: fetcher, serf: serf, update_tags!: nil)
+  end
+
+  subject(:handler) { described_class.new(agent, event) }
+
+  before do
+    allow(fetcher).to receive(:queue_size).and_return(0)
+    allow(fetcher).to receive(:enqueue) do |&block|
+      block.call
+    end
+  end
+
+  it "enqueue a request" do
+    expect(fetcher).to receive(:enqueue).with('app', 'package', kind_of(Hash))
+
+    handler.run!
+  end
+
+  it "responds ack" do
+    allow(fetcher).to receive(:enqueue).with('app', 'package')
+    expect(serf).to receive(:event).with('mamiya:fetch-result:ack',
+      {name: serf.name, application: 'app', package: 'package', pending: 1}.to_json)
+
+    handler.run!
+  end
+
+  it "kicks update_tag! on start" do
+    allow(fetcher).to receive(:enqueue) do |app, package, options, &block|
+      options[:before].call
+    end
+    expect(agent).to receive(:update_tags!)
+    handler.run!
+  end
+
+  it "reports start" do
+    allow(fetcher).to receive(:enqueue) do |app, package, options, &block|
+      options[:before].call
+    end
+    expect(serf).to receive(:event).with('mamiya:fetch-result:start',
+      {name: serf.name, application: 'app', package: 'package', pending: 1}.to_json)
+
+    handler.run!
+  end
+
+  it "responds success" do
+    callback = nil
+    allow(fetcher).to receive(:enqueue).with('app', 'package', kind_of(Hash)) do |&block|
+      callback = block
+    end
+
+    handler.run!
+
+    expect(serf).to receive(:event).with(
+      'mamiya:fetch-result:success',
+      {name: serf.name, application: 'app', package: 'package', pending: 0}.to_json
+    )
+
+    callback.call
+  end
+
+  it "updates tag" do
+    expect(agent).to receive(:update_tags!)
+    handler.run!
+  end
+
+  context "when failed" do
+    it "notifies error" do
+      callback = nil
+      allow(fetcher).to receive(:enqueue).with('app', 'package', kind_of(Hash)) do |&block|
+        callback = block
+      end
+
+      handler.run!
+
+      error = RuntimeError.new('test')
+      expect(serf).to receive(:event).with(
+        'mamiya:fetch-result:error',
+        {
+          name: serf.name, application: 'app', package: 'package',
+          error: error.inspect, pending: 0,
+        }.to_json,
+      )
+
+      callback.call(error)
+    end
+
+    it "updates tag" do
+      allow(fetcher).to receive(:enqueue) do |&block|
+        block.call(Exception.new)
+      end
+
+      expect(agent).to receive(:update_tags!)
+      handler.run!
+    end
+  end
+end

data/spec/agent_spec.rb

@@ -0,0 +1,255 @@
+require 'spec_helper'
+
+require 'tmpdir'
+require 'fileutils'
+require 'json'
+require 'villein/event'
+
+require 'mamiya/agent'
+require 'mamiya/agent/fetcher'
+require 'mamiya/agent/actions'
+
+require_relative './support/dummy_serf.rb'
+
+describe Mamiya::Agent do
+
+  let(:serf) { DummySerf.new }
+  let(:fetcher) do
+    double('fetcher', start!: nil, working?: false).tap do |f|
+      cleanup_hook = nil
+      allow(f).to receive(:cleanup_hook=) { |_| cleanup_hook = _ }
+      allow(f).to receive(:cleanup_hook)  { cleanup_hook }
+    end
+  end
+
+  let(:config) do
+    {serf: {agent: {rpc_addr: '127.0.0.1:17373', bind: '127.0.0.1:17946'}}}
+  end
+
+  before do
+    allow(Villein::Agent).to receive(:new).and_return(serf)
+    allow(Mamiya::Agent::Fetcher).to receive(:new).and_return(fetcher)
+  end
+
+  subject(:agent) { described_class.new(config) }
+
+  it "includes actions" do
+    expect(described_class.ancestors).to include(Mamiya::Agent::Actions)
+  end
+
+  describe "fetcher" do
+    it "sends events on cleanup hook" do
+      expect(serf).to receive(:event).with(
+        'mamiya:fetch-result:remove',
+        {
+          name: serf.name, application: 'foo', package: 'bar',
+        }.to_json,
+      )
+
+      agent.fetcher.cleanup_hook.call('foo', 'bar')
+    end
+  end
+
+  describe "#run!" do
+    it "starts serf and fetcher" do
+      begin
+        flag = false
+
+        expect(fetcher).to receive(:start!)
+        expect(serf).to receive(:start!)
+        expect(serf).to receive(:auto_stop) do
+          flag = true
+        end
+
+        th = Thread.new { agent.run! }
+
+        10.times { break if flag; sleep 0.1 }
+      ensure
+        th.kill if th && th.alive?
+      end
+    end
+  end
+
+  describe "#update_tags!" do
+    describe "(status)" do
+      context "when it is not busy" do
+        it "shows ready" do
+          agent.update_tags!
+
+          expect(serf.tags['mamiya']).to eq ',ready,'
+        end
+      end
+
+      context "when it is fetching" do
+        before do
+          allow(fetcher).to receive(:working?).and_return(true)
+        end
+
+        it "shows fetching" do
+          agent.update_tags!
+
+          expect(serf.tags['mamiya']).to eq ',fetching,'
+        end
+      end
+
+      context "when it is running multiple jobs" do
+        pending
+      end
+    end
+
+    describe "(prepared)" do
+      pending
+    end
+
+    describe "(current)" do
+      pending
+    end
+  end
+
+  describe "#status" do
+    before do
+      allow(agent).to receive(:existing_packages).and_return("app" => ["pkg"])
+      allow(fetcher).to receive(:queue_size).and_return(42)
+      allow(fetcher).to receive(:working?).and_return(false)
+          allow(fetcher).to receive(:current_job).and_return(nil)
+    end
+
+    subject(:status) { agent.status }
+
+    it "includes agent name" do
+      expect(status[:name]).to eq serf.name
+    end
+
+    it "includes packages" do
+      expect(status[:packages]).to eq agent.existing_packages
+    end
+
+    describe "(fetcher)" do
+      it "includes queue_size as pending" do
+        expect(status[:fetcher][:pending]).to eq 42
+      end
+
+      it "shows fetching status" do
+        expect(status[:fetcher][:fetching]).to be_nil
+      end
+
+      context "when it's fetching" do
+        before do
+          allow(fetcher).to receive(:working?).and_return(true)
+          allow(fetcher).to receive(:current_job).and_return(%w(foo bar))
+        end
+
+        it "shows fetching true" do
+          expect(status[:fetcher][:fetching]).to eq ['foo', 'bar']
+        end
+      end
+    end
+  end
+
+  describe "#existing_packages" do
+    let!(:packages_dir) { Dir.mktmpdir('mamiya-agent-spec') }
+    after { FileUtils.remove_entry_secure(packages_dir) }
+
+    let(:config) { {packages_dir: packages_dir} }
+
+    subject(:existing_packages) { agent.existing_packages }
+
+    before do
+      dir = Pathname.new(packages_dir)
+      %w(a b).each do |app|
+        dir.join(app).mkdir
+        %w(valid.tar.gz valid.json
+           valid-2.tar.gz valid-2.json
+           invalid-1.tar.gz invalid-2.json invalid-3.txt).each do |name|
+          File.write dir.join(app, name), "\n"
+        end
+      end
+    end
+
+    it "returns valid packages" do
+      expect(existing_packages).to eq(
+        "a" => ["valid", "valid-2"],
+        "b" => ["valid", "valid-2"],
+      )
+    end
+  end
+
+  describe "query responder" do
+    it "responds to 'mamiya:status'" do
+      allow(agent).to receive(:status).and_return("my" => "status")
+
+      response = serf.trigger_query('mamiya:status', '')
+      expect(JSON.parse(response)).to eq("my" => "status")
+    end
+  end
+
+  describe "event handler" do
+    let(:handler_class) do
+      Class.new(Mamiya::Agent::Handlers::Abstract) do
+      end
+    end
+
+    def trigger(name, payload={})
+      serf.trigger('user_event', Villein::Event.new(
+        {
+          'SERF_EVENT' => 'user',
+          'SERF_USER_EVENT' => "mamiya:#{name}",
+        },
+        payload: payload.to_json,
+      ))
+    end
+
+    before do
+      stub_const("Mamiya::Agent::Handlers::Test", handler_class)
+      agent # to create
+    end
+
+    it "finds handler class then call #run!" do
+      expect_any_instance_of(handler_class).to receive(:run!)
+
+      trigger('test')
+    end
+
+    it "passes proper argument to handler"
+
+    context "when handler not found" do
+      it "ignores event"
+    end
+
+    context "with events_only" do
+      subject(:agent) { described_class.new(config, events_only: [/foo/]) }
+
+      let(:handler_foo) do
+        Class.new(Mamiya::Agent::Handlers::Abstract) do
+        end
+      end
+
+      let(:handler_bar) do
+        Class.new(Mamiya::Agent::Handlers::Abstract) do
+          def run!
+            raise 'oops?'
+          end
+        end
+      end
+
+      before do
+        stub_const("Mamiya::Agent::Handlers::Foo", handler_foo)
+        stub_const("Mamiya::Agent::Handlers::Bar", handler_bar)
+      end
+
+      it "handles events only matches any of them" do
+        expect_any_instance_of(handler_foo).to receive(:run!)
+
+        trigger('foo')
+        trigger('bar')
+      end
+    end
+
+    context "with action" do
+      it "calls another method instead of run!" do
+        expect_any_instance_of(handler_class).to receive(:hello)
+        trigger('test:hello')
+      end
+    end
+  end
+end