mamiya 0.0.1.alpha8 → 0.0.1.alpha9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: aaf47b18849f90c0acb28dcea4b4c367466357af
-  data.tar.gz: 2b3bafda6e415b6adec0626f9dcceeb737a16ae9
+  metadata.gz: 3f7822be0f2ea92b439156730db75fc2608820ec
+  data.tar.gz: 34495ba570dab6778304c2f0cc68b2fdc9914477
 SHA512:
-  metadata.gz: d541ca28c6c904b33ce224226955795b0d1087e36b1a3f0183707a092a3f868ef1a05fcc2edf8b58898319f8cd9aa952d00acac25f0648efc1bc11a41b12c10d
-  data.tar.gz: 122b661348d2b91d397671e49aacd35d81c83ef2a8b9e9b3af0e92753224e516984396e4dc0abc9dd552b5cd0bd1557a12e8a9de461000443b1335fe866e465a
+  metadata.gz: a15d9f6afd8b69c60e12e22609fe1e713f5da3d85c1272923b171fc04946655e2cc6c19142e9af7d1b6e44927a80a59ccc79b59644dc15700a8b898da2a48a19
+  data.tar.gz: c3fddda1d60e1e2b348c6c0b635426abc9d265fe9a42bcbb9b22908063a62e41a4e32f536299d5c69f7841b7e99b5625ce995a4f311a00e25c0e30cd12ec2b64

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+/coverage/

data/docs/cli.md

@@ -0,0 +1,78 @@
+# Commands
+
+## Global options
+
+- `-C`, `--config`: path to configuration file (default: `./config.yml`)
+  - config file won't be required if a command don't need it.
+- `-S`, `--script`: path to script file (default: `./deploy.rb`)
+  - script file won't be required if a command don't need it.
+- `-a`, `--app`: Specify application name to operate
+- `-d`, `--debug`: Enable debug mode
+- `--color`, `--no-color`: Enable or disable colored output. When the stdout is a terminal, it'll be enabled by default.
+
+## Storage related commands
+
+### `list-applications` - list application in storage
+
+```
+$ mamiya list-applications -C ./config.yml
+```
+
+### `list-packages` - list packages for specified app in storage
+
+```
+$ mamiya list-packages -C ./config.yml -a myapp
+```
+
+- __Requires:__ configuration file, application name
+  - application name will be retrieved from deploy script when not specified.
+- __Options:__
+  - `-n`, `--name-only`: Show only names (without heading text)
+
+### `show` - show package information
+
+```
+$ mamiya list-packages -C ./config.yml -a myapp PACKAGE_NAME
+```
+
+- __Requires:__ configuration file, application name, package name
+  - application name will be retrieved from deploy script when not specified.
+- __Options:__
+  - `-f`, `--format`: Choose output format from `pp`, `json`, or `yaml`. Default: `pp`.
+
+## Build, pushing and fetching packages
+
+### `build` - build package using script
+
+```
+$ mamiya build --script ./deploy.rb --source source_dir --destination dest_dir
+```
+
+- __Requires:__ application name, source, destination, deploy script
+  - source, description, and app name will be taken from deploy script, if omitted
+- __Options:__
+  - `-f`, `--source`, `--build-from`: directory for package source.
+  - `-t`, `--destination`, `--build-to`: directory to save built packages.
+  - `-P`, `--skip-prepare-build`: Skip prepare build phase of deploy script.
+
+### `push` - push built package to the storage
+
+### `fetch`
+
+## Package related commands
+
+### `extract`
+
+## `client` - API client for `mamiya master`
+
+### `list-applications`
+
+### `list-packages`
+
+### `show-package`
+
+### `list-agents`
+
+### `show-agent`
+
+### `show-distribution`

data/lib/mamiya/agent/fetcher.rb

@@ -11,52 +11,63 @@ module Mamiya
       GRACEFUL_TIMEOUT = 60
 
       def initialize(config, logger: Mamiya::Logger.new)
-        @thread = nil
-        @queue = Queue.new
+        @worker_thread = nil
+        @queueing_thread = nil
+        @external_queue = Queue.new
+        @internal_queue = Queue.new
 
         @config = config
         @destination = config[:packages_dir]
         @keep_packages = config[:keep_packages]
         @current_job = nil
+        @pending_jobs = []
 
         @logger = logger['fetcher']
-        @working = nil
       end
 
-      attr_reader :thread
+      attr_reader :worker_thread
+      attr_reader :queueing_thread
       attr_reader :current_job
+      attr_reader :pending_jobs
       attr_writer :cleanup_hook
 
       def enqueue(app, package, before: nil, &callback)
-        @queue << [app, package, before, callback]
+        @external_queue << [app, package, before, callback]
       end
 
       def queue_size
-        @queue.size
+        @internal_queue.size
       end
 
       def start!
+        stop!
         @logger.info 'Starting...'
 
-        @thread = Thread.new(&method(:main_loop))
-        @thread.abort_on_exception = true
+        @worker_thread = Thread.new(&method(:main_loop))
+        @worker_thread.abort_on_exception = true
+
+        @queueing_thread = Thread.new(&method(:queueing_loop))
+        @queueing_thread.abort_on_exception = true
       end
 
       def stop!(graceful = false)
-        return unless @thread
+        {@external_queue => @queueing_thread, @internal_queue => @worker_thread}.each do |q, th|
+          next unless th
+          if graceful
+            q << :suicide
+            th.join(GRACEFUL_TIMEOUT)
+          end
 
-        if graceful
-          @queue << :suicide
-          @thread.join(GRACEFUL_TIMEOUT)
+          th.kill if th.alive?
         end
-
-        @thread.kill if @thread.alive?
       ensure
-        @thread = nil
+        @worker_thread = nil
+        @queueing_thread = nil
       end
 
       def running?
-        @thread && @thread.alive?
+        @worker_thread && @worker_thread.alive? && \
+        @queueing_thread && @queueing_thread.alive?
       end
 
       def working?
@@ -88,15 +99,31 @@ module Mamiya
       private
 
       def main_loop
-        while order = @queue.pop
+        while order = @internal_queue.pop
           break if order == :suicide
+          @pending_jobs.delete(order)
           handle_order(*order)
         end
       end
 
+      def queueing_loop
+        while order = @external_queue.pop
+          break if order == :suicide
+          @pending_jobs << order
+          @internal_queue << order
+        end
+      end
+
       def handle_order(app, package, before_hook = nil, callback = nil)
         @current_job = [app, package]
         @logger.info "fetching #{app}:#{package}"
+
+        if @config[:fetch_sleep]
+          wait = rand(@config[:fetch_sleep])
+          @logger.debug "Sleeping #{wait} before starting fetch"
+          sleep wait
+        end
+
         # TODO: Limit apps by configuration
 
         destination = File.join(@destination, app)
@@ -113,6 +140,7 @@ module Mamiya
           config: @config,
         ).run!
 
+        @current_job = nil
         callback.call if callback
 
         @logger.info "fetched #{app}:#{package}"

data/lib/mamiya/agent.rb

@@ -18,6 +18,8 @@ module Mamiya
       @serf = init_serf
       @events_only = events_only
 
+      @terminate = false
+
       @logger = logger['agent']
     end
 
@@ -35,15 +37,30 @@ module Mamiya
       logger.info "Started."
 
       loop do
-        sleep 10
+        if @terminate
+          terminate
+          return
+        end
+        sleep 1
       end
     end
 
+    def stop!
+      @terminate = true
+    end
+
     def start
       serf_start
       fetcher_start
     end
 
+    def terminate
+      serf.stop!
+      fetcher.stop!
+    ensure
+      @terminate = false
+    end
+
     def update_tags!
       serf.tags['mamiya'] = ','.tap do |status|
         status.concat('fetching,') if fetcher.working?
@@ -63,6 +80,7 @@ module Mamiya
         s[:fetcher] = {
           fetching: fetcher.current_job,
           pending: fetcher.queue_size,
+          pending_jobs: fetcher.pending_jobs.map{ |_| _[0,2] },
         }
 
         s[:packages] = self.existing_packages

data/lib/mamiya/cli.rb

@@ -179,8 +179,8 @@ module Mamiya
       prepare_agent_behavior!
       merge_serf_option!
 
-      agent = Agent.new(config, logger: logger)
-      agent.run!
+      @agent = Agent.new(config, logger: logger)
+      @agent.run!
     end
 
     desc "master", "Start master"
@@ -192,8 +192,8 @@ module Mamiya
       prepare_agent_behavior!
       merge_serf_option!
 
-      agent = Master.new(config, logger: logger)
-      agent.run!
+      @agent = Master.new(config, logger: logger)
+      @agent.run!
     end
 
     # def worker
@@ -216,9 +216,13 @@ module Mamiya
       end
 
       trap(:HUP) do
-        $stderr.puts "hi"
         logger.reopen
       end
+
+      trap(:TERM) do
+        puts "Received SIGTERM..."
+        @agent.stop! if @agent
+      end
     end
 
     def config(dont_raise_error = false)

data/lib/mamiya/master/agent_monitor_handlers.rb

@@ -6,11 +6,17 @@ module Mamiya
       def fetch_result__ack(status, payload, event)
         status['fetcher'] ||= {}
         status['fetcher']['pending'] = payload['pending']
+
+        status['fetcher']['pending_jobs'] ||= []
+        status['fetcher']['pending_jobs'] << [payload['application'], payload['package']]
       end
 
       def fetch_result__start(status, payload, event)
         status['fetcher'] ||= {}
         status['fetcher']['fetching'] = [payload['application'], payload['package']]
+
+        status['fetcher']['pending_jobs'] ||= []
+        status['fetcher']['pending_jobs'].delete [payload['application'], payload['package']]
       end
 
       def fetch_result__error(status, payload, event)

data/lib/mamiya/master.rb

@@ -33,6 +33,11 @@ module Mamiya
       monitor_start
     end
 
+    def terminate
+      @agent_monitor.stop!
+      super
+    end
+
     def distribute(application, package)
       trigger(:fetch, application: application, package: package)
     end

data/lib/mamiya/version.rb

@@ -1,3 +1,3 @@
 module Mamiya
-  VERSION = "0.0.1.alpha8"
+  VERSION = "0.0.1.alpha9"
 end

data/mamiya.gemspec

@@ -32,4 +32,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency 'simplecov', '~> 0.7.1'
 end

data/spec/agent/fetcher_spec.rb

@@ -6,6 +6,7 @@ require 'fileutils'
 require 'mamiya/agent/fetcher'
 require 'mamiya/steps/fetch'
 
+
 describe Mamiya::Agent::Fetcher do
   let!(:tmpdir) { Dir.mktmpdir('mamiya-agent-fetcher-spec') }
   after { FileUtils.remove_entry_secure(tmpdir) if File.exist?(tmpdir) }
@@ -18,22 +19,22 @@ describe Mamiya::Agent::Fetcher do
 
   describe "lifecycle" do
     it "can start and stop" do
-      expect(fetcher.thread).to be_nil
+      expect(fetcher.worker_thread).to be_nil
       expect(fetcher).not_to be_running
 
       fetcher.start!
 
       expect(fetcher).to be_running
-      expect(fetcher.thread).to be_a(Thread)
-      expect(fetcher.thread).to be_alive
-      th = fetcher.thread
+      expect(fetcher.worker_thread).to be_a(Thread)
+      expect(fetcher.worker_thread).to be_alive
+      th = fetcher.worker_thread
 
       fetcher.stop!
 
-      10.times { break unless th.alive?; sleep 0.1 }
+      20.times { break unless th.alive?; sleep 0.1 }
       expect(th).not_to be_alive
 
-      expect(fetcher.thread).to be_nil
+      expect(fetcher.worker_thread).to be_nil
       expect(fetcher).not_to be_running
     end
 
@@ -92,6 +93,33 @@ describe Mamiya::Agent::Fetcher do
     end
   end
 
+  describe "#pending_jobs" do
+    before do
+      step = double('fetch-step')
+      allow(step).to receive(:run!)
+      allow(Mamiya::Steps::Fetch).to receive(:new).with(
+        application: 'myapp',
+        package: 'package',
+        destination: File.join(tmpdir, 'myapp'),
+        config: config,
+      ).and_return(step)
+    end
+
+    it "shows remaining jobs" do
+      fetcher.start!; fetcher.worker_thread.kill
+
+      expect {
+        fetcher.enqueue('myapp', 'package')
+        fetcher.stop!(:graceful)
+      }.to change { fetcher.pending_jobs } \
+        .from([]).to([['myapp', 'package', nil, nil]])
+
+      fetcher.start!; fetcher.stop!(:graceful)
+
+      expect(fetcher.pending_jobs).to be_empty
+    end
+  end
+
   describe "mainloop" do
     before do
       allow(step).to receive(:run!)
@@ -122,6 +150,7 @@ describe Mamiya::Agent::Fetcher do
       received = true
 
       fetcher.enqueue('myapp', 'package') do |succeeded|
+        expect(fetcher.working?).to be_false
         received = succeeded
       end
 
@@ -141,11 +170,11 @@ describe Mamiya::Agent::Fetcher do
       expect(fetcher.current_job).to be_nil
 
       received = false
-      fetcher.enqueue('myapp', 'package') do |error|
-        received = true
+      fetcher.enqueue 'myapp', 'package', before: proc { |error|
         expect(fetcher.working?).to be_true
         expect(fetcher.current_job).to eq %w(myapp package)
-      end
+        received = true
+      }
 
       fetcher.stop!(:graceful)
       expect(received).to be_true
@@ -153,6 +182,15 @@ describe Mamiya::Agent::Fetcher do
       expect(fetcher.current_job).to be_nil
     end
 
+    context "with config.fetch_sleep" do
+      it "calls sleep" do
+        config[:fetch_sleep] = 1
+        expect(fetcher).to receive(:sleep)
+        fetcher.enqueue 'myapp', 'package'
+        fetcher.stop!(:graceful)
+      end
+    end
+
     context "with before hook" do
       it "calls callback" do
         run = false

data/spec/agent_spec.rb

@@ -111,7 +111,8 @@ describe Mamiya::Agent do
       allow(agent).to receive(:existing_packages).and_return("app" => ["pkg"])
       allow(fetcher).to receive(:queue_size).and_return(42)
       allow(fetcher).to receive(:working?).and_return(false)
-          allow(fetcher).to receive(:current_job).and_return(nil)
+      allow(fetcher).to receive(:current_job).and_return(nil)
+      allow(fetcher).to receive(:pending_jobs).and_return([['app', 'pkg2', nil, nil]])
     end
 
     subject(:status) { agent.status }
@@ -129,6 +130,10 @@ describe Mamiya::Agent do
         expect(status[:fetcher][:pending]).to eq 42
       end
 
+      it "includes pendings job" do
+        expect(status[:fetcher][:pending_jobs]).to eq([['app', 'pkg2']])
+      end
+
       it "shows fetching status" do
         expect(status[:fetcher][:fetching]).to be_nil
       end

data/spec/master/agent_monitor_spec.rb

@@ -157,20 +157,22 @@ describe Mamiya::Master::AgentMonitor do
         end
 
         it "updates pending" do
-          commit('mamiya:fetch-result:ack', pending: 72)
+          commit('mamiya:fetch-result:ack', pending: 72, application: 'foo', package: 'bar')
           expect(new_status["fetcher"]["pending"]).to eq 72
+          expect(new_status["fetcher"]["pending_jobs"]).to eq [%w(foo bar)]
         end
       end
 
       describe ":start" do
         let(:status) do
-          {fetcher: {fetching: nil, pending: 0}}
+          {fetcher: {fetching: nil, pending: 1, pending_jobs: [%w(app pkg)]}}
         end
 
         it "updates fetching" do
           commit('mamiya:fetch-result:start',
                  application: 'app', package: 'pkg', pending: 0)
           expect(new_status["fetcher"]["fetching"]).to eq ['app', 'pkg']
+          expect(new_status["fetcher"]["pending_jobs"]).to be_empty
         end
       end
 

data/spec/spec_helper.rb

@@ -1,4 +1,9 @@
+require 'simplecov'
 require 'rack/test'
+SimpleCov.start do
+  add_filter "/spec/"
+end
+
 require 'mamiya/storages/mock'
 require 'mamiya/logger'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mamiya
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha8
+  version: 0.0.1.alpha9
 platform: ruby
 authors:
 - Shota Fukumori (sora_h)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-03 00:00:00.000000000 Z
+date: 2014-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.1
 description: Deploy tool using tarballs and serf for lot of servers
 email:
 - her@sorah.jp
@@ -153,6 +167,7 @@ files:
 - Rakefile
 - bin/mamiya
 - config.example.yml
+- docs/cli.md
 - docs/sequences/deploy.png
 - docs/sequences/deploy.uml
 - example.rb