mamiya 0.0.1.alpha2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9a69092961cc7d8939847bd440a24c6a8447bed4
+  data.tar.gz: 4e33b7d99d62ddc876182e0c53e4b92f850215b5
+SHA512:
+  metadata.gz: edd97f707ef719186c9625b37e40d188df1dd8d9275331f87dceb71df01ef7bbb6a7434f91ccc5cf612e704bad30dcadc84c6d9b0dbc44c6977ffb783c379295
+  data.tar.gz: 6df6bfbe6585fa8c4813a59042aead942bef7f94cd5c69444898dcb60fa46b02457394713ff9c37f07487c6780ab431f4b1eceeb3560b4f5cc15fc1251e76913

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format progress

data/.travis.yml

@@ -0,0 +1,16 @@
+language: ruby
+cache: bundler
+rvm:
+  - "2.1.1"
+  - "2.0.0"
+  - "ruby-head"
+
+matrix:
+  allow_failures:
+    - rvm:
+      - "ruby-head"
+  fast_finish: true
+notifications:
+  email:
+    - travis-ci@sorah.jp
+script: bundle exec rspec -fd ./spec

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+if ENV["MAMIYA_VILLEIN_PATH"]
+  gem 'villein', path: File.expand_path(ENV["MAMIYA_VILLEIN_PATH"])
+end
+
+# Specify your gem's dependencies in mamiya.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Shota Fukumori (sora_h)
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Mamiya - Faster deploy tool using tarballs and serf
+
+[![Build Status](https://travis-ci.org/sorah/mamiya.png?branch=master)](https://travis-ci.org/sorah/mamiya)
+
+Mamiya allows you to deploy using without ssh -- tarballs in such storage (Amazon S3), and [serf](http://www.serfdom.io/).
+
+_Still developing_
+
+## Problem for other deploy tool
+
+Existing major deploy tools (capistrano, mina, ...) use SSH to operate servers.
+But connecting to lot of servers via SSH makes deployment slow.
+
+This solves such problem by using [Serf](http://www.serfdom.io/) and tarball on one storage (Amazon S3).
+
+Also, I'm planning to allow to distribute files before the deploy command. I guess this can skip or shorten
+file transferring phase in deploy.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mamiya'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mamiya
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( http://github.com/sorah/mamiya/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/mamiya

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+retried = false
+begin
+  require 'mamiya'
+  require 'mamiya/cli'
+rescue LoadError
+  $:.unshift File.expand_path(File.join(__dir__, '..', 'lib'))
+  if retried
+    raise
+  else
+    retried = true
+    retry
+  end
+end
+
+Mamiya::CLI.start(ARGV)

data/config.example.yml

@@ -0,0 +1,11 @@
+storage:
+  type: s3
+  bucket: ...
+  region: ap-northeast-1
+
+deploy_script_dir: /path/to/deploy/scripts
+packages_dir: ''
+
+apps:
+  myapp:
+    deploy_to: /path/to/myapp

data/docs/sequences/deploy.uml

@@ -0,0 +1,58 @@
+@startuml
+== Building ==
+Master --> Master: Build package
+note right of Master: e.g. assets precompile
+Master --> Storage: Upload package
+
+== Distributing ==
+Master --> Client: Download request
+activate Master
+Master --> Master: Wait all clients to have package to deploy
+
+alt if doesn't have package
+    activate Client
+    Client --> Client: Set status serf tag
+    Client --> Storage: HTTP request
+    Storage --> Client: Response package
+    Client --> Client: Set packages serf tag
+    Client --> Master: Report download finish
+    deactivate Client
+end
+
+deactivate Master
+
+== Preparing ==
+
+Master --> Client: Request to prepare package
+activate Master
+Master --> Master: Wait all clients to have package to deploy
+activate Client
+Client --> Client: Set status serf tag
+Client --> Client: Unpack package
+Client --> Client: Run prepare phase
+note left of Client: e.g. bundle install, config files
+Client --> Client: Set prepared serf tag
+Client --> Client: Set status serf tag
+Client --> Master: Report preparation finish
+deactivate Client
+deactivate Master
+
+== Finalize ==
+
+Master --> Client: Request finalize
+activate Master
+Master --> Master: Wait all clients restarted
+activate Client
+Client --> Client: Set status serf tag
+Client --> Client: Run finalize phase
+note left of Client: e.g. restart web server
+Client --> Master: Report finalize complete
+Client --> Client: Set status serf tag
+Client --> Client: Wait to be reloaded
+Client --> Client: Set status serf tag
+Client --> Client: Set deployed serf tag
+Client --> Master: Report deploy complete
+deactivate Client
+deactivate Master
+
+@enduml

data/example.rb

@@ -0,0 +1,74 @@
+# Setting variables
+set :application, 'myapp'
+set :repository, "some@where:repo.git"
+set :ref, "master"
+
+set :build_from, "/tmp/app-working-copy"
+set :build_to, "/tmp/app-build"
+set :package_under, 'suffix'
+set :dereference_symlinks, true
+
+set :package_to, "#{__dir__}/pkg"
+set :prepare_to, "/home/app/app-prepare"
+set :deploy_to, "/home/app/app"
+
+set :discover_servers, false
+set :on_client_failure, :warn
+
+servers ec2.instances
+filter_servers application, :production # to restrict where server to be used
+
+use :git, skip_naming: true
+use :rails
+
+# chainable
+package_name do
+  "#{full_ref}"
+end
+
+build(:prepend) do
+  # ...
+end
+
+prepare(only: :app) do
+  invoke :'bundle:install'
+end
+
+finalize(except: :web) do
+  run "unicorn", "graceful"
+end
+
+# finalize(:overwrite) do
+#   invoke :'deploy:switch_current'
+#   run "unicorn", "graceful"
+# end
+
+# ---- git.rb
+
+set_default :branch, "master"
+
+task :'git:clone' do
+  run 'git', 'clone', ...
+end
+
+task :'git:checkout' do
+  next if skip_checkout
+  # ...
+end
+
+unless options[:no_build_prepare]
+  prepare_build(:overwrite) do
+    invoke :'git:clone'
+    invoke :'git:checkout'
+  end
+end
+
+# ---- rails.rb
+
+task :'rails:assets_precompile' do
+  # ...
+end
+
+build do
+  invoke :'rails:assets_precompile'
+end

data/lib/mamiya.rb

@@ -0,0 +1,5 @@
+require "mamiya/version"
+
+module Mamiya
+  # Your code goes here...
+end

data/lib/mamiya/agent.rb

@@ -0,0 +1,181 @@
+require 'thread'
+require 'villein'
+require 'mamiya/logger'
+
+require 'mamiya/steps/fetch'
+require 'mamiya/agent/fetcher'
+
+require 'mamiya/agent/handlers/fetch'
+require 'mamiya/agent/actions'
+
+module Mamiya
+  class Agent
+    include Mamiya::Agent::Actions
+    FETCH_REMOVE_EVENT = 'mamiya:fetch-result:remove'
+
+    def initialize(config, logger: Mamiya::Logger.new, events_only: nil)
+      @config = config
+      @serf = init_serf
+      @events_only = events_only
+
+      @logger = logger['agent']
+    end
+
+    attr_reader :config, :serf, :logger
+
+    def fetcher
+      @fetcher ||= Mamiya::Agent::Fetcher.new(config).tap do |f|
+        f.cleanup_hook = self.method(:cleanup_handler)
+      end
+    end
+
+    def run!
+      logger.info "Starting..."
+      start()
+      logger.info "Started."
+
+      loop do
+        sleep 10
+      end
+    end
+
+    def start
+      serf_start
+      fetcher_start
+    end
+
+    def update_tags!
+      serf.tags['mamiya'] = ','.tap do |status|
+        status.concat('fetching,') if fetcher.working?
+        status.concat('ready,') if status == ','
+      end
+
+      nil
+    end
+
+    ##
+    # Returns agent status. Used for HTTP API and `serf query` inspection.
+    def status
+      {}.tap do |s|
+        s[:master] = false
+        s[:name] = serf.name
+
+        s[:fetcher] = {
+          fetching: fetcher.current_job,
+          pending: fetcher.queue_size,
+        }
+
+        s[:packages] = self.existing_packages
+      end
+    end
+
+    ##
+    # Returns hash with existing packages (where valid) by app name.
+    # Packages which has json and tarball is considered as valid.
+    def existing_packages
+      paths_by_app = Dir[File.join(config[:packages_dir], '*', '*.{tar.gz,json}')].group_by { |path|
+        path.split('/')[-2]
+      }
+
+      Hash[
+        paths_by_app.map { |app, paths|
+          names_by_base = paths.group_by do |path|
+            File.basename(path).sub(/\.(?:tar\.gz|json)\z/, '')
+          end
+
+          packages = names_by_base.flat_map { |base, names|
+            names.map do |name|
+              (
+                name.end_with?(".tar.gz") &&
+                names.find { |_| _.end_with?(".json") } &&
+                base
+              ) || nil
+            end
+          }.compact
+
+          [app, packages.sort]
+        }
+      ]
+    end
+
+    def trigger(type, action: nil, **payload)
+      name = "mamiya:#{type}"
+      name << ":#{action}" if action
+
+      serf.event(name, payload.to_json)
+    end
+
+    private
+
+    def init_serf
+      agent_config = (config[:serf] && config[:serf][:agent]) || {}
+      Villein::Agent.new(**agent_config).tap do |serf|
+        serf.on_user_event do |event|
+          user_event_handler(event)
+        end
+
+        serf.respond('mamiya:status') do |event|
+          self.status.to_json
+        end
+      end
+    end
+
+    def serf_start
+      logger.debug "Starting serf"
+
+      @serf.start!
+      @serf.auto_stop
+      @serf.wait_for_ready
+
+      logger.debug "Serf became ready"
+    end
+
+    def fetcher_start
+      logger.debug "Starting fetcher"
+
+      fetcher.start!
+    end
+
+    def user_event_handler(event)
+      user_event, payload = event.user_event, JSON.parse(event.payload)
+
+      return unless user_event.start_with?('mamiya:')
+      user_event = user_event.sub(/^mamiya:/, '')
+
+      type, action = user_event.split(/:/, 2)
+
+      return if @events_only && !@events_only.any?{ |_| _ === type }
+
+      class_name = type.capitalize.gsub(/-./) { |_| _[1].upcase }
+
+      logger.debug "Received user event #{type}"
+      logger.debug payload.inspect
+
+      if Handlers.const_defined?(class_name)
+        handler = Handlers.const_get(class_name).new(self, event)
+        handler.send(action || :run!)
+      else
+        logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
+      end
+    rescue Exception => e
+      logger.fatal("Error during handling event: #{e.inspect}")
+      e.backtrace.each do |line|
+        logger.fatal line.prepend("\t")
+      end
+
+      raise e if $0.end_with?('rspec')
+    rescue JSON::ParserError
+      logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
+    end
+
+    def cleanup_handler(app, package)
+      serf.event(FETCH_REMOVE_EVENT,
+        {
+          name: self.serf.name,
+          application: app,
+          package: package,
+        }.to_json
+      )
+    end
+  end
+end