mamiya 0.0.1.alpha2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +17 -0
- data/.rspec +2 -0
- data/.travis.yml +16 -0
- data/Gemfile +8 -0
- data/LICENSE.txt +22 -0
- data/README.md +43 -0
- data/Rakefile +6 -0
- data/bin/mamiya +17 -0
- data/config.example.yml +11 -0
- data/docs/sequences/deploy.png +0 -0
- data/docs/sequences/deploy.uml +58 -0
- data/example.rb +74 -0
- data/lib/mamiya.rb +5 -0
- data/lib/mamiya/agent.rb +181 -0
- data/lib/mamiya/agent/actions.rb +12 -0
- data/lib/mamiya/agent/fetcher.rb +137 -0
- data/lib/mamiya/agent/handlers/abstract.rb +20 -0
- data/lib/mamiya/agent/handlers/fetch.rb +68 -0
- data/lib/mamiya/cli.rb +322 -0
- data/lib/mamiya/cli/client.rb +172 -0
- data/lib/mamiya/config.rb +57 -0
- data/lib/mamiya/dsl.rb +192 -0
- data/lib/mamiya/helpers/git.rb +75 -0
- data/lib/mamiya/logger.rb +190 -0
- data/lib/mamiya/master.rb +118 -0
- data/lib/mamiya/master/agent_monitor.rb +146 -0
- data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
- data/lib/mamiya/master/web.rb +148 -0
- data/lib/mamiya/package.rb +122 -0
- data/lib/mamiya/script.rb +117 -0
- data/lib/mamiya/steps/abstract.rb +19 -0
- data/lib/mamiya/steps/build.rb +72 -0
- data/lib/mamiya/steps/extract.rb +26 -0
- data/lib/mamiya/steps/fetch.rb +24 -0
- data/lib/mamiya/steps/push.rb +34 -0
- data/lib/mamiya/storages.rb +17 -0
- data/lib/mamiya/storages/abstract.rb +48 -0
- data/lib/mamiya/storages/mock.rb +61 -0
- data/lib/mamiya/storages/s3.rb +127 -0
- data/lib/mamiya/util/label_matcher.rb +38 -0
- data/lib/mamiya/version.rb +3 -0
- data/mamiya.gemspec +35 -0
- data/misc/logger_test.rb +12 -0
- data/spec/agent/actions_spec.rb +37 -0
- data/spec/agent/fetcher_spec.rb +199 -0
- data/spec/agent/handlers/fetch_spec.rb +121 -0
- data/spec/agent_spec.rb +255 -0
- data/spec/config_spec.rb +50 -0
- data/spec/dsl_spec.rb +291 -0
- data/spec/fixtures/dsl_test_load.rb +1 -0
- data/spec/fixtures/dsl_test_use.rb +1 -0
- data/spec/fixtures/helpers/foo.rb +1 -0
- data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
- data/spec/fixtures/test-package-source/greeting +1 -0
- data/spec/fixtures/test-package.tar.gz +0 -0
- data/spec/fixtures/test.yml +4 -0
- data/spec/logger_spec.rb +68 -0
- data/spec/master/agent_monitor_spec.rb +269 -0
- data/spec/master/web_spec.rb +121 -0
- data/spec/master_spec.rb +94 -0
- data/spec/package_spec.rb +394 -0
- data/spec/script_spec.rb +78 -0
- data/spec/spec_helper.rb +38 -0
- data/spec/steps/build_spec.rb +261 -0
- data/spec/steps/extract_spec.rb +68 -0
- data/spec/steps/fetch_spec.rb +96 -0
- data/spec/steps/push_spec.rb +73 -0
- data/spec/storages/abstract_spec.rb +22 -0
- data/spec/storages/s3_spec.rb +342 -0
- data/spec/storages_spec.rb +33 -0
- data/spec/support/dummy_serf.rb +70 -0
- data/spec/util/label_matcher_spec.rb +85 -0
- metadata +272 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 9a69092961cc7d8939847bd440a24c6a8447bed4
|
4
|
+
data.tar.gz: 4e33b7d99d62ddc876182e0c53e4b92f850215b5
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: edd97f707ef719186c9625b37e40d188df1dd8d9275331f87dceb71df01ef7bbb6a7434f91ccc5cf612e704bad30dcadc84c6d9b0dbc44c6977ffb783c379295
|
7
|
+
data.tar.gz: 6df6bfbe6585fa8c4813a59042aead942bef7f94cd5c69444898dcb60fa46b02457394713ff9c37f07487c6780ab431f4b1eceeb3560b4f5cc15fc1251e76913
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014 Shota Fukumori (sora_h)
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
# Mamiya - Faster deploy tool using tarballs and serf
|
2
|
+
|
3
|
+
[](https://travis-ci.org/sorah/mamiya)
|
4
|
+
|
5
|
+
Mamiya allows you to deploy using without ssh -- tarballs in such storage (Amazon S3), and [serf](http://www.serfdom.io/).
|
6
|
+
|
7
|
+
_Still developing_
|
8
|
+
|
9
|
+
## Problem for other deploy tool
|
10
|
+
|
11
|
+
Existing major deploy tools (capistrano, mina, ...) use SSH to operate servers.
|
12
|
+
But connecting to lot of servers via SSH makes deployment slow.
|
13
|
+
|
14
|
+
This solves such problem by using [Serf](http://www.serfdom.io/) and tarball on one storage (Amazon S3).
|
15
|
+
|
16
|
+
Also, I'm planning to allow to distribute files before the deploy command. I guess this can skip or shorten
|
17
|
+
file transferring phase in deploy.
|
18
|
+
|
19
|
+
## Installation
|
20
|
+
|
21
|
+
Add this line to your application's Gemfile:
|
22
|
+
|
23
|
+
gem 'mamiya'
|
24
|
+
|
25
|
+
And then execute:
|
26
|
+
|
27
|
+
$ bundle
|
28
|
+
|
29
|
+
Or install it yourself as:
|
30
|
+
|
31
|
+
$ gem install mamiya
|
32
|
+
|
33
|
+
## Usage
|
34
|
+
|
35
|
+
TODO: Write usage instructions here
|
36
|
+
|
37
|
+
## Contributing
|
38
|
+
|
39
|
+
1. Fork it ( http://github.com/sorah/mamiya/fork )
|
40
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
41
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
42
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
43
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
data/bin/mamiya
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
retried = false
|
4
|
+
begin
|
5
|
+
require 'mamiya'
|
6
|
+
require 'mamiya/cli'
|
7
|
+
rescue LoadError
|
8
|
+
$:.unshift File.expand_path(File.join(__dir__, '..', 'lib'))
|
9
|
+
if retried
|
10
|
+
raise
|
11
|
+
else
|
12
|
+
retried = true
|
13
|
+
retry
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
Mamiya::CLI.start(ARGV)
|
data/config.example.yml
ADDED
Binary file
|
@@ -0,0 +1,58 @@
|
|
1
|
+
@startuml
|
2
|
+
== Building ==
|
3
|
+
Master --> Master: Build package
|
4
|
+
note right of Master: e.g. assets precompile
|
5
|
+
Master --> Storage: Upload package
|
6
|
+
|
7
|
+
== Distributing ==
|
8
|
+
Master --> Client: Download request
|
9
|
+
activate Master
|
10
|
+
Master --> Master: Wait all clients to have package to deploy
|
11
|
+
|
12
|
+
alt if doesn't have package
|
13
|
+
activate Client
|
14
|
+
Client --> Client: Set status serf tag
|
15
|
+
Client --> Storage: HTTP request
|
16
|
+
Storage --> Client: Response package
|
17
|
+
Client --> Client: Set packages serf tag
|
18
|
+
Client --> Master: Report download finish
|
19
|
+
deactivate Client
|
20
|
+
end
|
21
|
+
|
22
|
+
deactivate Master
|
23
|
+
|
24
|
+
== Preparing ==
|
25
|
+
|
26
|
+
Master --> Client: Request to prepare package
|
27
|
+
activate Master
|
28
|
+
Master --> Master: Wait all clients to have package to deploy
|
29
|
+
activate Client
|
30
|
+
Client --> Client: Set status serf tag
|
31
|
+
Client --> Client: Unpack package
|
32
|
+
Client --> Client: Run prepare phase
|
33
|
+
note left of Client: e.g. bundle install, config files
|
34
|
+
Client --> Client: Set prepared serf tag
|
35
|
+
Client --> Client: Set status serf tag
|
36
|
+
Client --> Master: Report preparation finish
|
37
|
+
deactivate Client
|
38
|
+
deactivate Master
|
39
|
+
|
40
|
+
== Finalize ==
|
41
|
+
|
42
|
+
Master --> Client: Request finalize
|
43
|
+
activate Master
|
44
|
+
Master --> Master: Wait all clients restarted
|
45
|
+
activate Client
|
46
|
+
Client --> Client: Set status serf tag
|
47
|
+
Client --> Client: Run finalize phase
|
48
|
+
note left of Client: e.g. restart web server
|
49
|
+
Client --> Master: Report finalize complete
|
50
|
+
Client --> Client: Set status serf tag
|
51
|
+
Client --> Client: Wait to be reloaded
|
52
|
+
Client --> Client: Set status serf tag
|
53
|
+
Client --> Client: Set deployed serf tag
|
54
|
+
Client --> Master: Report deploy complete
|
55
|
+
deactivate Client
|
56
|
+
deactivate Master
|
57
|
+
|
58
|
+
@enduml
|
data/example.rb
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
# Setting variables
|
2
|
+
set :application, 'myapp'
|
3
|
+
set :repository, "some@where:repo.git"
|
4
|
+
set :ref, "master"
|
5
|
+
|
6
|
+
set :build_from, "/tmp/app-working-copy"
|
7
|
+
set :build_to, "/tmp/app-build"
|
8
|
+
set :package_under, 'suffix'
|
9
|
+
set :dereference_symlinks, true
|
10
|
+
|
11
|
+
set :package_to, "#{__dir__}/pkg"
|
12
|
+
set :prepare_to, "/home/app/app-prepare"
|
13
|
+
set :deploy_to, "/home/app/app"
|
14
|
+
|
15
|
+
set :discover_servers, false
|
16
|
+
set :on_client_failure, :warn
|
17
|
+
|
18
|
+
servers ec2.instances
|
19
|
+
filter_servers application, :production # to restrict where server to be used
|
20
|
+
|
21
|
+
use :git, skip_naming: true
|
22
|
+
use :rails
|
23
|
+
|
24
|
+
# chainable
|
25
|
+
package_name do
|
26
|
+
"#{full_ref}"
|
27
|
+
end
|
28
|
+
|
29
|
+
build(:prepend) do
|
30
|
+
# ...
|
31
|
+
end
|
32
|
+
|
33
|
+
prepare(only: :app) do
|
34
|
+
invoke :'bundle:install'
|
35
|
+
end
|
36
|
+
|
37
|
+
finalize(except: :web) do
|
38
|
+
run "unicorn", "graceful"
|
39
|
+
end
|
40
|
+
|
41
|
+
# finalize(:overwrite) do
|
42
|
+
# invoke :'deploy:switch_current'
|
43
|
+
# run "unicorn", "graceful"
|
44
|
+
# end
|
45
|
+
|
46
|
+
# ---- git.rb
|
47
|
+
|
48
|
+
set_default :branch, "master"
|
49
|
+
|
50
|
+
task :'git:clone' do
|
51
|
+
run 'git', 'clone', ...
|
52
|
+
end
|
53
|
+
|
54
|
+
task :'git:checkout' do
|
55
|
+
next if skip_checkout
|
56
|
+
# ...
|
57
|
+
end
|
58
|
+
|
59
|
+
unless options[:no_build_prepare]
|
60
|
+
prepare_build(:overwrite) do
|
61
|
+
invoke :'git:clone'
|
62
|
+
invoke :'git:checkout'
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# ---- rails.rb
|
67
|
+
|
68
|
+
task :'rails:assets_precompile' do
|
69
|
+
# ...
|
70
|
+
end
|
71
|
+
|
72
|
+
build do
|
73
|
+
invoke :'rails:assets_precompile'
|
74
|
+
end
|
data/lib/mamiya.rb
ADDED
data/lib/mamiya/agent.rb
ADDED
@@ -0,0 +1,181 @@
|
|
1
|
+
require 'thread'
|
2
|
+
require 'villein'
|
3
|
+
require 'mamiya/logger'
|
4
|
+
|
5
|
+
require 'mamiya/steps/fetch'
|
6
|
+
require 'mamiya/agent/fetcher'
|
7
|
+
|
8
|
+
require 'mamiya/agent/handlers/fetch'
|
9
|
+
require 'mamiya/agent/actions'
|
10
|
+
|
11
|
+
module Mamiya
|
12
|
+
class Agent
|
13
|
+
include Mamiya::Agent::Actions
|
14
|
+
FETCH_REMOVE_EVENT = 'mamiya:fetch-result:remove'
|
15
|
+
|
16
|
+
def initialize(config, logger: Mamiya::Logger.new, events_only: nil)
|
17
|
+
@config = config
|
18
|
+
@serf = init_serf
|
19
|
+
@events_only = events_only
|
20
|
+
|
21
|
+
@logger = logger['agent']
|
22
|
+
end
|
23
|
+
|
24
|
+
attr_reader :config, :serf, :logger
|
25
|
+
|
26
|
+
def fetcher
|
27
|
+
@fetcher ||= Mamiya::Agent::Fetcher.new(config).tap do |f|
|
28
|
+
f.cleanup_hook = self.method(:cleanup_handler)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def run!
|
33
|
+
logger.info "Starting..."
|
34
|
+
start()
|
35
|
+
logger.info "Started."
|
36
|
+
|
37
|
+
loop do
|
38
|
+
sleep 10
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def start
|
43
|
+
serf_start
|
44
|
+
fetcher_start
|
45
|
+
end
|
46
|
+
|
47
|
+
def update_tags!
|
48
|
+
serf.tags['mamiya'] = ','.tap do |status|
|
49
|
+
status.concat('fetching,') if fetcher.working?
|
50
|
+
status.concat('ready,') if status == ','
|
51
|
+
end
|
52
|
+
|
53
|
+
nil
|
54
|
+
end
|
55
|
+
|
56
|
+
##
|
57
|
+
# Returns agent status. Used for HTTP API and `serf query` inspection.
|
58
|
+
def status
|
59
|
+
{}.tap do |s|
|
60
|
+
s[:master] = false
|
61
|
+
s[:name] = serf.name
|
62
|
+
|
63
|
+
s[:fetcher] = {
|
64
|
+
fetching: fetcher.current_job,
|
65
|
+
pending: fetcher.queue_size,
|
66
|
+
}
|
67
|
+
|
68
|
+
s[:packages] = self.existing_packages
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
##
|
73
|
+
# Returns hash with existing packages (where valid) by app name.
|
74
|
+
# Packages which has json and tarball is considered as valid.
|
75
|
+
def existing_packages
|
76
|
+
paths_by_app = Dir[File.join(config[:packages_dir], '*', '*.{tar.gz,json}')].group_by { |path|
|
77
|
+
path.split('/')[-2]
|
78
|
+
}
|
79
|
+
|
80
|
+
Hash[
|
81
|
+
paths_by_app.map { |app, paths|
|
82
|
+
names_by_base = paths.group_by do |path|
|
83
|
+
File.basename(path).sub(/\.(?:tar\.gz|json)\z/, '')
|
84
|
+
end
|
85
|
+
|
86
|
+
packages = names_by_base.flat_map { |base, names|
|
87
|
+
names.map do |name|
|
88
|
+
(
|
89
|
+
name.end_with?(".tar.gz") &&
|
90
|
+
names.find { |_| _.end_with?(".json") } &&
|
91
|
+
base
|
92
|
+
) || nil
|
93
|
+
end
|
94
|
+
}.compact
|
95
|
+
|
96
|
+
[app, packages.sort]
|
97
|
+
}
|
98
|
+
]
|
99
|
+
end
|
100
|
+
|
101
|
+
def trigger(type, action: nil, **payload)
|
102
|
+
name = "mamiya:#{type}"
|
103
|
+
name << ":#{action}" if action
|
104
|
+
|
105
|
+
serf.event(name, payload.to_json)
|
106
|
+
end
|
107
|
+
|
108
|
+
private
|
109
|
+
|
110
|
+
def init_serf
|
111
|
+
agent_config = (config[:serf] && config[:serf][:agent]) || {}
|
112
|
+
Villein::Agent.new(**agent_config).tap do |serf|
|
113
|
+
serf.on_user_event do |event|
|
114
|
+
user_event_handler(event)
|
115
|
+
end
|
116
|
+
|
117
|
+
serf.respond('mamiya:status') do |event|
|
118
|
+
self.status.to_json
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def serf_start
|
124
|
+
logger.debug "Starting serf"
|
125
|
+
|
126
|
+
@serf.start!
|
127
|
+
@serf.auto_stop
|
128
|
+
@serf.wait_for_ready
|
129
|
+
|
130
|
+
logger.debug "Serf became ready"
|
131
|
+
end
|
132
|
+
|
133
|
+
def fetcher_start
|
134
|
+
logger.debug "Starting fetcher"
|
135
|
+
|
136
|
+
fetcher.start!
|
137
|
+
end
|
138
|
+
|
139
|
+
def user_event_handler(event)
|
140
|
+
user_event, payload = event.user_event, JSON.parse(event.payload)
|
141
|
+
|
142
|
+
return unless user_event.start_with?('mamiya:')
|
143
|
+
user_event = user_event.sub(/^mamiya:/, '')
|
144
|
+
|
145
|
+
type, action = user_event.split(/:/, 2)
|
146
|
+
|
147
|
+
return if @events_only && !@events_only.any?{ |_| _ === type }
|
148
|
+
|
149
|
+
class_name = type.capitalize.gsub(/-./) { |_| _[1].upcase }
|
150
|
+
|
151
|
+
logger.debug "Received user event #{type}"
|
152
|
+
logger.debug payload.inspect
|
153
|
+
|
154
|
+
if Handlers.const_defined?(class_name)
|
155
|
+
handler = Handlers.const_get(class_name).new(self, event)
|
156
|
+
handler.send(action || :run!)
|
157
|
+
else
|
158
|
+
logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
|
159
|
+
end
|
160
|
+
rescue Exception => e
|
161
|
+
logger.fatal("Error during handling event: #{e.inspect}")
|
162
|
+
e.backtrace.each do |line|
|
163
|
+
logger.fatal line.prepend("\t")
|
164
|
+
end
|
165
|
+
|
166
|
+
raise e if $0.end_with?('rspec')
|
167
|
+
rescue JSON::ParserError
|
168
|
+
logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
|
169
|
+
end
|
170
|
+
|
171
|
+
def cleanup_handler(app, package)
|
172
|
+
serf.event(FETCH_REMOVE_EVENT,
|
173
|
+
{
|
174
|
+
name: self.serf.name,
|
175
|
+
application: app,
|
176
|
+
package: package,
|
177
|
+
}.to_json
|
178
|
+
)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
end
|