mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 9a69092961cc7d8939847bd440a24c6a8447bed4
4
+ data.tar.gz: 4e33b7d99d62ddc876182e0c53e4b92f850215b5
5
+ SHA512:
6
+ metadata.gz: edd97f707ef719186c9625b37e40d188df1dd8d9275331f87dceb71df01ef7bbb6a7434f91ccc5cf612e704bad30dcadc84c6d9b0dbc44c6977ffb783c379295
7
+ data.tar.gz: 6df6bfbe6585fa8c4813a59042aead942bef7f94cd5c69444898dcb60fa46b02457394713ff9c37f07487c6780ab431f4b1eceeb3560b4f5cc15fc1251e76913
data/.gitignore ADDED
@@ -0,0 +1,17 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/.travis.yml ADDED
@@ -0,0 +1,16 @@
1
+ language: ruby
2
+ cache: bundler
3
+ rvm:
4
+ - "2.1.1"
5
+ - "2.0.0"
6
+ - "ruby-head"
7
+
8
+ matrix:
9
+ allow_failures:
10
+ - rvm:
11
+ - "ruby-head"
12
+ fast_finish: true
13
+ notifications:
14
+ email:
15
+ - travis-ci@sorah.jp
16
+ script: bundle exec rspec -fd ./spec
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ source 'https://rubygems.org'
2
+
3
+ if ENV["MAMIYA_VILLEIN_PATH"]
4
+ gem 'villein', path: File.expand_path(ENV["MAMIYA_VILLEIN_PATH"])
5
+ end
6
+
7
+ # Specify your gem's dependencies in mamiya.gemspec
8
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Shota Fukumori (sora_h)
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,43 @@
1
+ # Mamiya - Faster deploy tool using tarballs and serf
2
+
3
+ [![Build Status](https://travis-ci.org/sorah/mamiya.png?branch=master)](https://travis-ci.org/sorah/mamiya)
4
+
5
+ Mamiya allows you to deploy using without ssh -- tarballs in such storage (Amazon S3), and [serf](http://www.serfdom.io/).
6
+
7
+ _Still developing_
8
+
9
+ ## Problem for other deploy tool
10
+
11
+ Existing major deploy tools (capistrano, mina, ...) use SSH to operate servers.
12
+ But connecting to lot of servers via SSH makes deployment slow.
13
+
14
+ This solves such problem by using [Serf](http://www.serfdom.io/) and tarball on one storage (Amazon S3).
15
+
16
+ Also, I'm planning to allow to distribute files before the deploy command. I guess this can skip or shorten
17
+ file transferring phase in deploy.
18
+
19
+ ## Installation
20
+
21
+ Add this line to your application's Gemfile:
22
+
23
+ gem 'mamiya'
24
+
25
+ And then execute:
26
+
27
+ $ bundle
28
+
29
+ Or install it yourself as:
30
+
31
+ $ gem install mamiya
32
+
33
+ ## Usage
34
+
35
+ TODO: Write usage instructions here
36
+
37
+ ## Contributing
38
+
39
+ 1. Fork it ( http://github.com/sorah/mamiya/fork )
40
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
41
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
42
+ 4. Push to the branch (`git push origin my-new-feature`)
43
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rspec/core/rake_task'
3
+
4
+ RSpec::Core::RakeTask.new(:spec)
5
+
6
+ task :default => :spec
data/bin/mamiya ADDED
@@ -0,0 +1,17 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ retried = false
4
+ begin
5
+ require 'mamiya'
6
+ require 'mamiya/cli'
7
+ rescue LoadError
8
+ $:.unshift File.expand_path(File.join(__dir__, '..', 'lib'))
9
+ if retried
10
+ raise
11
+ else
12
+ retried = true
13
+ retry
14
+ end
15
+ end
16
+
17
+ Mamiya::CLI.start(ARGV)
@@ -0,0 +1,11 @@
1
+ storage:
2
+ type: s3
3
+ bucket: ...
4
+ region: ap-northeast-1
5
+
6
+ deploy_script_dir: /path/to/deploy/scripts
7
+ packages_dir: ''
8
+
9
+ apps:
10
+ myapp:
11
+ deploy_to: /path/to/myapp
Binary file
@@ -0,0 +1,58 @@
1
+ @startuml
2
+ == Building ==
3
+ Master --> Master: Build package
4
+ note right of Master: e.g. assets precompile
5
+ Master --> Storage: Upload package
6
+
7
+ == Distributing ==
8
+ Master --> Client: Download request
9
+ activate Master
10
+ Master --> Master: Wait all clients to have package to deploy
11
+
12
+ alt if doesn't have package
13
+ activate Client
14
+ Client --> Client: Set status serf tag
15
+ Client --> Storage: HTTP request
16
+ Storage --> Client: Response package
17
+ Client --> Client: Set packages serf tag
18
+ Client --> Master: Report download finish
19
+ deactivate Client
20
+ end
21
+
22
+ deactivate Master
23
+
24
+ == Preparing ==
25
+
26
+ Master --> Client: Request to prepare package
27
+ activate Master
28
+ Master --> Master: Wait all clients to have package to deploy
29
+ activate Client
30
+ Client --> Client: Set status serf tag
31
+ Client --> Client: Unpack package
32
+ Client --> Client: Run prepare phase
33
+ note left of Client: e.g. bundle install, config files
34
+ Client --> Client: Set prepared serf tag
35
+ Client --> Client: Set status serf tag
36
+ Client --> Master: Report preparation finish
37
+ deactivate Client
38
+ deactivate Master
39
+
40
+ == Finalize ==
41
+
42
+ Master --> Client: Request finalize
43
+ activate Master
44
+ Master --> Master: Wait all clients restarted
45
+ activate Client
46
+ Client --> Client: Set status serf tag
47
+ Client --> Client: Run finalize phase
48
+ note left of Client: e.g. restart web server
49
+ Client --> Master: Report finalize complete
50
+ Client --> Client: Set status serf tag
51
+ Client --> Client: Wait to be reloaded
52
+ Client --> Client: Set status serf tag
53
+ Client --> Client: Set deployed serf tag
54
+ Client --> Master: Report deploy complete
55
+ deactivate Client
56
+ deactivate Master
57
+
58
+ @enduml
data/example.rb ADDED
@@ -0,0 +1,74 @@
1
+ # Setting variables
2
+ set :application, 'myapp'
3
+ set :repository, "some@where:repo.git"
4
+ set :ref, "master"
5
+
6
+ set :build_from, "/tmp/app-working-copy"
7
+ set :build_to, "/tmp/app-build"
8
+ set :package_under, 'suffix'
9
+ set :dereference_symlinks, true
10
+
11
+ set :package_to, "#{__dir__}/pkg"
12
+ set :prepare_to, "/home/app/app-prepare"
13
+ set :deploy_to, "/home/app/app"
14
+
15
+ set :discover_servers, false
16
+ set :on_client_failure, :warn
17
+
18
+ servers ec2.instances
19
+ filter_servers application, :production # to restrict where server to be used
20
+
21
+ use :git, skip_naming: true
22
+ use :rails
23
+
24
+ # chainable
25
+ package_name do
26
+ "#{full_ref}"
27
+ end
28
+
29
+ build(:prepend) do
30
+ # ...
31
+ end
32
+
33
+ prepare(only: :app) do
34
+ invoke :'bundle:install'
35
+ end
36
+
37
+ finalize(except: :web) do
38
+ run "unicorn", "graceful"
39
+ end
40
+
41
+ # finalize(:overwrite) do
42
+ # invoke :'deploy:switch_current'
43
+ # run "unicorn", "graceful"
44
+ # end
45
+
46
+ # ---- git.rb
47
+
48
+ set_default :branch, "master"
49
+
50
+ task :'git:clone' do
51
+ run 'git', 'clone', ...
52
+ end
53
+
54
+ task :'git:checkout' do
55
+ next if skip_checkout
56
+ # ...
57
+ end
58
+
59
+ unless options[:no_build_prepare]
60
+ prepare_build(:overwrite) do
61
+ invoke :'git:clone'
62
+ invoke :'git:checkout'
63
+ end
64
+ end
65
+
66
+ # ---- rails.rb
67
+
68
+ task :'rails:assets_precompile' do
69
+ # ...
70
+ end
71
+
72
+ build do
73
+ invoke :'rails:assets_precompile'
74
+ end
data/lib/mamiya.rb ADDED
@@ -0,0 +1,5 @@
1
+ require "mamiya/version"
2
+
3
+ module Mamiya
4
+ # Your code goes here...
5
+ end
@@ -0,0 +1,181 @@
1
+ require 'thread'
2
+ require 'villein'
3
+ require 'mamiya/logger'
4
+
5
+ require 'mamiya/steps/fetch'
6
+ require 'mamiya/agent/fetcher'
7
+
8
+ require 'mamiya/agent/handlers/fetch'
9
+ require 'mamiya/agent/actions'
10
+
11
+ module Mamiya
12
+ class Agent
13
+ include Mamiya::Agent::Actions
14
+ FETCH_REMOVE_EVENT = 'mamiya:fetch-result:remove'
15
+
16
+ def initialize(config, logger: Mamiya::Logger.new, events_only: nil)
17
+ @config = config
18
+ @serf = init_serf
19
+ @events_only = events_only
20
+
21
+ @logger = logger['agent']
22
+ end
23
+
24
+ attr_reader :config, :serf, :logger
25
+
26
+ def fetcher
27
+ @fetcher ||= Mamiya::Agent::Fetcher.new(config).tap do |f|
28
+ f.cleanup_hook = self.method(:cleanup_handler)
29
+ end
30
+ end
31
+
32
+ def run!
33
+ logger.info "Starting..."
34
+ start()
35
+ logger.info "Started."
36
+
37
+ loop do
38
+ sleep 10
39
+ end
40
+ end
41
+
42
+ def start
43
+ serf_start
44
+ fetcher_start
45
+ end
46
+
47
+ def update_tags!
48
+ serf.tags['mamiya'] = ','.tap do |status|
49
+ status.concat('fetching,') if fetcher.working?
50
+ status.concat('ready,') if status == ','
51
+ end
52
+
53
+ nil
54
+ end
55
+
56
+ ##
57
+ # Returns agent status. Used for HTTP API and `serf query` inspection.
58
+ def status
59
+ {}.tap do |s|
60
+ s[:master] = false
61
+ s[:name] = serf.name
62
+
63
+ s[:fetcher] = {
64
+ fetching: fetcher.current_job,
65
+ pending: fetcher.queue_size,
66
+ }
67
+
68
+ s[:packages] = self.existing_packages
69
+ end
70
+ end
71
+
72
+ ##
73
+ # Returns hash with existing packages (where valid) by app name.
74
+ # Packages which has json and tarball is considered as valid.
75
+ def existing_packages
76
+ paths_by_app = Dir[File.join(config[:packages_dir], '*', '*.{tar.gz,json}')].group_by { |path|
77
+ path.split('/')[-2]
78
+ }
79
+
80
+ Hash[
81
+ paths_by_app.map { |app, paths|
82
+ names_by_base = paths.group_by do |path|
83
+ File.basename(path).sub(/\.(?:tar\.gz|json)\z/, '')
84
+ end
85
+
86
+ packages = names_by_base.flat_map { |base, names|
87
+ names.map do |name|
88
+ (
89
+ name.end_with?(".tar.gz") &&
90
+ names.find { |_| _.end_with?(".json") } &&
91
+ base
92
+ ) || nil
93
+ end
94
+ }.compact
95
+
96
+ [app, packages.sort]
97
+ }
98
+ ]
99
+ end
100
+
101
+ def trigger(type, action: nil, **payload)
102
+ name = "mamiya:#{type}"
103
+ name << ":#{action}" if action
104
+
105
+ serf.event(name, payload.to_json)
106
+ end
107
+
108
+ private
109
+
110
+ def init_serf
111
+ agent_config = (config[:serf] && config[:serf][:agent]) || {}
112
+ Villein::Agent.new(**agent_config).tap do |serf|
113
+ serf.on_user_event do |event|
114
+ user_event_handler(event)
115
+ end
116
+
117
+ serf.respond('mamiya:status') do |event|
118
+ self.status.to_json
119
+ end
120
+ end
121
+ end
122
+
123
+ def serf_start
124
+ logger.debug "Starting serf"
125
+
126
+ @serf.start!
127
+ @serf.auto_stop
128
+ @serf.wait_for_ready
129
+
130
+ logger.debug "Serf became ready"
131
+ end
132
+
133
+ def fetcher_start
134
+ logger.debug "Starting fetcher"
135
+
136
+ fetcher.start!
137
+ end
138
+
139
+ def user_event_handler(event)
140
+ user_event, payload = event.user_event, JSON.parse(event.payload)
141
+
142
+ return unless user_event.start_with?('mamiya:')
143
+ user_event = user_event.sub(/^mamiya:/, '')
144
+
145
+ type, action = user_event.split(/:/, 2)
146
+
147
+ return if @events_only && !@events_only.any?{ |_| _ === type }
148
+
149
+ class_name = type.capitalize.gsub(/-./) { |_| _[1].upcase }
150
+
151
+ logger.debug "Received user event #{type}"
152
+ logger.debug payload.inspect
153
+
154
+ if Handlers.const_defined?(class_name)
155
+ handler = Handlers.const_get(class_name).new(self, event)
156
+ handler.send(action || :run!)
157
+ else
158
+ logger.warn("Discarded event[#{event.user_event}] because we don't handle it")
159
+ end
160
+ rescue Exception => e
161
+ logger.fatal("Error during handling event: #{e.inspect}")
162
+ e.backtrace.each do |line|
163
+ logger.fatal line.prepend("\t")
164
+ end
165
+
166
+ raise e if $0.end_with?('rspec')
167
+ rescue JSON::ParserError
168
+ logger.warn("Discarded event[#{event.user_event}] with invalid payload (unable to parse as json)")
169
+ end
170
+
171
+ def cleanup_handler(app, package)
172
+ serf.event(FETCH_REMOVE_EVENT,
173
+ {
174
+ name: self.serf.name,
175
+ application: app,
176
+ package: package,
177
+ }.to_json
178
+ )
179
+ end
180
+ end
181
+ end