mamiya 0.0.1.alpha2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +17 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +16 -0
  5. data/Gemfile +8 -0
  6. data/LICENSE.txt +22 -0
  7. data/README.md +43 -0
  8. data/Rakefile +6 -0
  9. data/bin/mamiya +17 -0
  10. data/config.example.yml +11 -0
  11. data/docs/sequences/deploy.png +0 -0
  12. data/docs/sequences/deploy.uml +58 -0
  13. data/example.rb +74 -0
  14. data/lib/mamiya.rb +5 -0
  15. data/lib/mamiya/agent.rb +181 -0
  16. data/lib/mamiya/agent/actions.rb +12 -0
  17. data/lib/mamiya/agent/fetcher.rb +137 -0
  18. data/lib/mamiya/agent/handlers/abstract.rb +20 -0
  19. data/lib/mamiya/agent/handlers/fetch.rb +68 -0
  20. data/lib/mamiya/cli.rb +322 -0
  21. data/lib/mamiya/cli/client.rb +172 -0
  22. data/lib/mamiya/config.rb +57 -0
  23. data/lib/mamiya/dsl.rb +192 -0
  24. data/lib/mamiya/helpers/git.rb +75 -0
  25. data/lib/mamiya/logger.rb +190 -0
  26. data/lib/mamiya/master.rb +118 -0
  27. data/lib/mamiya/master/agent_monitor.rb +146 -0
  28. data/lib/mamiya/master/agent_monitor_handlers.rb +44 -0
  29. data/lib/mamiya/master/web.rb +148 -0
  30. data/lib/mamiya/package.rb +122 -0
  31. data/lib/mamiya/script.rb +117 -0
  32. data/lib/mamiya/steps/abstract.rb +19 -0
  33. data/lib/mamiya/steps/build.rb +72 -0
  34. data/lib/mamiya/steps/extract.rb +26 -0
  35. data/lib/mamiya/steps/fetch.rb +24 -0
  36. data/lib/mamiya/steps/push.rb +34 -0
  37. data/lib/mamiya/storages.rb +17 -0
  38. data/lib/mamiya/storages/abstract.rb +48 -0
  39. data/lib/mamiya/storages/mock.rb +61 -0
  40. data/lib/mamiya/storages/s3.rb +127 -0
  41. data/lib/mamiya/util/label_matcher.rb +38 -0
  42. data/lib/mamiya/version.rb +3 -0
  43. data/mamiya.gemspec +35 -0
  44. data/misc/logger_test.rb +12 -0
  45. data/spec/agent/actions_spec.rb +37 -0
  46. data/spec/agent/fetcher_spec.rb +199 -0
  47. data/spec/agent/handlers/fetch_spec.rb +121 -0
  48. data/spec/agent_spec.rb +255 -0
  49. data/spec/config_spec.rb +50 -0
  50. data/spec/dsl_spec.rb +291 -0
  51. data/spec/fixtures/dsl_test_load.rb +1 -0
  52. data/spec/fixtures/dsl_test_use.rb +1 -0
  53. data/spec/fixtures/helpers/foo.rb +1 -0
  54. data/spec/fixtures/test-package-source/.mamiya.meta.json +1 -0
  55. data/spec/fixtures/test-package-source/greeting +1 -0
  56. data/spec/fixtures/test-package.tar.gz +0 -0
  57. data/spec/fixtures/test.yml +4 -0
  58. data/spec/logger_spec.rb +68 -0
  59. data/spec/master/agent_monitor_spec.rb +269 -0
  60. data/spec/master/web_spec.rb +121 -0
  61. data/spec/master_spec.rb +94 -0
  62. data/spec/package_spec.rb +394 -0
  63. data/spec/script_spec.rb +78 -0
  64. data/spec/spec_helper.rb +38 -0
  65. data/spec/steps/build_spec.rb +261 -0
  66. data/spec/steps/extract_spec.rb +68 -0
  67. data/spec/steps/fetch_spec.rb +96 -0
  68. data/spec/steps/push_spec.rb +73 -0
  69. data/spec/storages/abstract_spec.rb +22 -0
  70. data/spec/storages/s3_spec.rb +342 -0
  71. data/spec/storages_spec.rb +33 -0
  72. data/spec/support/dummy_serf.rb +70 -0
  73. data/spec/util/label_matcher_spec.rb +85 -0
  74. metadata +272 -0
@@ -0,0 +1,72 @@
1
+ require 'mamiya/steps/abstract'
2
+ require 'mamiya/package'
3
+
4
+ module Mamiya
5
+ module Steps
6
+ class Build < Abstract
7
+ def run!
8
+ @exception = nil
9
+
10
+ logger.info "Initiating package build"
11
+
12
+ logger.info "Running script.before_build"
13
+ script.before_build[]
14
+
15
+ unless script.skip_prepare_build
16
+ logger.info "Running script.prepare_build"
17
+ script.prepare_build[File.exists?(script.build_from)]
18
+ else
19
+ logger.debug "prepare_build skipped due to script.skip_prepare_build"
20
+ end
21
+
22
+ old_pwd = Dir.pwd
23
+ begin
24
+ # Using without block because chdir in block shows warning
25
+ Dir.chdir(script.build_from)
26
+ logger.info "Running script.build ..."
27
+ logger.debug "pwd=#{Dir.pwd}"
28
+ script.build[]
29
+ ensure
30
+ Dir.chdir old_pwd
31
+ end
32
+
33
+
34
+ logger.debug "Determining package name..."
35
+ package_name = Dir.chdir(script.build_from) {
36
+ script.package_name[
37
+ [Time.now.strftime("%Y-%m-%d_%H.%M.%S"), script.application]
38
+ ].join('-')
39
+ }
40
+ logger.info "Package name determined: #{package_name}"
41
+
42
+ package_path = File.join(script.build_to, package_name)
43
+ package = Mamiya::Package.new(package_path)
44
+ package.meta[:application] = script.application
45
+
46
+ Dir.chdir(script.build_from) do
47
+ package.meta.replace script.package_meta[package.meta]
48
+ end
49
+
50
+ logger.info "Packaging to: #{package.path}"
51
+ logger.debug "meta=#{package.meta.inspect}"
52
+ package.build!(script.build_from,
53
+ exclude_from_package: script.exclude_from_package || [],
54
+ dereference_symlinks: script.dereference_symlinks || false,
55
+ package_under: script.package_under || nil,
56
+ logger: logger)
57
+ logger.info "Packed."
58
+
59
+ rescue Exception => e
60
+ @exception = e
61
+ raise
62
+ ensure
63
+ logger.warn "Exception occured, cleaning up..." if @exception
64
+
65
+ logger.info "Running script.after_build"
66
+ script.after_build[@exception]
67
+
68
+ logger.info "DONE!" unless @exception
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,26 @@
1
+ require 'mamiya/steps/abstract'
2
+
3
+
4
+ module Mamiya
5
+ module Steps
6
+ class Extract < Abstract
7
+ def run!
8
+ package = case options[:package]
9
+ when Mamiya::Package
10
+ options[:package]
11
+ else
12
+ Mamiya::Package.new(options[:package])
13
+ end
14
+
15
+ if File.exists?(options[:destination])
16
+ destination = File.join(options[:destination], package.name)
17
+ else
18
+ destination = options[:destination]
19
+ end
20
+
21
+ logger.info "Extracting #{package.path} onto #{destination}"
22
+ package.extract_onto!(destination)
23
+ end
24
+ end
25
+ end
26
+ end
@@ -0,0 +1,24 @@
1
+ require 'mamiya/steps/abstract'
2
+ require 'mamiya/package'
3
+
4
+ module Mamiya
5
+ module Steps
6
+ class Fetch < Abstract
7
+ def run!
8
+ application = options[:application] || (script && script.application)
9
+
10
+ raise 'no application name given' unless application
11
+
12
+ storage = config.storage_class.new(
13
+ config[:storage].merge(
14
+ application: application
15
+ )
16
+ )
17
+
18
+ logger.info("Fetching package #{options[:package]} from storage(app=#{storage.application}) to #{options[:destination]}...")
19
+ storage.fetch(options[:package], options[:destination])
20
+ end
21
+ end
22
+ end
23
+ end
24
+
@@ -0,0 +1,34 @@
1
+ require 'mamiya/steps/abstract'
2
+ require 'mamiya/package'
3
+
4
+ module Mamiya
5
+ module Steps
6
+ class Push < Abstract
7
+ def run!
8
+ package = case options[:package]
9
+ when Mamiya::Package
10
+ options[:package]
11
+ else
12
+ Mamiya::Package.new(options[:package])
13
+ end
14
+
15
+ application = options[:application] || package.application
16
+
17
+ raise 'no application name given' unless application
18
+
19
+ storage = config.storage_class.new(
20
+ config[:storage].merge(
21
+ application: application
22
+ )
23
+ )
24
+
25
+ logger.info "Pushing #{package.path} to storage(app=#{storage.application})..."
26
+
27
+ storage.push(package)
28
+
29
+ logger.info "DONE!"
30
+ end
31
+ end
32
+ end
33
+ end
34
+
@@ -0,0 +1,17 @@
1
+ module Mamiya
2
+ module Storages
3
+ def self.find(name)
4
+ name = name.to_s
5
+ classish_name = name.capitalize.gsub(/_./) { |s| s[1].upcase }
6
+
7
+ begin
8
+ return const_get(classish_name)
9
+ rescue NameError; end
10
+
11
+ require "mamiya/storages/#{File.basename(name)}"
12
+ const_get(classish_name)
13
+ rescue NameError, LoadError
14
+ return nil
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,48 @@
1
+ module Mamiya
2
+ module Storages
3
+ class Abstract
4
+ class NotBuilt < Exception; end
5
+ class NotFound < Exception; end
6
+ class AlreadyExists < Exception; end
7
+ class AlreadyFetched < Exception; end
8
+
9
+ def initialize(config = {})
10
+ @config = config.dup
11
+ @application = config.delete(:application)
12
+ end
13
+
14
+ attr_reader :config, :application
15
+
16
+ def self.find(config={})
17
+ {}
18
+ end
19
+
20
+ def packages
21
+ []
22
+ end
23
+
24
+ def push(package)
25
+ raise NotImplementedError
26
+ end
27
+
28
+ def fetch(package_name, dir)
29
+ raise NotImplementedError
30
+ end
31
+
32
+ def meta(package_name)
33
+ raise NotImplementedError
34
+ end
35
+
36
+ def remove(package)
37
+ raise NotImplementedError
38
+ end
39
+
40
+ def prune(nums_to_keep)
41
+ packages = self.packages()
42
+ (packages - packages.last(nums_to_keep)).each do |package|
43
+ self.remove(package)
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,61 @@
1
+ require 'mamiya/package'
2
+ require 'mamiya/storages/abstract'
3
+ require 'fileutils'
4
+
5
+ module Mamiya
6
+ module Storages
7
+ class Mock < Mamiya::Storages::Abstract
8
+ def self.storage
9
+ @storage ||= {}
10
+ end
11
+
12
+ def self.clear
13
+ @storage = {}
14
+ end
15
+
16
+ def self.find(config={})
17
+ storage.keys
18
+ end
19
+
20
+ def initialize(*)
21
+ super
22
+ self.class.storage[application] ||= {}
23
+ end
24
+
25
+ def packages
26
+ self.class.storage[application].keys
27
+ end
28
+
29
+ def push(package)
30
+ raise TypeError, "package should be a kind of Mamiya::Package" unless package.kind_of?(Mamiya::Package)
31
+ raise NotBuilt, "package not built" unless package.exists?
32
+ self.class.storage[application] ||= {}
33
+ raise AlreadyExists if self.class.storage[application][package.name]
34
+ self.class.storage[application][package.name] = {package: package, config: config}
35
+ end
36
+
37
+ def fetch(package_name, destination)
38
+ self.class.storage[application] ||= {}
39
+ raise NotFound unless self.class.storage[application][package_name]
40
+ package_path = File.join(destination, "#{package_name}.tar.gz")
41
+ meta_path = File.join(destination, "#{package_name}.json")
42
+
43
+ if File.exists?(package_path) || File.exists?(meta_path)
44
+ raise AlreadyFetched
45
+ end
46
+
47
+ package = self.class.storage[application][package_name][:package]
48
+ FileUtils.cp package.path, package_path
49
+ FileUtils.cp package.meta_path, meta_path
50
+ return package
51
+ end
52
+
53
+ def meta(package_name)
54
+ package = self.class.storage[application][package_name]
55
+ return unless package
56
+
57
+ package[:package].meta
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,127 @@
1
+ require 'mamiya/package'
2
+ require 'mamiya/storages/abstract'
3
+ require 'aws-sdk-core'
4
+ require 'json'
5
+
6
+ module Mamiya
7
+ module Storages
8
+ class S3 < Mamiya::Storages::Abstract
9
+ def self.find(config={})
10
+ s3 = initiate_s3_with_config(config)
11
+ Hash[s3.list_objects(bucket: config[:bucket], delimiter: '/').common_prefixes.map { |prefix|
12
+ app = prefix.prefix.gsub(%r{/$},'')
13
+ [app, self.new(config.merge(application: app))]
14
+ }]
15
+ end
16
+
17
+ def packages
18
+ s3.list_objects(bucket: @config[:bucket], delimiter: '/', prefix: "#{self.application}/").contents.map { |content|
19
+ content.key.sub(/\A#{Regexp.escape(self.application)}\//, '')
20
+ }.group_by { |key|
21
+ key.sub(Package::PATH_SUFFIXES,'')
22
+ }.select { |key, files|
23
+ files.find { |file| file.end_with?('.tar.gz') } && files.find { |file| file.end_with?('.json') }
24
+ }.keys
25
+ end
26
+
27
+ def push(package)
28
+ raise TypeError, "package should be a kind of Mamiya::Package" unless package.kind_of?(Mamiya::Package)
29
+ raise NotBuilt, "package not built" unless package.exists?
30
+
31
+ package_key, meta_key = package_and_meta_key_for(package.name)
32
+
33
+ [package_key, meta_key].each do |key|
34
+ raise AlreadyExists if key_exists_in_s3?(key)
35
+ end
36
+
37
+ open(package.path, 'rb') do |io|
38
+ s3.put_object(bucket: @config[:bucket], key: package_key, body: io)
39
+ end
40
+ open(package.meta_path, 'rb') do |io|
41
+ s3.put_object(bucket: @config[:bucket], key: meta_key, body: io)
42
+ end
43
+ end
44
+
45
+ def fetch(package_name, dir)
46
+ package_key, meta_key = package_and_meta_key_for(package_name)
47
+
48
+ package_path = File.join(dir, File.basename(package_key))
49
+ meta_path = File.join(dir, File.basename(meta_key))
50
+
51
+ if File.exists?(package_path) || File.exists?(meta_path)
52
+ raise AlreadyFetched
53
+ end
54
+
55
+ open(package_path, 'wb+') do |io|
56
+ s3.get_object({bucket: @config[:bucket], key: package_key}, target: io)
57
+ end
58
+ open(meta_path, 'wb+') do |io|
59
+ s3.get_object({bucket: @config[:bucket], key: meta_key}, target: io)
60
+ end
61
+
62
+ return Mamiya::Package.new(package_path)
63
+ rescue Aws::S3::Errors::NoSuchKey
64
+ File.unlink package_path if package_path && File.exists?(package_path)
65
+ File.unlink meta_path if meta_path && File.exists?(meta_path)
66
+
67
+ raise NotFound
68
+ rescue Exception => e
69
+ File.unlink package_path if package_path && File.exists?(package_path)
70
+ File.unlink meta_path if meta_path && File.exists?(meta_path)
71
+
72
+ raise e
73
+ end
74
+
75
+ def meta(package_name)
76
+ _, meta_key = package_and_meta_key_for(package_name)
77
+ JSON.parse(s3.get_object(bucket: @config[:bucket], key: meta_key).body.string)
78
+ rescue Aws::S3::Errors::NoSuchKey
79
+ return nil
80
+ end
81
+
82
+ def remove(package_name)
83
+ package_key, meta_key = package_and_meta_key_for(package_name)
84
+
85
+ objs_to_delete = [package_key, meta_key].map { |key|
86
+ if key_exists_in_s3?(key)
87
+ {key: key}
88
+ else
89
+ nil
90
+ end
91
+ }.compact
92
+ raise NotFound if objs_to_delete.empty?
93
+
94
+ s3.delete_objects(bucket: @config[:bucket], objects: objs_to_delete)
95
+ end
96
+
97
+ def self.initiate_s3_with_config(config) # :nodoc:
98
+ s3_config = config.dup
99
+ s3_config.delete(:bucket)
100
+ s3_config.delete(:application)
101
+ s3_config.delete(:type)
102
+ Aws::S3.new(s3_config)
103
+ end
104
+
105
+ private
106
+
107
+ def s3
108
+ @s3 ||= self.class.initiate_s3_with_config(@config)
109
+ end
110
+
111
+ def package_and_meta_key_for(package_name)
112
+ package_name = package_name.sub(/\.(?:tar\.gz|json)\z/, '')
113
+ ["#{self.application}/#{package_name}.tar.gz", "#{self.application}/#{package_name}.json"]
114
+ end
115
+
116
+ def key_exists_in_s3?(key)
117
+ begin
118
+ if s3.head_object(bucket: @config[:bucket], key: key)
119
+ true
120
+ end
121
+ rescue Aws::S3::Errors::NotFound
122
+ false
123
+ end
124
+ end
125
+ end
126
+ end
127
+ end
@@ -0,0 +1,38 @@
1
+ module Mamiya
2
+ module Util
3
+ module LabelMatcher
4
+ def match?(*expressions)
5
+ labels = self.labels()
6
+
7
+ if expressions.all? { |_| _.kind_of?(Symbol) || _.kind_of?(String) }
8
+ return self.match?(expressions)
9
+ end
10
+
11
+ expressions.any? do |expression|
12
+ case expression
13
+ when Symbol, String
14
+ labels.include?(expression)
15
+ when Array
16
+ if expression.any? { |_| _.kind_of?(Array) }
17
+ self.match?(*expression)
18
+ else
19
+ expression.all? { |_| labels.include?(_) }
20
+ end
21
+ end
22
+ end
23
+ end
24
+
25
+ alias matches? match?
26
+
27
+ class Simple
28
+ def initialize(*labels)
29
+ @labels = labels.flatten
30
+ end
31
+
32
+ attr_reader :labels
33
+
34
+ include LabelMatcher
35
+ end
36
+ end
37
+ end
38
+ end